diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index d14974679..37998395d 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -38,8 +38,9 @@ emqx_test(){ packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.zip) unzip -q "${PACKAGE_PATH}/${packagename}" export EMQX_ZONE__EXTERNAL__SERVER_KEEPALIVE=60 \ - EMQX_MQTT__MAX_TOPIC_ALIAS=10 - sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins + EMQX_MQTT__MAX_TOPIC_ALIAS=10 + [[ $(arch) == *arm* || $(arch) == aarch64 ]] && export EMQX_ZONES__DEFAULT__LISTENERS__MQTT_QUIC__ENABLED=false + # sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins echo "running ${packagename} start" if ! "${PACKAGE_PATH}"/emqx/bin/emqx start; then @@ -48,7 +49,7 @@ emqx_test(){ exit 1 fi IDLE_TIME=0 - while ! curl http://localhost:8081/status >/dev/null 2>&1; do + while ! curl http://localhost:8081/api/v5/status >/dev/null 2>&1; do if [ $IDLE_TIME -gt 10 ] then echo "emqx running error" @@ -113,17 +114,31 @@ emqx_test(){ } running_test(){ - export EMQX_ZONE__EXTERNAL__SERVER_KEEPALIVE=60 \ - EMQX_MQTT__MAX_TOPIC_ALIAS=10 - sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins + # sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins + emqx_env_vars=$(dirname "$(readlink "$(command -v emqx)")")/../releases/emqx_vars - if ! emqx start; then + if [ -f "$emqx_env_vars" ]; + then + tee -a "$emqx_env_vars" </dev/null 2>&1; do + while ! curl http://localhost:8081/api/v5/status >/dev/null 2>&1; do if [ $IDLE_TIME -gt 10 ] then echo "emqx running error" @@ -138,14 +153,13 @@ running_test(){ if [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = ubuntu ] \ || [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = debian ] ;then - if ! service emqx start; then cat /var/log/emqx/erlang.log.1 || true cat /var/log/emqx/emqx.log.1 || true exit 1 fi IDLE_TIME=0 - while ! curl http://localhost:8081/status >/dev/null 2>&1; do + while ! curl http://localhost:8081/api/v5/status >/dev/null 2>&1; do if [ $IDLE_TIME -gt 10 ] then echo "emqx service error" diff --git a/.ci/docker-compose-file/conf.cluster.env b/.ci/docker-compose-file/conf.cluster.env index d8294a785..2a10bd321 100644 --- a/.ci/docker-compose-file/conf.cluster.env +++ b/.ci/docker-compose-file/conf.cluster.env @@ -1,7 +1,8 @@ EMQX_NAME=emqx -EMQX_CLUSTER__DISCOVERY=static -EMQX_CLUSTER__STATIC__SEEDS="emqx@node1.emqx.io, emqx@node2.emqx.io" -EMQX_LISTENER__TCP__EXTERNAL__PROXY_PROTOCOL=on -EMQX_LISTENER__WS__EXTERNAL__PROXY_PROTOCOL=on -EMQX_LOG__LEVEL=debug -EMQX_LOADED_PLUGINS=emqx_sn +EMQX_CLUSTER__DISCOVERY_STRATEGY=static +EMQX_CLUSTER__STATIC__SEEDS="[emqx@node1.emqx.io, emqx@node2.emqx.io]" +EMQX_ZONES__DEFAULT__LISTENERS__MQTT_TCP__PROXY_PROTOCOL=true +EMQX_ZONES__DEFAULT__LISTENERS__MQTT_WS__PROXY_PROTOCOL=true +EMQX_LOG__CONSOLE_HANDLER__ENABLE=true +EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug +EMQX_LOG__PRIMARY_LEVEL=debug diff --git a/.ci/docker-compose-file/conf.env b/.ci/docker-compose-file/conf.env index 0b1b7c512..d80c79b12 100644 --- a/.ci/docker-compose-file/conf.env +++ b/.ci/docker-compose-file/conf.env @@ -10,5 +10,4 @@ EMQX_AUTH__PGSQL__PASSWORD=public EMQX_AUTH__PGSQL__DATABASE=mqtt EMQX_AUTH__REDIS__SERVER=redis_server:6379 EMQX_AUTH__REDIS__PASSWORD=public -CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ HOCON_ENV_OVERRIDE_PREFIX=EMQX_ diff --git a/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml index 6bc8e67e2..e810e77c3 100644 --- a/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml +++ b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml @@ -33,13 +33,6 @@ services: - conf.cluster.env environment: - "EMQX_HOST=node1.emqx.io" - command: - - /bin/sh - - -c - - | - sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf - sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins - /opt/emqx/bin/emqx foreground healthcheck: test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"] interval: 5s @@ -57,13 +50,6 @@ services: - conf.cluster.env environment: - "EMQX_HOST=node2.emqx.io" - command: - - /bin/sh - - -c - - | - sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf - sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins - /opt/emqx/bin/emqx foreground healthcheck: test: ["CMD", "/opt/emqx/bin/emqx", "ping"] interval: 5s diff --git a/.ci/docker-compose-file/docker-compose-mongo-replicaset-tcp.yaml b/.ci/docker-compose-file/docker-compose-mongo-replicaset-tcp.yaml new file mode 100644 index 000000000..f83fe0932 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-mongo-replicaset-tcp.yaml @@ -0,0 +1,81 @@ +version: "3" + +services: + mongo1: + hostname: mongo1 + container_name: mongo1 + image: mongo:${MONGO_TAG} + environment: + MONGO_INITDB_DATABASE: mqtt + networks: + - emqx_bridge + expose: + - 27017 + ports: + - 27011:27017 + restart: always + command: + --ipv6 + --bind_ip_all + --replSet rs0 + + mongo2: + hostname: mongo2 + container_name: mongo2 + image: mongo:${MONGO_TAG} + environment: + MONGO_INITDB_DATABASE: mqtt + networks: + - emqx_bridge + expose: + - 27017 + ports: + - 27012:27017 + restart: always + command: + --ipv6 + --bind_ip_all + --replSet rs0 + + mongo3: + hostname: mongo3 + container_name: mongo3 + image: mongo:${MONGO_TAG} + environment: + MONGO_INITDB_DATABASE: mqtt + networks: + - emqx_bridge + expose: + - 27017 + ports: + - 27013:27017 + restart: always + command: + --ipv6 + --bind_ip_all + --replSet rs0 + + mongo_client: + image: mongo:${MONGO_TAG} + container_name: mongo_client + networks: + - emqx_bridge + depends_on: + - mongo1 + - mongo2 + - mongo3 + command: + - /bin/bash + - -c + - | + while ! mongo --host mongo1 --eval 'db.runCommand("ping").ok' --quiet > /dev/null 2>&1; do + sleep 1 + done + while ! mongo --host mongo2 --eval 'db.runCommand("ping").ok' --quiet > /dev/null 2>&1; do + sleep 1 + done + while ! mongo --host mongo3 --eval 'db.runCommand("ping").ok' --quiet > /dev/null 2>&1; do + sleep 1 + done + mongo --host mongo1 --eval "rs.initiate( { _id : 'rs0', members: [ { _id : 0, host : 'mongo1:27017' }, { _id : 1, host : 'mongo2:27017' }, { _id : 2, host : 'mongo3:27017' } ] })" --quiet + mongo --host mongo1 --eval "rs.status()" --quiet diff --git a/.ci/docker-compose-file/docker-compose-mongo-replicaset-tls.yaml b/.ci/docker-compose-file/docker-compose-mongo-replicaset-tls.yaml new file mode 100644 index 000000000..be8f7ea21 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-mongo-replicaset-tls.yaml @@ -0,0 +1,98 @@ +version: "3" + +services: + mongo1: + hostname: mongo1 + container_name: mongo1 + image: mongo:${MONGO_TAG} + environment: + MONGO_INITDB_DATABASE: mqtt + networks: + - emqx_bridge + expose: + - 27017 + ports: + - 27011:27017 + restart: always + volumes: + - ../../apps/emqx/etc/certs/cert.pem:/etc/certs/cert.pem + - ../../apps/emqx/etc/certs/key.pem:/etc/certs/key.pem + command: + - /bin/bash + - -c + - | + cat /etc/certs/key.pem /etc/certs/cert.pem > /etc/certs/mongodb.pem + mongod --ipv6 --bind_ip_all --tlsMode requireTLS --tlsCertificateKeyFile /etc/certs/mongodb.pem --replSet rs0 + + mongo2: + hostname: mongo2 + container_name: mongo2 + image: mongo:${MONGO_TAG} + environment: + MONGO_INITDB_DATABASE: mqtt + networks: + - emqx_bridge + expose: + - 27017 + ports: + - 27012:27017 + restart: always + volumes: + - ../../apps/emqx/etc/certs/cert.pem:/etc/certs/cert.pem + - ../../apps/emqx/etc/certs/key.pem:/etc/certs/key.pem + command: + - /bin/bash + - -c + - | + cat /etc/certs/key.pem /etc/certs/cert.pem > /etc/certs/mongodb.pem + mongod --ipv6 --bind_ip_all --tlsMode requireTLS --tlsCertificateKeyFile /etc/certs/mongodb.pem --replSet rs0 + + mongo3: + hostname: mongo3 + container_name: mongo3 + image: mongo:${MONGO_TAG} + environment: + MONGO_INITDB_DATABASE: mqtt + networks: + - emqx_bridge + expose: + - 27017 + ports: + - 27013:27017 + restart: always + volumes: + - ../../apps/emqx/etc/certs/cert.pem:/etc/certs/cert.pem + - ../../apps/emqx/etc/certs/key.pem:/etc/certs/key.pem + command: + - /bin/bash + - -c + - | + cat /etc/certs/key.pem /etc/certs/cert.pem > /etc/certs/mongodb.pem + mongod --ipv6 --bind_ip_all --tlsMode requireTLS --tlsCertificateKeyFile /etc/certs/mongodb.pem --replSet rs0 + + mongo_client: + image: mongo:${MONGO_TAG} + container_name: mongo_client + networks: + - emqx_bridge + depends_on: + - mongo1 + - mongo2 + - mongo3 + volumes: + - ../../apps/emqx/etc/certs/cacert.pem:/etc/certs/cacert.pem + command: + - /bin/bash + - -c + - | + while ! mongo --host mongo1 --tls --tlsCAFile /etc/certs/cacert.pem --tlsAllowInvalidHostnames --eval 'db.runCommand("ping").ok' --quiet > /dev/null 2>&1; do + sleep 1 + done + while ! mongo --host mongo2 --tls --tlsCAFile /etc/certs/cacert.pem --tlsAllowInvalidHostnames --eval 'db.runCommand("ping").ok' --quiet > /dev/null 2>&1; do + sleep 1 + done + while ! mongo --host mongo3 --tls --tlsCAFile /etc/certs/cacert.pem --tlsAllowInvalidHostnames --eval 'db.runCommand("ping").ok' --quiet > /dev/null 2>&1; do + sleep 1 + done + mongo --host mongo1 --tls --tlsCAFile /etc/certs/cacert.pem --tlsAllowInvalidHostnames --eval "rs.initiate( { _id : 'rs0', members: [ { _id : 0, host : 'mongo1:27017' }, { _id : 1, host : 'mongo2:27017' }, { _id : 2, host : 'mongo3:27017' } ] })" --quiet + mongo --host mongo1 --tls --tlsCAFile /etc/certs/cacert.pem --tlsAllowInvalidHostnames --eval "rs.status()" --quiet diff --git a/.ci/docker-compose-file/docker-compose-mongo-tcp.yaml b/.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml similarity index 88% rename from .ci/docker-compose-file/docker-compose-mongo-tcp.yaml rename to .ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml index dee2daff6..494b42ce4 100644 --- a/.ci/docker-compose-file/docker-compose-mongo-tcp.yaml +++ b/.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml @@ -9,6 +9,8 @@ services: MONGO_INITDB_DATABASE: mqtt networks: - emqx_bridge + ports: + - "27017:27017" command: --ipv6 --bind_ip_all diff --git a/.ci/docker-compose-file/docker-compose-mongo-single-tls.yaml b/.ci/docker-compose-file/docker-compose-mongo-single-tls.yaml new file mode 100644 index 000000000..c4f162783 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-mongo-single-tls.yaml @@ -0,0 +1,23 @@ +version: '3.9' + +services: + mongo_server: + container_name: mongo + image: mongo:${MONGO_TAG} + restart: always + environment: + MONGO_INITDB_DATABASE: mqtt + volumes: + - ../../apps/emqx/etc/certs/cert.pem:/etc/certs/cert.pem + - ../../apps/emqx/etc/certs/key.pem:/etc/certs/key.pem + networks: + - emqx_bridge + ports: + - "27017:27017" + command: + - /bin/bash + - -c + - | + cat /etc/certs/key.pem /etc/certs/cert.pem > /etc/certs/mongodb.pem + mongod --ipv6 --bind_ip_all --sslMode requireSSL --sslPEMKeyFile /etc/certs/mongodb.pem + diff --git a/.ci/docker-compose-file/docker-compose-mongo-tls.yaml b/.ci/docker-compose-file/docker-compose-mongo-tls.yaml deleted file mode 100644 index a09bc803d..000000000 --- a/.ci/docker-compose-file/docker-compose-mongo-tls.yaml +++ /dev/null @@ -1,18 +0,0 @@ -version: '3.9' - -services: - mongo_server: - container_name: mongo - image: mongo:${MONGO_TAG} - restart: always - environment: - MONGO_INITDB_DATABASE: mqtt - volumes: - - ../../apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/mongodb.pem/:/etc/certs/mongodb.pem - networks: - - emqx_bridge - command: - --ipv6 - --bind_ip_all - --sslMode requireSSL - --sslPEMKeyFile /etc/certs/mongodb.pem diff --git a/.ci/docker-compose-file/docker-compose-mysql-tls.yaml b/.ci/docker-compose-file/docker-compose-mysql-tls.yaml index c4d5bd500..17dfdcc8e 100644 --- a/.ci/docker-compose-file/docker-compose-mysql-tls.yaml +++ b/.ci/docker-compose-file/docker-compose-mysql-tls.yaml @@ -11,9 +11,11 @@ services: MYSQL_USER: ssluser MYSQL_PASSWORD: public volumes: - - ../../apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem:/etc/certs/ca-cert.pem - - ../../apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/server-cert.pem:/etc/certs/server-cert.pem - - ../../apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/server-key.pem:/etc/certs/server-key.pem + - ../../apps/emqx/etc/certs/cacert.pem:/etc/certs/ca-cert.pem + - ../../apps/emqx/etc/certs/cert.pem:/etc/certs/server-cert.pem + - ../../apps/emqx/etc/certs/key.pem:/etc/certs/server-key.pem + ports: + - "3306:3306" networks: - emqx_bridge command: diff --git a/.ci/docker-compose-file/docker-compose-redis-cluster-tls.yaml b/.ci/docker-compose-file/docker-compose-redis-cluster-tls.yaml index 78b655946..c5cefd9e6 100644 --- a/.ci/docker-compose-file/docker-compose-redis-cluster-tls.yaml +++ b/.ci/docker-compose-file/docker-compose-redis-cluster-tls.yaml @@ -5,7 +5,9 @@ services: container_name: redis image: redis:${REDIS_TAG} volumes: - - ../../apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs:/tls + - ../../apps/emqx/etc/certs/cacert.pem:/etc/certs/ca.crt + - ../../apps/emqx/etc/certs/cert.pem:/etc/certs/redis.crt + - ../../apps/emqx/etc/certs/key.pem:/etc/certs/redis.key - ./redis/:/data/conf command: bash -c "/bin/bash /data/conf/redis.sh --node cluster --tls-enabled && tail -f /var/log/redis-server.log" networks: diff --git a/.ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml b/.ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml index 7c7f46ce2..045570d5c 100644 --- a/.ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml +++ b/.ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml @@ -5,7 +5,9 @@ services: container_name: redis image: redis:${REDIS_TAG} volumes: - - ../../apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs:/tls + - ../../apps/emqx/etc/certs/cacert.pem:/etc/certs/ca.crt + - ../../apps/emqx/etc/certs/cert.pem:/etc/certs/redis.crt + - ../../apps/emqx/etc/certs/key.pem:/etc/certs/redis.key - ./redis/:/data/conf command: bash -c "/bin/bash /data/conf/redis.sh --node sentinel --tls-enabled && tail -f /var/log/redis-server.log" networks: diff --git a/.ci/docker-compose-file/docker-compose-redis-single-tls.yaml b/.ci/docker-compose-file/docker-compose-redis-single-tls.yaml index 814a0f1cb..bb6c3ff15 100644 --- a/.ci/docker-compose-file/docker-compose-redis-single-tls.yaml +++ b/.ci/docker-compose-file/docker-compose-redis-single-tls.yaml @@ -5,15 +5,17 @@ services: container_name: redis image: redis:${REDIS_TAG} volumes: - - ../../apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs:/tls + - ../../apps/emqx/etc/certs/cacert.pem:/etc/certs/ca.crt + - ../../apps/emqx/etc/certs/cert.pem:/etc/certs/redis.crt + - ../../apps/emqx/etc/certs/key.pem:/etc/certs/redis.key command: - redis-server - "--bind 0.0.0.0 ::" - --requirepass public - --tls-port 6380 - - --tls-cert-file /tls/redis.crt - - --tls-key-file /tls/redis.key - - --tls-ca-cert-file /tls/ca.crt + - --tls-cert-file /etc/certs/redis.crt + - --tls-key-file /etc/certs/redis.key + - --tls-ca-cert-file /etc/certs/ca.crt restart: always networks: - emqx_bridge diff --git a/.ci/docker-compose-file/pgsql/Dockerfile b/.ci/docker-compose-file/pgsql/Dockerfile index e4c973258..db2cd59fe 100644 --- a/.ci/docker-compose-file/pgsql/Dockerfile +++ b/.ci/docker-compose-file/pgsql/Dockerfile @@ -2,9 +2,9 @@ ARG BUILD_FROM=postgres:11 FROM ${BUILD_FROM} ARG POSTGRES_USER=postgres COPY --chown=$POSTGRES_USER .ci/docker-compose-file/pgsql/pg_hba.conf /var/lib/postgresql/pg_hba.conf -COPY --chown=$POSTGRES_USER apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-key.pem /var/lib/postgresql/server.key -COPY --chown=$POSTGRES_USER apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-cert.pem /var/lib/postgresql/server.crt -COPY --chown=$POSTGRES_USER apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca.pem /var/lib/postgresql/root.crt +COPY --chown=$POSTGRES_USER apps/emqx/etc/certs/key.pem /var/lib/postgresql/server.key +COPY --chown=$POSTGRES_USER apps/emqx/etc/certs/cert.pem /var/lib/postgresql/server.crt +COPY --chown=$POSTGRES_USER apps/emqx/etc/certs/cacert.pem /var/lib/postgresql/root.crt RUN chmod 600 /var/lib/postgresql/pg_hba.conf RUN chmod 600 /var/lib/postgresql/server.key RUN chmod 600 /var/lib/postgresql/server.crt diff --git a/.ci/docker-compose-file/redis/redis-tls.conf b/.ci/docker-compose-file/redis/redis-tls.conf index 325c200c3..e304c814f 100644 --- a/.ci/docker-compose-file/redis/redis-tls.conf +++ b/.ci/docker-compose-file/redis/redis-tls.conf @@ -1,9 +1,9 @@ daemonize yes bind 0.0.0.0 :: logfile /var/log/redis-server.log -tls-cert-file /tls/redis.crt -tls-key-file /tls/redis.key -tls-ca-cert-file /tls/ca.crt +tls-cert-file /etc/certs/redis.crt +tls-key-file /etc/certs/redis.key +tls-ca-cert-file /etc/certs/ca.crt tls-replication yes tls-cluster yes protected-mode no diff --git a/.ci/docker-compose-file/redis/redis.sh b/.ci/docker-compose-file/redis/redis.sh index 272a5b443..6cc7ce98b 100755 --- a/.ci/docker-compose-file/redis/redis.sh +++ b/.ci/docker-compose-file/redis/redis.sh @@ -91,7 +91,7 @@ do fi if [ "${node}" = "cluster" ] ; then if $tls ; then - yes "yes" | redis-cli --cluster create "$LOCAL_IP:8000" "$LOCAL_IP:8001" "$LOCAL_IP:8002" --pass public --no-auth-warning --tls true --cacert /tls/ca.crt --cert /tls/redis.crt --key /tls/redis.key; + yes "yes" | redis-cli --cluster create "$LOCAL_IP:8000" "$LOCAL_IP:8001" "$LOCAL_IP:8002" --pass public --no-auth-warning --tls true --cacert /etc/certs/ca.crt --cert /etc/certs/redis.crt --key /etc/certs/redis.key; else yes "yes" | redis-cli --cluster create "$LOCAL_IP:7000" "$LOCAL_IP:7001" "$LOCAL_IP:7002" --pass public --no-auth-warning; fi @@ -107,9 +107,9 @@ EOF cat >>/_sentinel.conf< -Fixes - -**If your build fails** due to your commit message not passing the build checks, please review the guidelines here: https://github.com/emqx/emqx/blob/master/CONTRIBUTING.md. - -## PR Checklist -Please convert it to a draft if any of the following conditions are not met. Reviewers may skip over until all the items are checked: - -- [ ] Tests for the changes have been added (for bug fixes / features) -- [ ] Docs have been added / updated (for bug fixes / features) -- [ ] In case of non-backward compatible changes, reviewer should check this item as a write-off, and add details in **Backward Compatibility** section - -## Backward Compatibility - -## More information \ No newline at end of file diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index b983eaa67..141645043 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -42,6 +42,7 @@ jobs: if: endsWith(github.repository, 'emqx') run: | make -C source deps-all + rm source/rebar.lock zip -ryq source.zip source/* source/.[^.]* - name: get_all_deps if: endsWith(github.repository, 'enterprise') @@ -63,6 +64,7 @@ jobs: if: endsWith(github.repository, 'emqx') strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} exclude: @@ -131,6 +133,7 @@ jobs: needs: prepare strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} erl_otp: @@ -179,11 +182,11 @@ jobs: cd source pkg_name=$(basename _packages/${{ matrix.profile }}/${{ matrix.profile }}-*.zip) unzip -q _packages/${{ matrix.profile }}/$pkg_name - gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins + # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' for i in {1..10}; do - if curl -fs 127.0.0.1:18083 > /dev/null; then + if curl -fs 127.0.0.1:8081/api/v5/status > /dev/null; then ready='yes' break fi @@ -210,6 +213,7 @@ jobs: needs: prepare strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} arch: @@ -336,19 +340,13 @@ jobs: needs: prepare strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} arch: - [amd64, x86_64] - [arm64v8, aarch64] - [arm32v7, arm] - - [i386, i386] - - [s390x, s390x] - exclude: - - profile: emqx-ee - arch: [i386, i386] - - profile: emqx-ee - arch: [s390x, s390x] steps: - uses: actions/download-artifact@v2 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index bf85578c5..162959040 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -108,11 +108,11 @@ jobs: run: | pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip) unzip -q _packages/${EMQX_NAME}/$pkg_name - gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins + # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' for i in {1..10}; do - if curl -fs 127.0.0.1:18083 > /dev/null; then + if curl -fs 127.0.0.1:8081/api/v5/status > /dev/null; then ready='yes' break fi diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 035c0d0e3..c6b160304 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -36,10 +36,9 @@ jobs: timeout-minutes: 5 run: | set -e -u -x - echo "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" >> .ci/docker-compose-file/conf.cluster.env echo "HOCON_ENV_OVERRIDE_PREFIX=EMQX_" >> .ci/docker-compose-file/conf.cluster.env - echo "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s" >> .ci/docker-compose-file/conf.cluster.env - echo "EMQX_MQTT__MAX_TOPIC_ALIAS=10" >> .ci/docker-compose-file/conf.cluster.env + echo "EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s" >> .ci/docker-compose-file/conf.cluster.env + echo "EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10" >> .ci/docker-compose-file/conf.cluster.env docker-compose \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ -f .ci/docker-compose-file/docker-compose-python.yaml \ @@ -48,13 +47,13 @@ jobs: echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:waiting emqx"; sleep 5; done - - name: verify EMQX_LOADED_PLUGINS override working - run: | - expected="{emqx_sn, true}." - output=$(docker exec -i node1.emqx.io bash -c "cat data/loaded_plugins" | tail -n1) - if [ "$expected" != "$output" ]; then - exit 1 - fi + # - name: verify EMQX_LOADED_PLUGINS override working + # run: | + # expected="{emqx_sn, true}." + # output=$(docker exec -i node1.emqx.io bash -c "cat data/loaded_plugins" | tail -n1) + # if [ "$expected" != "$output" ]; then + # exit 1 + # fi - name: make paho tests run: | if ! docker exec -i python /scripts/pytest.sh; then @@ -118,8 +117,8 @@ jobs: --set image.pullPolicy=Never \ --set emqxAclConfig="" \ --set image.pullPolicy=Never \ - --set emqxConfig.EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s \ - --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ + --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s \ + --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \ deploy/charts/emqx \ --debug @@ -131,11 +130,27 @@ jobs: echo "waiting emqx started"; sleep 10; done - - name: get pods log + - name: get emqx-0 pods log if: failure() env: KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" - run: kubectl describe pods emqx-0 + run: | + kubectl describe pods emqx-0 + kubectl logs emqx-0 + - name: get emqx-1 pods log + if: failure() + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + run: | + kubectl describe pods emqx-1 + kubectl logs emqx-1 + - name: get emqx-2 pods log + if: failure() + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + run: | + kubectl describe pods emqx-2 + kubectl logs emqx-2 - uses: actions/checkout@v2 with: repository: emqx/paho.mqtt.testing diff --git a/.gitignore b/.gitignore index dd8e9b82e..57be83882 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,8 @@ emqx_dialyzer_*_plt */emqx_dashboard/priv/www dist.zip scripts/git-token -etc/*.seg +apps/*/etc/*.all _upgrade_base/ TAGS +erlang_ls.config +.els_cache/ diff --git a/Makefile b/Makefile index cc8cdb0db..c39270367 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ $(shell $(CURDIR)/scripts/git-hooks-init.sh) -REBAR_VERSION = 3.14.3-emqx-8 +REBAR_VERSION = 3.16.1-emqx-1 REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts @@ -73,7 +73,8 @@ coveralls: $(REBAR) @ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send .PHONY: $(REL_PROFILES) -$(REL_PROFILES:%=%): $(REBAR) get-dashboard + +$(REL_PROFILES:%=%): $(REBAR) get-dashboard conf-segs @$(REBAR) as $(@) do compile,release ## Not calling rebar3 clean because @@ -91,6 +92,7 @@ $(PROFILES:%=clean-%): .PHONY: clean-all clean-all: + @rm -f rebar.lock @rm -rf _build .PHONY: deps-all @@ -111,7 +113,7 @@ xref: $(REBAR) dialyzer: $(REBAR) @$(REBAR) as check dialyzer -COMMON_DEPS := $(REBAR) get-dashboard $(CONF_SEGS) +COMMON_DEPS := $(REBAR) get-dashboard conf-segs ## rel target is to create release package without relup .PHONY: $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel) @@ -152,3 +154,6 @@ quickrun: ./_build/$(PROFILE)/rel/emqx/bin/emqx console include docker.mk + +conf-segs: + @scripts/merge-config.escript diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000..c32f1a01d --- /dev/null +++ b/NOTICE @@ -0,0 +1,5 @@ +EMQ X, highly scalable, highly available distributed MQTT messaging platform for IoT. +Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. + +This product contains code developed at EMQ Technologies Co., Ltd. +Visit https://www.emqx.come to learn more. diff --git a/README-CN.md b/README-CN.md index 2a9dcebf6..b430d4b5f 100644 --- a/README-CN.md +++ b/README-CN.md @@ -9,7 +9,7 @@ [![Community](https://img.shields.io/badge/Community-EMQ%20X-yellow)](https://askemq.com) [![YouTube](https://img.shields.io/badge/Subscribe-EMQ%20中文-FF0000?logo=youtube)](https://www.youtube.com/channel/UCir_r04HIsLjf2qqyZ4A8Cg) -[![最棒的物联网 MQTT 开源团队期待您的加入](https://www.emqx.io/static/img/github_readme_cn_bg.png)](https://careers.emqx.cn/) +[![最棒的物联网 MQTT 开源团队期待您的加入](https://static.emqx.net/images/github_readme_cn_bg.png)](https://careers.emqx.cn/) [English](./README.md) | 简体中文 | [日本語](./README-JP.md) | [русский](./README-RU.md) @@ -18,7 +18,7 @@ 从 3.0 版本开始，*EMQ X* 完整支持 MQTT V5.0 协议规范，向下兼容 MQTT V3.1 和 V3.1.1，并支持 MQTT-SN、CoAP、LwM2M、WebSocket 和 STOMP 等通信协议。EMQ X 3.0 单集群可支持千万级别的 MQTT 并发连接。 - 新功能的完整列表，请参阅 [EMQ X Release Notes](https://github.com/emqx/emqx/releases)。 -- 获取更多信息，请访问 [EMQ X 官网](https://www.emqx.cn/)。 +- 获取更多信息，请访问 [EMQ X 官网](https://www.emqx.io/zh)。 ## 安装 @@ -34,7 +34,7 @@ docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p #### 二进制软件包安装 -需从 [EMQ X 下载](https://www.emqx.cn/downloads) 页面获取相应操作系统的二进制软件包。 +需从 [EMQ X 下载](https://www.emqx.com/zh/downloads) 页面获取相应操作系统的二进制软件包。 - [单节点安装文档](https://docs.emqx.cn/broker/latest/getting-started/install.html) - [集群配置文档](https://docs.emqx.cn/broker/latest/advanced/cluster.html) @@ -133,7 +133,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_authz make dialyzer - [Facebook](https://www.facebook.com/emqxmqtt) - [Reddit](https://www.reddit.com/r/emqx/) - [Weibo](https://weibo.com/emqtt) -- [Blog](https://www.emqx.cn/blog) +- [Blog](https://www.emqx.com/zh/blog) 欢迎你将任何 bug、问题和功能请求提交到 [emqx/emqx](https://github.com/emqx/emqx/issues)。 @@ -145,7 +145,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_authz make dialyzer [MQTT Version 5.0](https://docs.oasis-open.org/mqtt/mqtt/v5.0/cs02/mqtt-v5.0-cs02.html) -[MQTT SN](http://mqtt.org/new/wp-content/uploads/2009/06/MQTT-SN_spec_v1.2.pdf) +[MQTT SN](https://www.oasis-open.org/committees/download.php/66091/MQTT-SN_spec_v1.2.pdf) ## 开源许可 diff --git a/README-JP.md b/README-JP.md index 580a76268..6e1c62f2f 100644 --- a/README-JP.md +++ b/README-JP.md @@ -8,7 +8,7 @@ [![Twitter](https://img.shields.io/badge/Twitter-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q) -[![The best IoT MQTT open source team looks forward to your joining](https://www.emqx.io/static/img/github_readme_en_bg.png)](https://www.emqx.io/careers) +[![The best IoT MQTT open source team looks forward to your joining](https://static.emqx.net/images/github_readme_en_bg.png)](https://www.emqx.com/en/careers) [English](./README.md) | [简体中文](./README-CN.md) | 日本語 | [русский](./README-RU.md) @@ -35,7 +35,7 @@ docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p #### バイナリパッケージによるインストール -それぞれのOSに対応したバイナリソフトウェアパッケージは、[EMQ Xのダウンロード](https://www.emqx.io/downloads)ページから取得できます。 +それぞれのOSに対応したバイナリソフトウェアパッケージは、[EMQ Xのダウンロード](https://www.emqx.com/en/downloads)ページから取得できます。 - [シングルノードインストール](https://docs.emqx.io/broker/latest/en/getting-started/installation.html) - [マルチノードインストール](https://docs.emqx.io/broker/latest/en/advanced/cluster.html) @@ -125,7 +125,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_authz make dialyzer [MQTT Version 5.0](https://docs.oasis-open.org/mqtt/mqtt/v5.0/cs02/mqtt-v5.0-cs02.html) -[MQTT SN](http://mqtt.org/new/wp-content/uploads/2009/06/MQTT-SN_spec_v1.2.pdf) +[MQTT SN](https://www.oasis-open.org/committees/download.php/66091/MQTT-SN_spec_v1.2.pdf) ## License diff --git a/README-RU.md b/README-RU.md index 5001f3fd3..2a06dac71 100644 --- a/README-RU.md +++ b/README-RU.md @@ -9,7 +9,7 @@ [![Community](https://img.shields.io/badge/Community-EMQ%20X-yellow?logo=github)](https://github.com/emqx/emqx/discussions) [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q) -[![The best IoT MQTT open source team looks forward to your joining](https://www.emqx.io/static/img/github_readme_en_bg.png)](https://www.emqx.io/careers) +[![The best IoT MQTT open source team looks forward to your joining](https://static.emqx.net/images/github_readme_en_bg.png)](https://www.emqx.com/en/careers) [English](./README.md) | [简体中文](./README-CN.md) | [日本語](./README-JP.md) | русский @@ -18,7 +18,7 @@ Начиная с релиза 3.0, брокер *EMQ X* полностью поддерживает протокол MQTT версии 5.0, и обратно совместим с версиями 3.1 и 3.1.1, а также протоколами MQTT-SN, CoAP, LwM2M, WebSocket и STOMP. Начиная с релиза 3.0, брокер *EMQ X* может масштабироваться до более чем 10 миллионов одновременных MQTT соединений на один кластер. - Полный список возможностей доступен по ссылке: [EMQ X Release Notes](https://github.com/emqx/emqx/releases). -- Более подробная информация доступна на нашем сайте: [EMQ X homepage](https://www.emqx.io). +- Более подробная информация доступна на нашем сайте: [EMQ X homepage](https://www.emqx.io/). ## Установка @@ -34,7 +34,7 @@ docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p #### Установка бинарного пакета -Сборки для различных операционных систем: [Загрузить EMQ X](https://www.emqx.io/downloads). +Сборки для различных операционных систем: [Загрузить EMQ X](https://www.emqx.com/en/downloads). - [Установка на одном сервере](https://docs.emqx.io/en/broker/latest/getting-started/install.html) - [Установка на кластере](https://docs.emqx.io/en/broker/latest/advanced/cluster.html) @@ -135,7 +135,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_authz make dialyzer [MQTT Version 5.0](https://docs.oasis-open.org/mqtt/mqtt/v5.0/cs02/mqtt-v5.0-cs02.html) -[MQTT SN](http://mqtt.org/new/wp-content/uploads/2009/06/MQTT-SN_spec_v1.2.pdf) +[MQTT SN](https://www.oasis-open.org/committees/download.php/66091/MQTT-SN_spec_v1.2.pdf) ## Лицензия diff --git a/README.md b/README.md index f76d7ae0a..1726d426b 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q) -[![The best IoT MQTT open source team looks forward to your joining](https://www.emqx.io/static/img/github_readme_en_bg.png)](https://www.emqx.io/careers) +[![The best IoT MQTT open source team looks forward to your joining](https://static.emqx.net/images/github_readme_en_bg.png)](https://www.emqx.com/en/careers) English | [简体中文](./README-CN.md) | [日本語](./README-JP.md) | [русский](./README-RU.md) @@ -17,7 +17,7 @@ English | [简体中文](./README-CN.md) | [日本語](./README-JP.md) | [рус Starting from 3.0 release, *EMQ X* broker fully supports MQTT V5.0 protocol specifications and backward compatible with MQTT V3.1 and V3.1.1, as well as other communication protocols such as MQTT-SN, CoAP, LwM2M, WebSocket and STOMP. The 3.0 release of the *EMQ X* broker can scaled to 10+ million concurrent MQTT connections on one cluster. - For full list of new features, please read [EMQ X Release Notes](https://github.com/emqx/emqx/releases). -- For more information, please visit [EMQ X homepage](https://www.emqx.io). +- For more information, please visit [EMQ X homepage](https://www.emqx.io/). ## Installation @@ -33,7 +33,7 @@ docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p #### Installing via Binary Package -Get the binary package of the corresponding OS from [EMQ X Download](https://www.emqx.io/downloads) page. +Get the binary package of the corresponding OS from [EMQ X Download](https://www.emqx.com/en/downloads) page. - [Single Node Install](https://docs.emqx.io/en/broker/latest/getting-started/install.html) - [Multi Node Install](https://docs.emqx.io/en/broker/latest/advanced/cluster.html) @@ -134,7 +134,7 @@ You can read the mqtt protocol via the following links: [MQTT Version 5.0](https://docs.oasis-open.org/mqtt/mqtt/v5.0/cs02/mqtt-v5.0-cs02.html) -[MQTT SN](http://mqtt.org/new/wp-content/uploads/2009/06/MQTT-SN_spec_v1.2.pdf) +[MQTT SN](https://www.oasis-open.org/committees/download.php/66091/MQTT-SN_spec_v1.2.pdf) ## License diff --git a/apps/emqx/NOTICE b/apps/emqx/NOTICE new file mode 100644 index 000000000..c32f1a01d --- /dev/null +++ b/apps/emqx/NOTICE @@ -0,0 +1,5 @@ +EMQ X, highly scalable, highly available distributed MQTT messaging platform for IoT. +Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. + +This product contains code developed at EMQ Technologies Co., Ltd. +Visit https://www.emqx.come to learn more. diff --git a/apps/emqx/etc/acl.conf b/apps/emqx/etc/acl.conf deleted file mode 100644 index af2fb0dd1..000000000 --- a/apps/emqx/etc/acl.conf +++ /dev/null @@ -1,26 +0,0 @@ -%%-------------------------------------------------------------------- -%% [ACL](https://docs.emqx.io/broker/v3/en/config.html) -%% -%% -type(who() :: all | binary() | -%% {ipaddr, esockd_access:cidr()} | -%% {client, binary()} | -%% {user, binary()}). -%% -%% -type(access() :: subscribe | publish | pubsub). -%% -%% -type(topic() :: binary()). -%% -%% -type(rule() :: {allow, all} | -%% {allow, who(), access(), list(topic())} | -%% {deny, all} | -%% {deny, who(), access(), list(topic())}). -%%-------------------------------------------------------------------- - -{allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}. - -{allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}. - -{deny, all, subscribe, ["$SYS/#", {eq, "#"}]}. - -{allow, all}. - diff --git a/apps/emqx/etc/acl.conf.paho b/apps/emqx/etc/acl.conf.paho deleted file mode 100644 index 5beec4347..000000000 --- a/apps/emqx/etc/acl.conf.paho +++ /dev/null @@ -1,14 +0,0 @@ -%%-------------------------------------------------------------------- -%% For paho interoperability test cases -%%-------------------------------------------------------------------- - -{deny, {client, "myclientid"}, subscribe, ["test/nosubscribe"]}. - -{allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}. - -{allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}. - -{deny, all, subscribe, ["$SYS/#", {eq, "#"}]}. - -{allow, all}. - diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index 3387a6439..46f81a87d 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -1,2467 +1,2399 @@ -## EMQ X Configuration 4.3 +## master-88df1713 -## NOTE: Do not change format of CONFIG_SECTION_{BGN,END} comments! +## NOTE: The configurations in this file will be overridden by +## `/data/emqx_overrides.conf` -## CONFIG_SECTION_BGN=cluster ================================================== - -## Cluster name. -## -## Value: String -cluster.name = emqxcl - -## Specify the erlang distributed protocol. -## -## Value: Enum -## - inet_tcp: the default; handles TCP streams with IPv4 addressing. -## - inet6_tcp: handles TCP with IPv6 addressing. -## - inet_tls: using TLS for Erlang Distribution. -## -## vm.args: -proto_dist inet_tcp -cluster.proto_dist = inet_tcp - -## Cluster auto-discovery strategy. -## -## Value: Enum -## - manual: Manual join command -## - static: Static node list -## - mcast: IP Multicast -## - dns: DNS A Record -## - etcd: etcd -## - k8s: Kubernetes -## -## Default: manual -cluster.discovery = manual - -## Enable cluster autoheal from network partition. -## -## Value: on | off -## -## Default: on -cluster.autoheal = on - -## Autoclean down node. A down node will be removed from the cluster -## if this value > 0. -## -## Value: Duration -## -h: hour, e.g. '2h' for 2 hours -## -m: minute, e.g. '5m' for 5 minutes -## -s: second, e.g. '30s' for 30 seconds -## -## Default: 5m -cluster.autoclean = 5m - -##-------------------------------------------------------------------- -## Cluster using static node list - -## Node list of the cluster. -## -## Value: String -## cluster.static.seeds = "emqx1@127.0.0.1,emqx2@127.0.0.1" - -##-------------------------------------------------------------------- -## Cluster using IP Multicast. - -## IP Multicast Address. -## -## Value: IP Address -## cluster.mcast.addr = "239.192.0.1" - -## Multicast Ports. -## -## Value: Port List -## cluster.mcast.ports = "4369,4370" - -## Multicast Iface. -## -## Value: Iface Address -## -## Default: "0.0.0.0" -## cluster.mcast.iface = "0.0.0.0" - -## Multicast Ttl. -## -## Value: 0-255 -## cluster.mcast.ttl = 255 - -## Multicast loop. -## -## Value: on | off -## cluster.mcast.loop = on - -##-------------------------------------------------------------------- -## Cluster using DNS A records. - -## DNS name. -## -## Value: String -## cluster.dns.name = localhost - -## The App name is used to build 'node.name' with IP address. -## -## Value: String -## cluster.dns.app = emqx - -##-------------------------------------------------------------------- -## Cluster using etcd - -## Etcd server list, seperated by ','. -## -## Value: String -## cluster.etcd.server = "http://127.0.0.1:2379" - -## Etcd api version -## -## Value: Enum -## - v2 -## - v3 -## cluster.etcd.version = v3 - -## The prefix helps build nodes path in etcd. Each node in the cluster -## will create a path in etcd: v2/keys/// -## -## Value: String -## cluster.etcd.prefix = emqxcl - -## The TTL for node's path in etcd. -## -## Value: Duration -## -## Default: 1m, 1 minute -## cluster.etcd.node_ttl = 1m - -## Path to a file containing the client's private PEM-encoded key. -## -## Value: File -## cluster.etcd.ssl.keyfile = "{{ platform_etc_dir }}/certs/client-key.pem" - -## The path to a file containing the client's certificate. -## -## Value: File -## cluster.etcd.ssl.certfile = "{{ platform_etc_dir }}/certs/client.pem" - -## Path to the file containing PEM-encoded CA certificates. The CA certificates -## are used during server authentication and when building the client certificate chain. -## -## Value: File -## cluster.etcd.ssl.cacertfile = "{{ platform_etc_dir }}/certs/ca.pem" - -##-------------------------------------------------------------------- -## Cluster using Kubernetes - -## Kubernetes API server list, seperated by ','. -## -## Value: String -## cluster.k8s.apiserver = "http://10.110.111.204:8080" - -## The service name helps lookup EMQ nodes in the cluster. -## -## Value: String -## cluster.k8s.service_name = emqx - -## The address type is used to extract host from k8s service. -## -## Value: ip | dns | hostname -## cluster.k8s.address_type = ip - -## The app name helps build 'node.name'. -## -## Value: String -## cluster.k8s.app_name = emqx - -## The suffix added to dns and hostname get from k8s service -## -## Value: String -## cluster.k8s.suffix = pod.cluster.local - -## Kubernetes Namespace -## -## Value: String -## cluster.k8s.namespace = default - -## CONFIG_SECTION_END=cluster ================================================== - -##-------------------------------------------------------------------- +##================================================================== ## Node -##-------------------------------------------------------------------- - -## Node name. -## -## See: http://erlang.org/doc/reference_manual/distributed.html -## -## Value: @ -## -## Default: emqx@127.0.0.1 -node.name = "emqx@127.0.0.1" - -## Cookie for distributed node communication. -## -## Value: String -node.cookie = "emqxsecretcookie" - -## Data dir for the node -## -## Value: Folder -node.data_dir = "{{ platform_data_dir }}" - -## The config file dir for the node -## -## Value: Folder -node.etc_dir = "{{ platform_etc_dir }}" - -## Heartbeat monitoring of an Erlang runtime system. Comment the line to disable -## heartbeat, or set the value as 'on' -## -## Value: on -## -## vm.args: -heart -## node.heartbeat = on - -## Sets the number of threads in async thread pool. Valid range is 0-1024. -## -## See: http://erlang.org/doc/man/erl.html -## -## Value: 0-1024 -## -## vm.args: +A Number -## node.async_threads = 4 - -## Sets the maximum number of simultaneously existing processes for this -## system if a Number is passed as value. -## -## See: http://erlang.org/doc/man/erl.html -## -## Value: Number [1024-134217727] -## -## vm.args: +P Number -## node.process_limit = 2097152 - -## Sets the maximum number of simultaneously existing ports for this system. -## -## See: http://erlang.org/doc/man/erl.html -## -## Value: Number [1024-134217727] -## -## vm.args: +Q Number -## node.max_ports = 1048576 - -## Sets the distribution buffer busy limit (dist_buf_busy_limit). -## -## See: http://erlang.org/doc/man/erl.html -## -## Value: Number [1KB-2GB] -## -## vm.args: +zdbbl size -## node.dist_buffer_size = 8MB - -## Sets the maximum number of ETS tables. Note that mnesia and SSL will -## create temporary ETS tables. -## -## Value: Number -## -## vm.args: +e Number -## node.max_ets_tables = 262144 - -## Global GC Interval. -## -## Value: Duration -## -## Examples: -## - 2h: 2 hours -## - 30m: 30 minutes -## - 20s: 20 seconds -## -## Defaut: 15 minutes -node.global_gc_interval = 15m - -## Tweak GC to run more often. -## -## Value: Number [0-65535] -## -## vm.args: -env ERL_FULLSWEEP_AFTER Number -## node.fullsweep_after = 1000 - -## Crash dump log file. -## -## Value: Log file -node.crash_dump = "{{ platform_log_dir }}/crash.dump" - -## Specify SSL Options in the file if using SSL for Erlang Distribution. -## -## Value: File -## -## vm.args: -ssl_dist_optfile -## node.ssl_dist_optfile = "{{ platform_etc_dir }}/ssl_dist.conf" - -## Sets the net_kernel tick time. TickTime is specified in seconds. -## Notice that all communicating nodes are to have the same TickTime -## value specified. -## -## See: http://www.erlang.org/doc/man/kernel_app.html#net_ticktime -## -## Value: Number -## -## vm.args: -kernel net_ticktime Number -## node.dist_net_ticktime = 120 - -## Sets the port range for the listener socket of a distributed Erlang node. -## Note that if there are firewalls between clustered nodes, this port segment -## for nodes’ communication should be allowed. -## -## See: http://www.erlang.org/doc/man/kernel_app.html -## -## Value: Port [1024-65535] -node.dist_listen_min = 6369 -node.dist_listen_max = 6369 - -node.backtrace_depth = 16 - -## CONFIG_SECTION_BGN=rpc ====================================================== - -## RPC Mode. -## -## Value: sync | async -rpc.mode = async - -## Max batch size of async RPC requests. -## -## Value: Integer -## Zero or negative value disables rpc batching. -## -## NOTE: RPC batch won't work when rpc.mode = sync -rpc.async_batch_size = 256 - -## RPC port discovery -## -## The strategy for discovering the RPC listening port of other nodes. -## -## Value: Enum -## - manual: discover ports by `tcp_server_port` and `tcp_client_port`. -## - stateless: discover ports in a stateless manner. -## If node name is `emqx@127.0.0.1`, where the `` is an integer, -## then the listening port will be `5370 + ` -## -## Defaults to `stateless`. -rpc.port_discovery = stateless - -## TCP port number for RPC server to listen on. -## -## Only takes effect when `rpc.port_discovery` = `manual`. -## -## NOTE: All nodes in the cluster should agree to this same config. -## -## Value: Port [1024-65535] -#rpc.tcp_server_port = 5369 - -## Number of outgoing RPC connections. -## -## Value: Interger [0-256] -## Default = 1 -#rpc.tcp_client_num = 1 - -## RCP Client connect timeout. -## -## Value: Seconds -rpc.connect_timeout = 5s - -## TCP send timeout of RPC client and server. -## -## Value: Seconds -rpc.send_timeout = 5s - -## Authentication timeout -## -## Value: Seconds -rpc.authentication_timeout = 5s - -## Default receive timeout for call() functions -## -## Value: Seconds -rpc.call_receive_timeout = 15s - -## Socket idle keepalive. -## -## Value: Seconds -rpc.socket_keepalive_idle = 900s - -## TCP Keepalive probes interval. -## -## Value: Seconds -rpc.socket_keepalive_interval = 75s - -## Probes lost to close the connection -## -## Value: Integer -rpc.socket_keepalive_count = 9 - -## Size of TCP send buffer. -## -## Value: Bytes -rpc.socket_sndbuf = 1MB - -## Size of TCP receive buffer. -## -## Value: Seconds -rpc.socket_recbuf = 1MB - -## Size of user-level software socket buffer. -## -## Value: Seconds -rpc.socket_buffer = 1MB - -## CONFIG_SECTION_END=rpc ====================================================== - -## CONFIG_SECTION_BGN=logger =================================================== - -## Where to emit the logs. -## Enable the console (standard output) logs. -## -## Value: file | console | both -## - file: write logs only to file -## - console: write logs only to standard I/O -## - both: write logs both to file and standard I/O -log.to = file - -## The log severity level. -## -## Value: debug | info | notice | warning | error | critical | alert | emergency -## -## Note: Only the messages with severity level higher than or equal to -## this level will be logged. -## -## Default: warning -log.level = warning - -## Timezone offset to display in logs -## Value: -## - "system" use system zone -## - "utc" for Universal Coordinated Time (UTC) -## - "+hh:mm" or "-hh:mm" for a specified offset -log.time_offset = system - -## The dir for log files. -## -## Value: Folder -log.dir = "{{ platform_log_dir }}" - -## The log filename for logs of level specified in "log.level". -## -## If `log.rotation` is enabled, this is the base name of the -## files. Each file in a rotated log is named .N, where N is an integer. -## -## Value: String -## Default: emqx.log -log.file = emqx.log - -## Limits the total number of characters printed for each log event. -## -## Value: Integer -## Default: No Limit -#log.chars_limit = 8192 - -## Maximum depth for Erlang term log formatting -## and Erlang process message queue inspection. -## -## Value: Integer or 'unlimited' (without quotes) -## Default: 80 -#log.max_depth = 80 - -## Log formatter -## Value: text | json -#log.formatter = text - -## Log to single line -## Value: Boolean -#log.single_line = true - -## Enables the log rotation. -## With this enabled, new log files will be created when the current -## log file is full, max to `log.rotation.size` files will be created. -## -## Value: on | off -## Default: on -log.rotation.enable = on - -## Maximum size of each log file. -## -## Value: Number -## Default: 10M -## Supported Unit: KB | MB | GB -log.rotation.size = 10MB - -## Maximum rotation count of log files. -## -## Value: Number -## Default: 5 -log.rotation.count = 5 - -## To create additional log files for specific log levels. -## -## Value: File Name -## Format: log.$level.file = $filename, -## where "$level" can be one of: debug, info, notice, warning, -## error, critical, alert, emergency -## Note: Log files for a specific log level will only contain all the logs -## that higher than or equal to that level -## -#log.info.file = info.log -#log.error.file = error.log - -## The max allowed queue length before switching to sync mode. -## -## Log overload protection parameter. If the message queue grows -## larger than this value the handler switches from anync to sync mode. -## -## Default: 100 -## -#log.sync_mode_qlen = 100 - -## The max allowed queue length before switching to drop mode. -## -## Log overload protection parameter. When the message queue grows -## larger than this threshold, the handler switches to a mode in which -## it drops all new events that senders want to log. -## -## Default: 3000 -## -#log.drop_mode_qlen = 3000 - -## The max allowed queue length before switching to flush mode. -## -## Log overload protection parameter. If the length of the message queue -## grows larger than this threshold, a flush (delete) operation takes place. -## To flush events, the handler discards the messages in the message queue -## by receiving them in a loop without logging. -## -## Default: 8000 -## -#log.flush_qlen = 8000 - -## Kill the log handler when it gets overloaded. -## -## Log overload protection parameter. It is possible that a handler, -## even if it can successfully manage peaks of high load without crashing, -## can build up a large message queue, or use a large amount of memory. -## We could kill the log handler in these cases and restart it after a -## few seconds. -## -## Default: on -## -#log.overload_kill = on - -## The max allowed queue length before killing the log hanlder. -## -## Log overload protection parameter. This is the maximum allowed queue -## length. If the message queue grows larger than this, the handler -## process is terminated. -## -## Default: 20000 -## -#log.overload_kill_qlen = 20000 - -## The max allowed memory size before killing the log hanlder. -## -## Log overload protection parameter. This is the maximum memory size -## that the handler process is allowed to use. If the handler grows -## larger than this, the process is terminated. -## -## Default: 30MB -## -#log.overload_kill_mem_size = 30MB - -## Restart the log hanlder after some seconds. -## -## Log overload protection parameter. If the handler is terminated, -## it restarts automatically after a delay specified in seconds. -## The value "infinity" prevents restarts. -## -## Default: 5s -## -#log.overload_kill_restart_after = 5s - -## Max burst count and time window for burst control. -## -## Log overload protection parameter. Large bursts of log events - many -## events received by the handler under a short period of time - can -## potentially cause problems. By specifying the maximum number of events -## to be handled within a certain time frame, the handler can avoid -## choking the log with massive amounts of printouts. -## -## This config controls the maximum number of events to handle within -## a time frame. After the limit is reached, successive events are -## dropped until the end of the time frame. -## -## Note that there would be no warning if any messages were -## dropped because of burst control. -## -## Comment this config out to disable the burst control feature. -## -## Value: MaxBurstCount,TimeWindow -## Default: disabled -## -#log.burst_limit = "20000, 1s" - -## CONFIG_SECTION_END=logger =================================================== - -##-------------------------------------------------------------------- -## Authentication/Access Control -##-------------------------------------------------------------------- - -## Allow anonymous authentication by default if no auth plugins loaded. -## Notice: Disable the option in production deployment! -## -## Value: true | false -acl.allow_anonymous = true - -## Allow or deny if no ACL rules matched. -## -## Value: allow | deny -acl.acl_nomatch = allow - -## Default ACL File. -## -## Value: File Name -acl.acl_file = "{{ platform_etc_dir }}/acl.conf" - -## Whether to enable ACL cache. -## -## If enabled, ACLs roles for each client will be cached in the memory -## -## Value: on | off -acl.enable_acl_cache = on - -## The maximum count of ACL entries can be cached for a client. -## -## Value: Integer greater than 0 -## Default: 32 -acl.acl_cache_max_size = 32 - -## The time after which an ACL cache entry will be deleted -## -## Value: Duration -## Default: 1 minute -acl.acl_cache_ttl = 1m - -## The action when acl check reject current operation -## -## Value: ignore | disconnect -## Default: ignore -acl.acl_deny_action = ignore - -## Specify the global flapping detect policy. -## The value is a string composed of flapping threshold, duration and banned interval. -## 1. threshold: an integer to specfify the disconnected times of a MQTT Client; -## 2. duration: the time window for flapping detect; -## 3. banned interval: the banned interval if a flapping is detected. -## -## Value: Integer,Duration,Duration -acl.flapping_detect_policy = "30, 1m, 5m" - -##-------------------------------------------------------------------- -## MQTT Protocol -##-------------------------------------------------------------------- - -## Maximum MQTT packet size allowed. -## -## Value: Bytes -## Default: 1MB -mqtt.max_packet_size = 1MB - -## Maximum length of MQTT clientId allowed. -## -## Value: Number [23-65535] -mqtt.max_clientid_len = 65535 - -## Maximum topic levels allowed. 0 means no limit. -## -## Value: Number -mqtt.max_topic_levels = 0 - -## Maximum QoS allowed. -## -## Value: 0 | 1 | 2 -mqtt.max_qos_allowed = 2 - -## Maximum Topic Alias, 0 means no topic alias supported. -## -## Value: 0-65535 -mqtt.max_topic_alias = 65535 - -## Whether the Server supports MQTT retained messages. -## -## Value: boolean -mqtt.retain_available = true - -## Whether the Server supports MQTT Wildcard Subscriptions -## -## Value: boolean -mqtt.wildcard_subscription = true - -## Whether the Server supports MQTT Shared Subscriptions. -## -## Value: boolean -mqtt.shared_subscription = true - -## Whether to ignore loop delivery of messages.(for mqtt v3.1.1) -## -## Value: true | false -mqtt.ignore_loop_deliver = false - -## Whether to parse the MQTT frame in strict mode -## -## Value: true | false -mqtt.strict_mode = false - -## Specify the response information returned to the client -## -## Value: String -## mqtt.response_information = example - -## CONFIG_SECTION_BGN=zones =================================================== - -##-------------------------------------------------------------------- -## External Zone - -## Idle timeout of the external MQTT connections. -## -## Value: duration -zone.external.idle_timeout = 15s - -## Enable ACL check. -## -## Value: Flag -zone.external.enable_acl = on - -## Enable ban check. -## -## Value: Flag -zone.external.enable_ban = on - -## Enable per connection statistics. -## -## Value: on | off -zone.external.enable_stats = on - -## The action when acl check reject current operation -## -## Value: ignore | disconnect -## Default: ignore -zone.external.acl_deny_action = ignore - -## Force the MQTT connection process GC after this number of -## messages | bytes passed through. -## -## Numbers delimited by `|'. Zero or negative is to disable. -zone.external.force_gc_policy = "16000|16MB" - -## Max message queue length and total heap size to force shutdown -## connection/session process. -## Message queue here is the Erlang process mailbox, but not the number -## of queued MQTT messages of QoS 1 and 2. -## -## Numbers delimited by `|'. Zero or negative is to disable. -## -## Default: -## - "10000|64MB" on ARCH_64 system -## - "1000|32MB" on ARCH_32 sytem -#zone.external.force_shutdown_policy = "10000|64MB" - -## Maximum MQTT packet size allowed. -## -## Value: Bytes -## Default: 1MB -## zone.external.max_packet_size = 64KB - -## Maximum length of MQTT clientId allowed. -## -## Value: Number [23-65535] -## zone.external.max_clientid_len = 1024 - -## Maximum topic levels allowed. 0 means no limit. -## -## Value: Number -## zone.external.max_topic_levels = 7 - -## Maximum QoS allowed. -## -## Value: 0 | 1 | 2 -## zone.external.max_qos_allowed = 2 - -## Maximum Topic Alias, 0 means no limit. -## -## Value: 0-65535 -## zone.external.max_topic_alias = 65535 - -## Whether the Server supports retained messages. -## -## Value: boolean -## zone.external.retain_available = true - -## Whether the Server supports Wildcard Subscriptions -## -## Value: boolean -## zone.external.wildcard_subscription = false - -## Whether the Server supports Shared Subscriptions -## -## Value: boolean -## zone.external.shared_subscription = false - -## Server Keep Alive -## -## Value: Number -## zone.external.server_keepalive = 0 - -## The backoff for MQTT keepalive timeout. The broker will kick a connection out -## until 'Keepalive * backoff * 2' timeout. -## -## Value: Float > 0.5 -zone.external.keepalive_backoff = 0.75 - -## Maximum number of subscriptions allowed, 0 means no limit. -## -## Value: Number -zone.external.max_subscriptions = 0 - -## Force to upgrade QoS according to subscription. -## -## Value: on | off -zone.external.upgrade_qos = off - -## Maximum size of the Inflight Window storing QoS1/2 messages delivered but unacked. -## -## Value: Number -zone.external.max_inflight = 32 - -## Retry interval for QoS1/2 message delivering. -## -## Value: Duration -zone.external.retry_interval = 30s - -## Maximum QoS2 packets (Client -> Broker) awaiting PUBREL, 0 means no limit. -## -## Value: Number -zone.external.max_awaiting_rel = 100 - -## The QoS2 messages (Client -> Broker) will be dropped if awaiting PUBREL timeout. -## -## Value: Duration -zone.external.await_rel_timeout = 300s - -## Default session expiry interval for MQTT V3.1.1 connections. -## -## Value: Duration -## -d: day -## -h: hour -## -m: minute -## -s: second -## -## Default: 2h, 2 hours -zone.external.session_expiry_interval = 2h - -## Maximum queue length. Enqueued messages when persistent client disconnected, -## or inflight window is full. 0 means no limit. -## -## Value: Number >= 0 -zone.external.max_mqueue_len = 1000 - -## Topic priorities. -## 'none' to indicate no priority table (by default), hence all messages -## are treated equal -## -## Priority number [1-255] -## Example: "topic/1=10,topic/2=8" -## NOTE: comma and equal signs are not allowed for priority topic names -## NOTE: messages for topics not in the priority table are treated as -## either highest or lowest priority depending on the configured -## value for mqueue_default_priority -## -zone.external.mqueue_priorities = none - -## Default to highest priority for topics not matching priority table -## -## Value: highest | lowest -zone.external.mqueue_default_priority = highest - -## Whether to enqueue QoS0 messages. -## -## Value: false | true -zone.external.mqueue_store_qos0 = true - -## Whether to turn on flapping detect -## -## Value: on | off -zone.external.enable_flapping_detect = off - -## Message limit for the a external MQTT connection. -## -## Value: Number,Duration -## Example: 100 messages per 10 seconds. -#zone.external.rate_limit.conn_messages_in = "100,10s" - -## Bytes limit for a external MQTT connections. -## -## Value: Number,Duration -## Example: 100KB incoming per 10 seconds. -#zone.external.rate_limit.conn_bytes_in = "100KB,10s" - -## Whether to alarm the congested connections. -## -## Sometimes the mqtt connection (usually an MQTT subscriber) may get "congested" because -## there're too many packets to sent. The socket trys to buffer the packets until the buffer is -## full. If more packets comes after that, the packets will be "pending" in a queue -## and we consider the connection is "congested". -## -## Enable this to send an alarm when there's any bytes pending in the queue. You could set -## the `listener.tcp..sndbuf` to a larger value if the alarm is triggered too often. -## -## The name of the alarm is of format "conn_congestion//". -## Where the is the client-id of the congested MQTT connection. -## And the is the username or "unknown_user" of not provided by the client. -## Default: off -#zone.external.conn_congestion.alarm = off - -## Won't clear the congested alarm in how long time. -## The alarm is cleared only when there're no pending bytes in the queue, and also it has been -## `min_alarm_sustain_duration` time since the last time we considered the connection is "congested". -## -## This is to avoid clearing and sending the alarm again too often. -## Default: 1m -#zone.external.conn_congestion.min_alarm_sustain_duration = 1m - -## Messages quota for the each of external MQTT connection. -## This value consumed by the number of recipient on a message. -## -## Value: Number, Duration -## -## Example: 100 messages per 1s -#zone.external.quota.conn_messages_routing = "100,1s" - -## Messages quota for the all of external MQTT connections. -## This value consumed by the number of recipient on a message. -## -## Value: Number, Duration -## -## Example: 200000 messages per 1s -#zone.external.quota.overall_messages_routing = "200000,1s" - -## All the topics will be prefixed with the mountpoint path if this option is enabled. -## -## Variables in mountpoint path: -## - %c: clientid -## - %u: username -## -## Value: String -## zone.external.mountpoint = "devicebound/" - -## Whether use username replace client id -## -## Value: boolean -## Default: false -zone.external.use_username_as_clientid = false - -## Whether to ignore loop delivery of messages.(for mqtt v3.1.1) -## -## Value: true | false -zone.external.ignore_loop_deliver = false - -## Whether to parse the MQTT frame in strict mode -## -## Value: true | false -zone.external.strict_mode = false - -## Specify the response information returned to the client -## -## Value: String -## zone.external.response_information = example - -##-------------------------------------------------------------------- -## Internal Zone - -zone.internal.allow_anonymous = true - -## Enable per connection stats. -## -## Value: Flag -zone.internal.enable_stats = on - -## Enable ACL check. -## -## Value: Flag -zone.internal.enable_acl = off - -## The action when acl check reject current operation -## -## Value: ignore | disconnect -## Default: ignore -zone.internal.acl_deny_action = ignore - -## See zone.$name.force_gc_policy -## zone.internal.force_gc_policy = "128000|128MB" - -## See zone.$name.wildcard_subscription. -## -## Value: boolean -## zone.internal.wildcard_subscription = true - -## See zone.$name.shared_subscription. -## -## Value: boolean -## zone.internal.shared_subscription = true - -## See zone.$name.max_subscriptions. -## -## Value: Integer -zone.internal.max_subscriptions = 0 - -## See zone.$name.max_inflight -## -## Value: Number -zone.internal.max_inflight = 128 - -## See zone.$name.max_awaiting_rel -## -## Value: Number -zone.internal.max_awaiting_rel = 1000 - -## See zone.$name.max_mqueue_len -## -## Value: Number >= 0 -zone.internal.max_mqueue_len = 10000 - -## Whether to enqueue Qos0 messages. -## -## Value: false | true -zone.internal.mqueue_store_qos0 = true - -## Whether to turn on flapping detect -## -## Value: on | off -zone.internal.enable_flapping_detect = off - -## See zone.$name.force_shutdown_policy -## -## Default: -## - "10000|64MB" on ARCH_64 system -## - "1000|32MB" on ARCH_32 sytem -#zone.internal.force_shutdown_policy = 10000|64MB - -## All the topics will be prefixed with the mountpoint path if this option is enabled. -## -## Variables in mountpoint path: -## - %c: clientid -## - %u: username -## -## Value: String -## zone.internal.mountpoint = "cloudbound/" - -## Whether to ignore loop delivery of messages.(for mqtt v3.1.1) -## -## Value: true | false -zone.internal.ignore_loop_deliver = false - -## Whether to parse the MQTT frame in strict mode -## -## Value: true | false -zone.internal.strict_mode = false - -## Specify the response information returned to the client -## -## Value: String -## zone.internal.response_information = example - -## Allow the zone's clients to bypass authentication step -## -## Value: true | false -zone.internal.bypass_auth_plugins = true - -## CONFIG_SECTION_END=zones ==================================================== - -## CONFIG_SECTION_BGN=listeners ================================================ - -##-------------------------------------------------------------------- -## MQTT/TCP - External TCP Listener for MQTT Protocol - -## listener.tcp.$name is the IP address and port that the MQTT/TCP -## listener will bind. -## -## Value: IP:Port | Port -## -## Examples: 1883, "127.0.0.1:1883", "::1:1883" -listener.tcp.external.endpoint = "0.0.0.0:1883" - -## The acceptor pool for external MQTT/TCP listener. -## -## Value: Number -listener.tcp.external.acceptors = 8 - -## Maximum number of concurrent MQTT/TCP connections. -## -## Value: Number -listener.tcp.external.max_connections = 1024000 - -## Maximum external connections per second. -## -## Value: Number -listener.tcp.external.max_conn_rate = 1000 - -## Specify the {active, N} option for the external MQTT/TCP Socket. -## -## Value: Number -listener.tcp.external.active_n = 100 - -## Zone of the external MQTT/TCP listener belonged to. -## -## See: zone.$name.* -## -## Value: String -listener.tcp.external.zone = external - -## The access control rules for the MQTT/TCP listener. -## -## See: https://github.com/emqtt/esockd#allowdeny -## -## Value: ACL Rule -## -## Example: "allow 192.168.0.0/24" -listener.tcp.external.access.1 = "allow all" - -## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed -## behind HAProxy or Nginx. -## -## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ -## -## Value: on | off -## listener.tcp.external.proxy_protocol = on - -## Sets the timeout for proxy protocol. EMQ X will close the TCP connection -## if no proxy protocol packet recevied within the timeout. -## -## Value: Duration -## listener.tcp.external.proxy_protocol_timeout = 3s - -## Enable the option for X.509 certificate based authentication. -## EMQX will use the common name of certificate as MQTT username. -## Only support Proxy Protocol V2, the CN is available in Proxy Protocol V2 additional info -## -## Value: cn -## listener.tcp.external.peer_cert_as_username = cn - -## Enable the option for X.509 certificate based authentication. -## EMQX will use the common name of certificate as MQTT clientid. -## Only support Proxy Protocol V2, the CN is available in Proxy Protocol V2 additional info -## -## Value: cn -## listener.tcp.external.peer_cert_as_clientid = cn - -## The TCP backlog defines the maximum length that the queue of pending -## connections can grow to. -## -## Value: Number >= 0 -listener.tcp.external.backlog = 1024 - -## The TCP send timeout for external MQTT connections. -## -## Value: Duration -listener.tcp.external.send_timeout = 15s - -## Close the TCP connection if send timeout. -## -## Value: on | off -listener.tcp.external.send_timeout_close = on - -## The TCP receive buffer(os kernel) for MQTT connections. -## -## See: http://erlang.org/doc/man/inet.html -## -## Value: Bytes -## listener.tcp.external.recbuf = 2KB - -## The TCP send buffer(os kernel) for MQTT connections. -## -## See: http://erlang.org/doc/man/inet.html -## -## Value: Bytes -## listener.tcp.external.sndbuf = 2KB - -## The size of the user-level software buffer used by the driver. -## Not to be confused with options sndbuf and recbuf, which correspond -## to the Kernel socket buffers. It is recommended to have val(buffer) -## >= max(val(sndbuf),val(recbuf)) to avoid performance issues because -## of unnecessary copying. val(buffer) is automatically set to the above -## maximum when values sndbuf or recbuf are set. -## -## See: http://erlang.org/doc/man/inet.html -## -## Value: Bytes -## listener.tcp.external.buffer = 2KB - -## Sets the 'buffer = max(sndbuf, recbuf)' if this option is enabled. -## -## Value: on | off -## listener.tcp.external.tune_buffer = off - -## The socket is set to a busy state when the amount of data queued internally -## by the ERTS socket implementation reaches this limit. -## -## Value: on | off -## Defaults to 1MB -## listener.tcp.external.high_watermark = 1MB - -## The TCP_NODELAY flag for MQTT connections. Small amounts of data are -## sent immediately if the option is enabled. -## -## Value: true | false -listener.tcp.external.nodelay = true - -## The SO_REUSEADDR flag for TCP listener. -## -## Value: true | false -listener.tcp.external.reuseaddr = true - -##-------------------------------------------------------------------- -## Internal TCP Listener for MQTT Protocol - -## The IP address and port that the internal MQTT/TCP protocol listener -## will bind. -## -## Value: IP:Port, Port -## -## Examples: 11883, "127.0.0.1:11883", "::1:11883" -listener.tcp.internal.endpoint = "127.0.0.1:11883" - -## The acceptor pool for internal MQTT/TCP listener. -## -## Value: Number -listener.tcp.internal.acceptors = 4 - -## Maximum number of concurrent MQTT/TCP connections. -## -## Value: Number -listener.tcp.internal.max_connections = 1024000 - -## Maximum internal connections per second. -## -## Value: Number -listener.tcp.internal.max_conn_rate = 1000 - -## Specify the {active, N} option for the internal MQTT/TCP Socket. -## -## Value: Number -listener.tcp.internal.active_n = 1000 - -## Zone of the internal MQTT/TCP listener belonged to. -## -## Value: String -listener.tcp.internal.zone = internal - -## The TCP backlog of internal MQTT/TCP Listener. -## -## See: listener.tcp.$name.backlog -## -## Value: Number >= 0 -listener.tcp.internal.backlog = 512 - -## The TCP send timeout for internal MQTT connections. -## -## See: listener.tcp.$name.send_timeout -## -## Value: Duration -listener.tcp.internal.send_timeout = 5s - -## Close the MQTT/TCP connection if send timeout. -## -## See: listener.tcp.$name.send_timeout_close -## -## Value: on | off -listener.tcp.internal.send_timeout_close = on - -## The TCP receive buffer(os kernel) for internal MQTT connections. -## -## See: listener.tcp.$name.recbuf -## -## Value: Bytes -listener.tcp.internal.recbuf = 64KB - -## The TCP send buffer(os kernel) for internal MQTT connections. -## -## See: http://erlang.org/doc/man/inet.html -## -## Value: Bytes -listener.tcp.internal.sndbuf = 64KB - -## The size of the user-level software buffer used by the driver. -## -## See: listener.tcp.$name.buffer -## -## Value: Bytes -## listener.tcp.internal.buffer = 16KB - -## Sets the 'buffer = max(sndbuf, recbuf)' if this option is enabled. -## -## See: listener.tcp.$name.tune_buffer -## -## Value: on | off -## listener.tcp.internal.tune_buffer = off - -## The TCP_NODELAY flag for internal MQTT connections. -## -## See: listener.tcp.$name.nodelay -## -## Value: true | false -listener.tcp.internal.nodelay = false - -## The SO_REUSEADDR flag for MQTT/TCP Listener. -## -## Value: true | false -listener.tcp.internal.reuseaddr = true - -##-------------------------------------------------------------------- -## MQTT/SSL - External SSL Listener for MQTT Protocol - -## listener.ssl.$name is the IP address and port that the MQTT/SSL -## listener will bind. -## -## Value: IP:Port | Port -## -## Examples: 8883, "127.0.0.1:8883", "::1:8883" -listener.ssl.external.endpoint = 8883 - -## The acceptor pool for external MQTT/SSL listener. -## -## Value: Number -listener.ssl.external.acceptors = 16 - -## Maximum number of concurrent MQTT/SSL connections. -## -## Value: Number -listener.ssl.external.max_connections = 102400 - -## Maximum MQTT/SSL connections per second. -## -## Value: Number -listener.ssl.external.max_conn_rate = 500 - -## Specify the {active, N} option for the internal MQTT/SSL Socket. -## -## Value: Number -listener.ssl.external.active_n = 100 - -## Zone of the external MQTT/SSL listener belonged to. -## -## Value: String -listener.ssl.external.zone = external - -## The access control rules for the MQTT/SSL listener. -## -## See: listener.tcp.$name.access -## -## Value: ACL Rule -listener.ssl.external.access.1 = "allow all" - -## Enable the Proxy Protocol V1/2 if the EMQ cluster is deployed behind -## HAProxy or Nginx. -## -## See: listener.tcp.$name.proxy_protocol -## -## Value: on | off -## listener.ssl.external.proxy_protocol = on - -## Sets the timeout for proxy protocol. -## -## See: listener.tcp.$name.proxy_protocol_timeout -## -## Value: Duration -## listener.ssl.external.proxy_protocol_timeout = 3s - -## TLS versions only to protect from POODLE attack. -## -## See: http://erlang.org/doc/man/ssl.html -## -## Value: String, seperated by ',' -## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier -## listener.ssl.external.tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" - -## TLS Handshake timeout. -## -## Value: Duration -listener.ssl.external.handshake_timeout = 15s - -## Maximum number of non-self-issued intermediate certificates that -## can follow the peer certificate in a valid certification path. -## -## Value: Number -## listener.ssl.external.depth = 10 - -## String containing the user's password. Only used if the private keyfile -## is password-protected. -## -## Value: String -## listener.ssl.external.key_password = yourpass - -## Path to the file containing the user's private PEM-encoded key. -## -## See: http://erlang.org/doc/man/ssl.html -## -## Value: File -listener.ssl.external.keyfile = "{{ platform_etc_dir }}/certs/key.pem" - -## Path to a file containing the user certificate. -## -## See: http://erlang.org/doc/man/ssl.html -## -## Value: File -listener.ssl.external.certfile = "{{ platform_etc_dir }}/certs/cert.pem" - -## Path to the file containing PEM-encoded CA certificates. The CA certificates -## are used during server authentication and when building the client certificate chain. -## -## Value: File -## listener.ssl.external.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" - -## The Ephemeral Diffie-Helman key exchange is a very effective way of -## ensuring Forward Secrecy by exchanging a set of keys that never hit -## the wire. Since the DH key is effectively signed by the private key, -## it needs to be at least as strong as the private key. In addition, -## the default DH groups that most of the OpenSSL installations have -## are only a handful (since they are distributed with the OpenSSL -## package that has been built for the operating system it’s running on) -## and hence predictable (not to mention, 1024 bits only). -## In order to escape this situation, first we need to generate a fresh, -## strong DH group, store it in a file and then use the option above, -## to force our SSL application to use the new DH group. Fortunately, -## OpenSSL provides us with a tool to do that. Simply run: -## openssl dhparam -out dh-params.pem 2048 -## -## Value: File -## listener.ssl.external.dhfile = "{{ platform_etc_dir }}/certs/dh-params.pem" - -## A server only does x509-path validation in mode verify_peer, -## as it then sends a certificate request to the client (this -## message is not sent if the verify option is verify_none). -## You can then also want to specify option fail_if_no_peer_cert. -## More information at: http://erlang.org/doc/man/ssl.html -## -## Value: verify_peer | verify_none -## listener.ssl.external.verify = verify_peer - -## Used together with {verify, verify_peer} by an SSL server. If set to true, -## the server fails if the client does not have a certificate to send, that is, -## sends an empty certificate. -## -## Value: true | false -## listener.ssl.external.fail_if_no_peer_cert = true - -## This is the single most important configuration option of an Erlang SSL -## application. Ciphers (and their ordering) define the way the client and -## server encrypt information over the wire, from the initial Diffie-Helman -## key exchange, the session key encryption ## algorithm and the message -## digest algorithm. Selecting a good cipher suite is critical for the -## application’s data security, confidentiality and performance. -## -## The cipher list above offers: -## -## A good balance between compatibility with older browsers. -## It can get stricter for Machine-To-Machine scenarios. -## Perfect Forward Secrecy. -## No old/insecure encryption and HMAC algorithms -## -## Most of it was copied from Mozilla’s Server Side TLS article -## -## Value: Ciphers -listener.ssl.external.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" - -## Ciphers for TLS PSK. -## Note that 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot -## be configured at the same time. -## See 'https://tools.ietf.org/html/rfc4279#section-2'. -#listener.ssl.external.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" - -## SSL parameter renegotiation is a feature that allows a client and a server -## to renegotiate the parameters of the SSL connection on the fly. -## RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation, -## you drop support for the insecure renegotiation, prone to MitM attacks. -## -## Value: on | off -## listener.ssl.external.secure_renegotiate = off - -## A performance optimization setting, it allows clients to reuse -## pre-existing sessions, instead of initializing new ones. -## Read more about it here. -## -## See: http://erlang.org/doc/man/ssl.html -## -## Value: on | off -## listener.ssl.external.reuse_sessions = on - -## An important security setting, it forces the cipher to be set based -## on the server-specified order instead of the client-specified order, -## hence enforcing the (usually more properly configured) security -## ordering of the server administrator. -## -## Value: on | off -## listener.ssl.external.honor_cipher_order = on - -## Use the CN, DN or CRT field from the client certificate as a username. -## Notice that 'verify' should be set as 'verify_peer'. -## 'pem' encodes CRT in base64, and md5 is the md5 hash of CRT. -## -## Value: cn | dn | crt | pem | md5 -## listener.ssl.external.peer_cert_as_username = cn - -## Use the CN, DN or CRT field from the client certificate as a username. -## Notice that 'verify' should be set as 'verify_peer'. -## 'pem' encodes CRT in base64, and md5 is the md5 hash of CRT. -## -## Value: cn | dn | crt | pem | md5 -## listener.ssl.external.peer_cert_as_clientid = cn - -## TCP backlog for the SSL connection. -## -## See listener.tcp.$name.backlog -## -## Value: Number >= 0 -## listener.ssl.external.backlog = 1024 - -## The TCP send timeout for the SSL connection. -## -## See listener.tcp.$name.send_timeout -## -## Value: Duration -## listener.ssl.external.send_timeout = 15s - -## Close the SSL connection if send timeout. -## -## See: listener.tcp.$name.send_timeout_close -## -## Value: on | off -## listener.ssl.external.send_timeout_close = on - -## The TCP receive buffer(os kernel) for the SSL connections. -## -## See: listener.tcp.$name.recbuf -## -## Value: Bytes -## listener.ssl.external.recbuf = 4KB - -## The TCP send buffer(os kernel) for internal MQTT connections. -## -## See: listener.tcp.$name.sndbuf -## -## Value: Bytes -## listener.ssl.external.sndbuf = 4KB - -## The size of the user-level software buffer used by the driver. -## -## See: listener.tcp.$name.buffer -## -## Value: Bytes -## listener.ssl.external.buffer = 4KB - -## Sets the 'buffer = max(sndbuf, recbuf)' if this option is enabled. -## -## See: listener.tcp.$name.tune_buffer -## -## Value: on | off -## listener.ssl.external.tune_buffer = off - -## The TCP_NODELAY flag for SSL connections. -## -## See: listener.tcp.$name.nodelay -## -## Value: true | false -## listener.ssl.external.nodelay = true - -## The SO_REUSEADDR flag for MQTT/SSL Listener. -## -## Value: true | false -listener.ssl.external.reuseaddr = true - -##-------------------------------------------------------------------- -## External WebSocket listener for MQTT protocol - -## listener.ws.$name is the IP address and port that the MQTT/WebSocket -## listener will bind. -## -## Value: IP:Port | Port -## -## Examples: 8083, "127.0.0.1:8083", "::1:8083" -listener.ws.external.endpoint = 8083 - -## The path of WebSocket MQTT endpoint -## -## Value: URL Path -listener.ws.external.mqtt_path = "/mqtt" - -## The acceptor pool for external MQTT/WebSocket listener. -## -## Value: Number -listener.ws.external.acceptors = 4 - -## Maximum number of concurrent MQTT/WebSocket connections. -## -## Value: Number -listener.ws.external.max_connections = 102400 - -## Maximum MQTT/WebSocket connections per second. -## -## Value: Number -listener.ws.external.max_conn_rate = 1000 - -## Simulate the {active, N} option for the MQTT/WebSocket connections. -## -## Value: Number -listener.ws.external.active_n = 100 - -## Zone of the external MQTT/WebSocket listener belonged to. -## -## Value: String -listener.ws.external.zone = external - -## The access control for the MQTT/WebSocket listener. -## -## See: listener.ws.$name.access -## -## Value: ACL Rule -listener.ws.external.access.1 = "allow all" - -## If set to true, the server fails if the client does not have a Sec-WebSocket-Protocol to send. -## Set to false for WeChat MiniApp. -## -## Value: true | false -## listener.ws.external.fail_if_no_subprotocol = true - -## Supported subprotocols -## -## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 -## listener.ws.external.supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" - -## Specify which HTTP header for real source IP if the EMQ X cluster is -## deployed behind NGINX or HAProxy. -## -## Default: X-Forwarded-For -## listener.ws.external.proxy_address_header = X-Forwarded-For - -## Specify which HTTP header for real source port if the EMQ X cluster is -## deployed behind NGINX or HAProxy. -## -## Default: X-Forwarded-Port -## listener.ws.external.proxy_port_header = X-Forwarded-Port - -## Enable the Proxy Protocol V1/2 if the EMQ cluster is deployed behind -## HAProxy or Nginx. -## -## See: listener.ws.$name.proxy_protocol -## -## Value: on | off -## listener.ws.external.proxy_protocol = on - -## Sets the timeout for proxy protocol. -## -## See: listener.ws.$name.proxy_protocol_timeout -## -## Value: Duration -## listener.ws.external.proxy_protocol_timeout = 3s - -## Enable the option for X.509 certificate based authentication. -## EMQX will use the common name of certificate as MQTT username. -## Only support Proxy Protocol V2, the CN is available in Proxy Protocol V2 additional info -## -## Value: cn -## listener.ws.external.peer_cert_as_username = cn - -## Enable the option for X.509 certificate based authentication. -## EMQX will use the common name of certificate as MQTT clientid. -## Only support Proxy Protocol V2, the CN is available in Proxy Protocol V2 additional info -## -## Value: cn -## listener.ws.external.peer_cert_as_clientid = cn - -## The TCP backlog of external MQTT/WebSocket Listener. -## -## See: listener.ws.$name.backlog -## -## Value: Number >= 0 -listener.ws.external.backlog = 1024 - -## The TCP send timeout for external MQTT/WebSocket connections. -## -## See: listener.ws.$name.send_timeout -## -## Value: Duration -listener.ws.external.send_timeout = 15s - -## Close the MQTT/WebSocket connection if send timeout. -## -## See: listener.ws.$name.send_timeout_close -## -## Value: on | off -listener.ws.external.send_timeout_close = on - -## The TCP receive buffer(os kernel) for external MQTT/WebSocket connections. -## -## See: listener.ws.$name.recbuf -## -## Value: Bytes -## listener.ws.external.recbuf = 2KB - -## The TCP send buffer(os kernel) for external MQTT/WebSocket connections. -## -## See: listener.ws.$name.sndbuf -## -## Value: Bytes -## listener.ws.external.sndbuf = 2KB - -## The size of the user-level software buffer used by the driver. -## -## See: listener.ws.$name.buffer -## -## Value: Bytes -## listener.ws.external.buffer = 2KB - -## Sets the 'buffer = max(sndbuf, recbuf)' if this option is enabled. -## -## See: listener.ws.$name.tune_buffer -## -## Value: on | off -## listener.ws.external.tune_buffer = off - -## The TCP_NODELAY flag for external MQTT/WebSocket connections. -## -## See: listener.ws.$name.nodelay -## -## Value: true | false -listener.ws.external.nodelay = true - -## The compress flag for external MQTT/WebSocket connections. -## -## If this Value is set true,the websocket message would be compressed -## -## Value: true | false -## listener.ws.external.compress = true - -## The level of deflate options for external MQTT/WebSocket connections. -## -## See: listener.ws.$name.deflate_opts.level -## -## Value: none | default | best_compression | best_speed -## listener.ws.external.deflate_opts.level = default - -## The mem_level of deflate options for external MQTT/WebSocket connections. -## -## See: listener.ws.$name.deflate_opts.mem_level -## -## Valid range is 1-9 -## listener.ws.external.deflate_opts.mem_level = 8 - -## The strategy of deflate options for external MQTT/WebSocket connections. -## -## See: listener.ws.$name.deflate_opts.strategy -## -## Value: default | filtered | huffman_only | rle -## listener.ws.external.deflate_opts.strategy = default - -## The deflate option for external MQTT/WebSocket connections. -## -## See: listener.ws.$name.deflate_opts.server_context_takeover -## -## Value: takeover | no_takeover -## listener.ws.external.deflate_opts.server_context_takeover = takeover - -## The deflate option for external MQTT/WebSocket connections. -## -## See: listener.ws.$name.deflate_opts.client_context_takeover -## -## Value: takeover | no_takeover -## listener.ws.external.deflate_opts.client_context_takeover = takeover - -## The deflate options for external MQTT/WebSocket connections. -## -## See: listener.ws.$name.deflate_opts.server_max_window_bits -## -## Valid range is 8-15 -## listener.ws.external.deflate_opts.server_max_window_bits = 15 - -## The deflate options for external MQTT/WebSocket connections. -## -## See: listener.ws.$name.deflate_opts.client_max_window_bits -## -## Valid range is 8-15 -## listener.ws.external.deflate_opts.client_max_window_bits = 15 - -## The idle timeout for external MQTT/WebSocket connections. -## -## See: listener.ws.$name.idle_timeout -## -## Value: Duration -## listener.ws.external.idle_timeout = 60s - -## The max frame size for external MQTT/WebSocket connections. -## -## -## Value: Number -## listener.ws.external.max_frame_size = 0 - -## Whether a WebSocket message is allowed to contain multiple MQTT packets -## -## Value: single | multiple -listener.ws.external.mqtt_piggyback = multiple - -## By default, EMQX web socket connection does not restrict connections to specific origins. -## It also, by default, does not enforce the presence of origin in request headers for WebSocket connections. -## Because of this, a malicious user could potentially hijack an existing web-socket connection to EMQX. - -## To prevent this, users can set allowed origin headers in their ws connection to EMQX. -## WS configs are set in listener.ws.external.* -## WSS configs are set in listener.wss.external.* - -## Example for WS connection -## To enables origin check in header for websocket connnection, -## set `listener.ws.external.check_origin_enable = true`. By default it is false, -## When it is set to true and no origin is present in the header of a ws connection request, the request fails. - -## To allow origins to be absent in header in the websocket connection when check_origin_enable is true, -## set `listener.ws.external.allow_origin_absence = true` - -## Enabling origin check implies there are specific valid origins allowed for ws connection. -## To set the list of allowed origins in header for websocket connection -## listener.ws.external.check_origins = http://localhost:18083(localhost dashboard url), http://yourapp.com` -## check_origins config allows a comma separated list of origins so you can specify as many origins are you want. -## With these configs, you can allow only connections from only authorized origins to your broker - -## Enable origin check in header for websocket connection -## -## Value: true | false (default false) -listener.ws.external.check_origin_enable = false - -## Allow origin to be absent in header in websocket connection when check_origin_enable is true -## -## Value: true | false (default true) -listener.ws.external.allow_origin_absence = true - -## Comma separated list of allowed origin in header for websocket connection -## -## Value: http://url eg. local http dashboard url - http://localhost:18083, http://127.0.0.1:18083 -listener.ws.external.check_origins = "http://localhost:18083, http://127.0.0.1:18083" - -##-------------------------------------------------------------------- -## External WebSocket/SSL listener for MQTT Protocol - -## listener.wss.$name is the IP address and port that the MQTT/WebSocket/SSL -## listener will bind. -## -## Value: IP:Port | Port -## -## Examples: 8084, "127.0.0.1:8084", "::1:8084" -listener.wss.external.endpoint = 8084 - -## The path of WebSocket MQTT endpoint -## -## Value: URL Path -listener.wss.external.mqtt_path = "/mqtt" - -## The acceptor pool for external MQTT/WebSocket/SSL listener. -## -## Value: Number -listener.wss.external.acceptors = 4 - -## Maximum number of concurrent MQTT/Webwocket/SSL connections. -## -## Value: Number -listener.wss.external.max_connections = 16 - -## Maximum MQTT/WebSocket/SSL connections per second. -## -## See: listener.tcp.$name.max_conn_rate -## -## Value: Number -listener.wss.external.max_conn_rate = 1000 - -## Simulate the {active, N} option for the MQTT/WebSocket/SSL connections. -## -## Value: Number -listener.wss.external.active_n = 100 - -## Zone of the external MQTT/WebSocket/SSL listener belonged to. -## -## Value: String -listener.wss.external.zone = external - -## The access control rules for the MQTT/WebSocket/SSL listener. -## -## See: listener.tcp.$name.access. -## -## Value: ACL Rule -listener.wss.external.access.1 = "allow all" - -## If set to true, the server fails if the client does not have a Sec-WebSocket-Protocol to send. -## Set to false for WeChat MiniApp. -## -## Value: true | false -## listener.wss.external.fail_if_no_subprotocol = true - -## Supported subprotocols -## -## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 -## listener.wss.external.supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" - -## Specify which HTTP header for real source IP if the EMQ X cluster is -## deployed behind NGINX or HAProxy. -## -## Default: X-Forwarded-For -## listener.wss.external.proxy_address_header = X-Forwarded-For - -## Specify which HTTP header for real source port if the EMQ X cluster is -## deployed behind NGINX or HAProxy. -## -## Default: X-Forwarded-Port -## listener.wss.external.proxy_port_header = X-Forwarded-Port - -## Enable the Proxy Protocol V1/2 support. -## -## See: listener.tcp.$name.proxy_protocol -## -## Value: on | off -## listener.wss.external.proxy_protocol = on - -## Sets the timeout for proxy protocol. -## -## See: listener.tcp.$name.proxy_protocol_timeout -## -## Value: Duration -## listener.wss.external.proxy_protocol_timeout = 3s - -## TLS versions only to protect from POODLE attack. -## -## See: listener.ssl.$name.tls_versions -## -## Value: String, seperated by ',' -## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier -## listener.wss.external.tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" - -## Path to the file containing the user's private PEM-encoded key. -## -## See: listener.ssl.$name.keyfile -## -## Value: File -listener.wss.external.keyfile = "{{ platform_etc_dir }}/certs/key.pem" - -## Path to a file containing the user certificate. -## -## See: listener.ssl.$name.certfile -## -## Value: File -listener.wss.external.certfile = "{{ platform_etc_dir }}/certs/cert.pem" - -## Path to the file containing PEM-encoded CA certificates. -## -## See: listener.ssl.$name.cacert -## -## Value: File -## listener.wss.external.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" - -## Maximum number of non-self-issued intermediate certificates that -## can follow the peer certificate in a valid certification path. -## -## See: listener.ssl.external.depth -## -## Value: Number -## listener.wss.external.depth = 10 - -## String containing the user's password. Only used if the private keyfile -## is password-protected. -## -## See: listener.ssl.$name.key_password -## -## Value: String -## listener.wss.external.key_password = yourpass - -## See: listener.ssl.$name.dhfile -## -## Value: File -## listener.ssl.external.dhfile = "{{ platform_etc_dir }}/certs/dh-params.pem" - -## See: listener.ssl.$name.verify -## -## Value: verify_peer | verify_none -## listener.wss.external.verify = verify_peer - -## See: listener.ssl.$name.fail_if_no_peer_cert -## -## Value: false | true -## listener.wss.external.fail_if_no_peer_cert = true - -## See: listener.ssl.$name.ciphers -## -## Value: Ciphers -listener.wss.external.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" - -## Ciphers for TLS PSK. -## Note that 'listener.wss.external.ciphers' and 'listener.wss.external.psk_ciphers' cannot -## be configured at the same time. -## See 'https://tools.ietf.org/html/rfc4279#section-2'. -## listener.wss.external.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" - -## See: listener.ssl.$name.secure_renegotiate -## -## Value: on | off -## listener.wss.external.secure_renegotiate = off - -## See: listener.ssl.$name.reuse_sessions -## -## Value: on | off -## listener.wss.external.reuse_sessions = on - -## See: listener.ssl.$name.honor_cipher_order -## -## Value: on | off -## listener.wss.external.honor_cipher_order = on - -## See: listener.ssl.$name.peer_cert_as_username -## -## Value: cn | dn | crt | pem | md5 -## listener.wss.external.peer_cert_as_username = cn - -## See: listener.ssl.$name.peer_cert_as_clientid -## -## Value: cn | dn | crt | pem | md5 -## listener.wss.external.peer_cert_as_clientid = cn - -## TCP backlog for the WebSocket/SSL connection. -## -## See: listener.tcp.$name.backlog -## -## Value: Number >= 0 -listener.wss.external.backlog = 1024 - -## The TCP send timeout for the WebSocket/SSL connection. -## -## See: listener.tcp.$name.send_timeout -## -## Value: Duration -listener.wss.external.send_timeout = 15s - -## Close the WebSocket/SSL connection if send timeout. -## -## See: listener.tcp.$name.send_timeout_close -## -## Value: on | off -listener.wss.external.send_timeout_close = on - -## The TCP receive buffer(os kernel) for the WebSocket/SSL connections. -## -## See: listener.tcp.$name.recbuf -## -## Value: Bytes -## listener.wss.external.recbuf = 4KB - -## The TCP send buffer(os kernel) for the WebSocket/SSL connections. -## -## See: listener.tcp.$name.sndbuf -## -## Value: Bytes -## listener.wss.external.sndbuf = 4KB - -## The size of the user-level software buffer used by the driver. -## -## See: listener.tcp.$name.buffer -## -## Value: Bytes -## listener.wss.external.buffer = 4KB - -## The TCP_NODELAY flag for WebSocket/SSL connections. -## -## See: listener.tcp.$name.nodelay -## -## Value: true | false -## listener.wss.external.nodelay = true - -## The compress flag for external WebSocket/SSL connections. -## -## If this Value is set true,the websocket message would be compressed -## -## Value: true | false -## listener.wss.external.compress = true - -## The level of deflate options for external WebSocket/SSL connections. -## -## See: listener.wss.$name.deflate_opts.level -## -## Value: none | default | best_compression | best_speed -## listener.wss.external.deflate_opts.level = default - -## The mem_level of deflate options for external WebSocket/SSL connections. -## -## See: listener.wss.$name.deflate_opts.mem_level -## -## Valid range is 1-9 -## listener.wss.external.deflate_opts.mem_level = 8 - -## The strategy of deflate options for external WebSocket/SSL connections. -## -## See: listener.wss.$name.deflate_opts.strategy -## -## Value: default | filtered | huffman_only | rle -## listener.wss.external.deflate_opts.strategy = default - -## The deflate option for external WebSocket/SSL connections. -## -## See: listener.wss.$name.deflate_opts.server_context_takeover -## -## Value: takeover | no_takeover -## listener.wss.external.deflate_opts.server_context_takeover = takeover - -## The deflate option for external WebSocket/SSL connections. -## -## See: listener.wss.$name.deflate_opts.client_context_takeover -## -## Value: takeover | no_takeover -## listener.wss.external.deflate_opts.client_context_takeover = takeover - -## The deflate options for external WebSocket/SSL connections. -## -## See: listener.wss.$name.deflate_opts.server_max_window_bits -## -## Valid range is 8-15 -## listener.wss.external.deflate_opts.server_max_window_bits = 15 - -## The deflate options for external WebSocket/SSL connections. -## -## See: listener.wss.$name.deflate_opts.client_max_window_bits -## -## Valid range is 8-15 -## listener.wss.external.deflate_opts.client_max_window_bits = 15 - -## The idle timeout for external WebSocket/SSL connections. -## -## See: listener.wss.$name.idle_timeout -## -## Value: Duration -## listener.wss.external.idle_timeout = 60s - -## The max frame size for external WebSocket/SSL connections. -## -## Value: Number -## listener.wss.external.max_frame_size = 0 - -## Whether a WebSocket message is allowed to contain multiple MQTT packets -## -## Value: single | multiple -listener.wss.external.mqtt_piggyback = multiple -## Enable origin check in header for secure websocket connection -## -## Value: true | false (default false) -listener.wss.external.check_origin_enable = false -## Allow origin to be absent in header in secure websocket connection when check_origin_enable is true -## -## Value: true | false (default true) -listener.wss.external.allow_origin_absence = true -## Comma separated list of allowed origin in header for secure websocket connection -## -## Value: http://url eg. https://localhost:8084, https://127.0.0.1:8084 -listener.wss.external.check_origins = "https://localhost:8084, https://127.0.0.1:8084" - -## CONFIG_SECTION_END=listeners ================================================ - -## CONFIG_SECTION_BGN=modules ================================================== - -## The file to store loaded module names. -## -## Value: File -module.loaded_file = "{{ platform_data_dir }}/loaded_modules" - -##-------------------------------------------------------------------- -## Presence Module - -## Sets the QoS for presence MQTT message. -## -## Value: 0 | 1 | 2 -module.presence.qos = 1 - -##-------------------------------------------------------------------- -## Subscription Module - -## Subscribe the Topics automatically when client connected. -## -## Value: String -## module.subscription.1.topic = "connected/%c/%u" - -## Qos of the proxy subscription. -## -## Value: 0 | 1 | 2 -## Default: 0 -## module.subscription.1.qos = 0 - -## No Local of the proxy subscription options. -## This configuration only takes effect in the MQTT V5 protocol. -## -## Value: 0 | 1 -## Default: 0 -## module.subscription.1.nl = 0 - -## Retain As Published of the proxy subscription options. -## This configuration only takes effect in the MQTT V5 protocol. -## -## Value: 0 | 1 -## Default: 0 -## module.subscription.1.rap = 0 - -## Retain Handling of the proxy subscription options. -## This configuration only takes effect in the MQTT V5 protocol. -## -## Value: 0 | 1 | 2 -## Default: 0 -## module.subscription.1.rh = 0 - -##-------------------------------------------------------------------- -## Rewrite Module - -## {rewrite, Topic, Re, Dest} -## module.rewrite.pub_rule.1 = "x/# ^x/y/(.+)$ z/y/$1" -## module.rewrite.sub_rule.1 = "y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2" - -## CONFIG_SECTION_END=modules ================================================== - -##------------------------------------------------------------------- -## Plugins -##------------------------------------------------------------------- - -## The etc dir for plugins' config. -## -## Value: Folder -plugins.etc_dir = "{{ platform_etc_dir }}/plugins/" - -## The file to store loaded plugin names. -## -## Value: File -plugins.loaded_file = "{{ platform_data_dir }}/loaded_plugins" - -## The directory of extension plugins. -## -## Value: File -plugins.expand_plugins_dir = "{{ platform_plugins_dir }}/" - -##-------------------------------------------------------------------- +##================================================================== +node { + ## Node name. + ## See: http://erlang.org/doc/reference_manual/distributed.html + ## + ## @doc node.name + ## ValueType: NodeName + ## Default: emqx@127.0.0.1 + name: "emqx@127.0.0.1" + + ## Cookie for distributed node communication. + ## + ## @doc node.cookie + ## ValueType: String + ## Default: emqxsecretcookie + cookie: emqxsecretcookie + + ## Data dir for the node + ## + ## @doc node.data_dir + ## ValueType: Folder + ## Default: "{{ platform_data_dir }}/" + data_dir: "{{ platform_data_dir }}/" + + ## Dir of crash dump file. + ## + ## @doc node.crash_dump_dir + ## ValueType: Folder + ## Default: "{{ platform_log_dir }}/" + crash_dump_dir: "{{ platform_log_dir }}/" + + ## Global GC Interval. + ## + ## @doc node.global_gc_interval + ## ValueType: Duration + ## Default: 15m + global_gc_interval: 15m + + ## Sets the net_kernel tick time in seconds. + ## Notice that all communicating nodes are to have the same + ## TickTime value specified. + ## + ## See: http://www.erlang.org/doc/man/kernel_app.html#net_ticktime + ## + ## @doc node.dist_net_ticktime + ## ValueType: Number + ## Default: 2m + dist_net_ticktime: 2m + + ## Sets the port range for the listener socket of a distributed + ## Erlang node. + ## Note that if there are firewalls between clustered nodes, this + ## port segment for nodes’ communication should be allowed. + ## + ## See: http://www.erlang.org/doc/man/kernel_app.html + ## + ## @doc node.dist_listen_min + ## ValueType: Integer + ## Range: [1024,65535] + ## Default: 6369 + dist_listen_min: 6369 + + ## Sets the port range for the listener socket of a distributed + ## Erlang node. + ## Note that if there are firewalls between clustered nodes, this + ## port segment for nodes’ communication should be allowed. + ## + ## See: http://www.erlang.org/doc/man/kernel_app.html + ## + ## @doc node.dist_listen_max + ## ValueType: Integer + ## Range: [1024,65535] + ## Default: 6369 + dist_listen_max: 6369 + + ## Sets the maximum depth of call stack back-traces in the exit + ## reason element of 'EXIT' tuples. + ## The flag also limits the stacktrace depth returned by + ## process_info item current_stacktrace. + ## + ## @doc node.backtrace_depth + ## ValueType: Integer + ## Range: [0,1024] + ## Default: 23 + backtrace_depth: 23 + +} + +##================================================================== +## Cluster +##================================================================== +cluster { + ## Cluster name. + ## + ## @doc cluster.name + ## ValueType: String + ## Default: emqxcl + name: emqxcl + + ## Enable cluster autoheal from network partition. + ## + ## @doc cluster.autoheal + ## ValueType: Boolean + ## Default: true + autoheal: true + + ## Autoclean down node. A down node will be removed from the cluster + ## if this value > 0. + ## + ## @doc cluster.autoclean + ## ValueType: Duration + ## Default: 5m + autoclean: 5m + + ## Node discovery strategy to join the cluster. + ## + ## @doc cluster.discovery_strategy + ## ValueType: manual | static | mcast | dns | etcd | k8s + ## - manual: Manual join command + ## - static: Static node list + ## - mcast: IP Multicast + ## - dns: DNS A Record + ## - etcd: etcd + ## - k8s: Kubernetes + ## + ## Default: manual + discovery_strategy: manual + + ##---------------------------------------------------------------- + ## Cluster using static node list + ##---------------------------------------------------------------- + static { + ## Node list of the cluster + ## + ## @doc cluster.static.seeds + ## ValueType: Array + ## Default: [] + seeds: ["emqx1@127.0.0.1", "emqx2@127.0.0.1"] + } + + ##---------------------------------------------------------------- + ## Cluster using IP Multicast + ##---------------------------------------------------------------- + mcast { + ## IP Multicast Address. + ## + ## @doc cluster.mcast.addr + ## ValueType: IPAddress + ## Default: "239.192.0.1" + addr: "239.192.0.1" + + ## Multicast Ports. + ## + ## @doc cluster.mcast.ports + ## ValueType: Array + ## Default: [4369, 4370] + ports: [4369, 4370] + + ## Multicast Iface. + ## + ## @doc cluster.mcast.iface + ## ValueType: IPAddress + ## Default: "0.0.0.0" + iface: "0.0.0.0" + + ## Multicast Ttl. + ## + ## @doc cluster.mcast.ttl + ## ValueType: Integer + ## Range: [0,255] + ## Default: 255 + ttl: 255 + + ## Multicast loop. + ## + ## @doc cluster.mcast.loop + ## ValueType: Boolean + ## Default: true + loop: true + } + + ##---------------------------------------------------------------- + ## Cluster using DNS A records + ##---------------------------------------------------------------- + dns { + ## DNS name. + ## + ## @doc cluster.dns.name + ## ValueType: String + ## Default: localhost + name: localhost + + ## The App name is used to build 'node.name' with IP address. + ## + ## @doc cluster.dns.app + ## ValueType: String + ## Default: emqx + app: emqx + } + + ##---------------------------------------------------------------- + ## Cluster using etcd + ##---------------------------------------------------------------- + etcd { + ## Etcd server list, seperated by ','. + ## + ## @doc cluster.etcd.server + ## ValueType: URL + ## Required: true + server: "http://127.0.0.1:2379" + + ## The prefix helps build nodes path in etcd. Each node in the cluster + ## will create a path in etcd: v2/keys/// + ## + ## @doc cluster.etcd.prefix + ## ValueType: String + ## Default: emqxcl + prefix: emqxcl + + ## The TTL for node's path in etcd. + ## + ## @doc cluster.etcd.node_ttl + ## ValueType: Duration + ## Default: 1m + node_ttl: 1m + + ## Path to the file containing the user's private PEM-encoded key. + ## + ## @doc cluster.etcd.ssl.keyfile + ## ValueType: File + ## Default: "{{ platform_etc_dir }}/certs/key.pem" + ssl.keyfile: "{{ platform_etc_dir }}/certs/key.pem" + + ## Path to a file containing the user certificate. + ## + ## @doc cluster.etcd.ssl.certfile + ## ValueType: File + ## Default: "{{ platform_etc_dir }}/certs/cert.pem" + ssl.certfile: "{{ platform_etc_dir }}/certs/cert.pem" + + ## Path to the file containing PEM-encoded CA certificates. The CA certificates + ## are used during server authentication and when building the client certificate chain. + ## + ## @doc cluster.etcd.ssl.cacertfile + ## ValueType: File + ## Default: "{{ platform_etc_dir }}/certs/cacert.pem" + ssl.cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem" + } + + ##---------------------------------------------------------------- + ## Cluster using Kubernetes + ##---------------------------------------------------------------- + k8s { + ## Kubernetes API server list, seperated by ','. + ## + ## @doc cluster.k8s.apiserver + ## ValueType: URL + ## Required: true + apiserver: "http://10.110.111.204:8080" + + ## The service name helps lookup EMQ nodes in the cluster. + ## + ## @doc cluster.k8s.service_name + ## ValueType: String + ## Default: emqx + service_name: emqx + + ## The address type is used to extract host from k8s service. + ## + ## @doc cluster.k8s.address_type + ## ValueType: ip | dns | hostname + ## Default: ip + address_type: ip + + ## The app name helps build 'node.name'. + ## + ## @doc cluster.k8s.app_name + ## ValueType: String + ## Default: emqx + app_name: emqx + + ## The suffix added to dns and hostname get from k8s service + ## + ## @doc cluster.k8s.suffix + ## ValueType: String + ## Default: "pod.local" + suffix: "pod.local" + + ## Kubernetes Namespace + ## + ## @doc cluster.k8s.namespace + ## ValueType: String + ## Default: default + namespace: default + } + + db_backend: mnesia + + rlog: { + # role: core + # core_nodes: [] + } + +} + +##================================================================== +## Log +##================================================================== +log { + ## The primary log level + ## + ## - all the log messages with levels lower than this level will + ## be dropped. + ## - all the log messages with levels higher than this level will + ## go into the log handlers. The handlers then decide to log it + ## out or drop it according to the level setting of the handler. + ## + ## Note: Only the messages with severity level higher than or + ## equal to this level will be logged. + ## + ## @doc log.primary_level + ## ValueType: debug | info | notice | warning | error | critical | alert | emergency + ## Default: warning + primary_level: warning + + ##---------------------------------------------------------------- + ## The console log handler send log messages to emqx console + ##---------------------------------------------------------------- + ## Log to single line + ## @doc log.console_handler.enable + ## ValueType: Boolean + ## Default: false + console_handler.enable: false + + ## The log level of this handler + ## All the log messages with levels lower than this level will + ## be dropped. + ## + ## @doc log.console_handler.level + ## ValueType: debug | info | notice | warning | error | critical | alert | emergency + ## Default: warning + console_handler.level: warning + + ##---------------------------------------------------------------- + ## The file log handlers send log messages to files + ##---------------------------------------------------------------- + ## file_handlers. + file_handlers.emqx_log: { + ## The log level filter of this handler + ## All the log messages with levels lower than this level will + ## be dropped. + ## + ## @doc log.file_handlers..level + ## ValueType: debug | info | notice | warning | error | critical | alert | emergency + ## Default: warning + level: warning + + ## The log file for specified level. + ## + ## If `rotation` is disabled, this is the file of the log files. + ## + ## If `rotation` is enabled, this is the base name of the files. + ## Each file in a rotated log is named .N, where N is an integer. + ## + ## Note: Log files for a specific log level will only contain all the logs + ## that higher than or equal to that level + ## + ## @doc log.file_handlers..file + ## ValueType: File + ## Required: true + file: "{{ platform_log_dir }}/emqx.log" + + ## Enables the log rotation. + ## With this enabled, new log files will be created when the current + ## log file is full, max to `rotation_count` files will be created. + ## + ## @doc log.file_handlers..rotation.enable + ## ValueType: Boolean + ## Default: true + rotation.enable: true + + ## Maximum rotation count of log files. + ## + ## @doc log.file_handlers..rotation.count + ## ValueType: Integer + ## Range: [1, 2048] + ## Default: 10 + rotation.count: 10 + + ## Maximum size of each log file. + ## + ## If the max_size reached and `rotation` is disabled, the handler + ## will stop sending log messages, if the `rotation` is enabled, + ## the file rotates. + ## + ## @doc log.file_handlers..max_size + ## ValueType: Size | infinity + ## Default: 10MB + max_size: 10MB + } + + ## file_handlers. + ## + ## You could also create multiple file handlers for different + ## log level for example: + file_handlers.emqx_error_log: { + level: error + file: "{{ platform_log_dir }}/error.log" + } + + ## Timezone offset to display in logs + ## + ## @doc log.time_offset + ## ValueType: system | utc | String + ## - "system" use system zone + ## - "utc" for Universal Coordinated Time (UTC) + ## - "+hh:mm" or "-hh:mm" for a specified offset + ## Default: system + time_offset: system + + ## Limits the total number of characters printed for each log event. + ## + ## @doc log.chars_limit + ## ValueType: Integer | infinity + ## Range: [0, infinity) + ## Default: infinity + chars_limit: infinity + + ## Maximum depth for Erlang term log formatting + ## and Erlang process message queue inspection. + ## + ## @doc log.max_depth + ## ValueType: Integer | infinity + ## Default: 80 + max_depth: 80 + + ## Log formatter + ## @doc log.formatter + ## ValueType: text | json + ## Default: text + formatter: text + + ## Log to single line + ## @doc log.single_line + ## ValueType: Boolean + ## Default: true + single_line: true + + ## The max allowed queue length before switching to sync mode. + ## + ## Log overload protection parameter. If the message queue grows + ## larger than this value the handler switches from anync to sync mode. + ## + ## @doc log.sync_mode_qlen + ## ValueType: Integer + ## Range: [0, ${log.drop_mode_qlen}] + ## Default: 100 + sync_mode_qlen: 100 + + ## The max allowed queue length before switching to drop mode. + ## + ## Log overload protection parameter. When the message queue grows + ## larger than this threshold, the handler switches to a mode in which + ## it drops all new events that senders want to log. + ## + ## @doc log.drop_mode_qlen + ## ValueType: Integer + ## Range: [${log.sync_mode_qlen}, ${log.flush_qlen}] + ## Default: 3000 + drop_mode_qlen: 3000 + + ## The max allowed queue length before switching to flush mode. + ## + ## Log overload protection parameter. If the length of the message queue + ## grows larger than this threshold, a flush (delete) operation takes place. + ## To flush events, the handler discards the messages in the message queue + ## by receiving them in a loop without logging. + ## + ## @doc log.flush_qlen + ## ValueType: Integer + ## Range: [${log.drop_mode_qlen}, infinity) + ## Default: 8000 + flush_qlen: 8000 + + ## Kill the log handler when it gets overloaded. + ## + ## Log overload protection parameter. It is possible that a handler, + ## even if it can successfully manage peaks of high load without crashing, + ## can build up a large message queue, or use a large amount of memory. + ## We could kill the log handler in these cases and restart it after a + ## few seconds. + ## + ## @doc log.overload_kill.enable + ## ValueType: Boolean + ## Default: true + overload_kill.enable: true + + ## The max allowed queue length before killing the log hanlder. + ## + ## Log overload protection parameter. This is the maximum allowed queue + ## length. If the message queue grows larger than this, the handler + ## process is terminated. + ## + ## @doc log.overload_kill.qlen + ## ValueType: Integer + ## Range: [0, 1048576] + ## Default: 20000 + overload_kill.qlen: 20000 + + ## The max allowed memory size before killing the log hanlder. + ## + ## Log overload protection parameter. This is the maximum memory size + ## that the handler process is allowed to use. If the handler grows + ## larger than this, the process is terminated. + ## + ## @doc log.overload_kill.mem_size + ## ValueType: Size + ## Default: 30MB + overload_kill.mem_size: 30MB + + ## Restart the log hanlder after some seconds. + ## + ## Log overload protection parameter. If the handler is terminated, + ## it restarts automatically after a delay specified in seconds. + ## + ## @doc log.overload_kill.restart_after + ## ValueType: Duration + ## Default: 5s + overload_kill.restart_after: 5s + + ## Controlling Bursts of Log Requests. + ## + ## Log overload protection parameter. Large bursts of log events - many + ## events received by the handler under a short period of time - can + ## potentially cause problems. By specifying the maximum number of events + ## to be handled within a certain time frame, the handler can avoid + ## choking the log with massive amounts of printouts. + ## + ## Note that there would be no warning if any messages were + ## dropped because of burst control. + ## + ## @doc log.burst_limit.enable + ## ValueType: Boolean + ## Default: false + burst_limit.enable: false + + ## This config controls the maximum number of events to handle within + ## a time frame. After the limit is reached, successive events are + ## dropped until the end of the time frame defined by `window_time`. + ## + ## @doc log.burst_limit.max_count + ## ValueType: Integer + ## Default: 10000 + burst_limit.max_count: 10000 + + ## See the previous description of burst_limit_max_count. + ## + ## @doc log.burst_limit.window_time + ## ValueType: duration + ## Default: 1s + burst_limit.window_time: 1s +} + +##================================================================== +## RPC +##================================================================== +rpc { + ## RPC Mode. + ## + ## @doc rpc.mode + ## ValueType: sync | async + ## Default: async + mode: async + + ## Max batch size of async RPC requests. + ## + ## NOTE: RPC batch won't work when rpc.mode = sync + ## Zero value disables rpc batching. + ## + ## @doc rpc.async_batch_size + ## ValueType: Integer + ## Range: [0, 1048576] + ## Default: 0 + async_batch_size: 256 + + ## RPC port discovery + ## + ## The strategy for discovering the RPC listening port of + ## other nodes. + ## + ## @doc cluster.discovery_strategy + ## ValueType: manual | stateless + ## - manual: discover ports by `tcp_server_port`. + ## - stateless: discover ports in a stateless manner. + ## If node name is `emqx@127.0.0.1`, where the `` is + ## an integer, then the listening port will be `5370 + ` + ## + ## Default: `stateless`. + port_discovery: stateless + + ## TCP server port for RPC. + ## + ## Only takes effect when `rpc.port_discovery` = `manual`. + ## + ## @doc rpc.tcp_server_port + ## ValueType: Integer + ## Range: [1024-65535] + ## Defaults: 5369 + tcp_server_port: 5369 + + ## Number of outgoing RPC connections. + ## + ## Set this to 1 to keep the message order sent from the same + ## client. + ## + ## @doc rpc.tcp_client_num + ## ValueType: Integer + ## Range: [1, 256] + ## Defaults: 1 + tcp_client_num: 1 + + ## RCP Client connect timeout. + ## + ## @doc rpc.connect_timeout + ## ValueType: Duration + ## Default: 5s + connect_timeout: 5s + + ## TCP send timeout of RPC client and server. + ## + ## @doc rpc.send_timeout + ## ValueType: Duration + ## Default: 5s + send_timeout: 5s + + ## Authentication timeout + ## + ## @doc rpc.authentication_timeout + ## ValueType: Duration + ## Default: 5s + authentication_timeout: 5s + + ## Default receive timeout for call() functions + ## + ## @doc rpc.call_receive_timeout + ## ValueType: Duration + ## Default: 15s + call_receive_timeout: 15s + + ## Socket idle keepalive. + ## + ## @doc rpc.socket_keepalive_idle + ## ValueType: Duration + ## Default: 900s + socket_keepalive_idle: 900s + + ## TCP Keepalive probes interval. + ## + ## @doc rpc.socket_keepalive_interval + ## ValueType: Duration + ## Default: 75s + socket_keepalive_interval: 75s + + ## Probes lost to close the connection + ## + ## @doc rpc.socket_keepalive_count + ## ValueType: Integer + ## Default: 9 + socket_keepalive_count: 9 + + ## Size of TCP send buffer. + ## + ## @doc rpc.socket_sndbuf + ## ValueType: Size + ## Default: 1MB + socket_sndbuf: 1MB + + ## Size of TCP receive buffer. + ## + ## @doc rpc.socket_recbuf + ## ValueType: Size + ## Default: 1MB + socket_recbuf: 1MB + + ## Size of user-level software socket buffer. + ## + ## @doc rpc.socket_buffer + ## ValueType: Size + ## Default: 1MB + socket_buffer: 1MB +} + +##================================================================== ## Broker -##-------------------------------------------------------------------- +##================================================================== +broker { + ## System interval of publishing $SYS messages. + ## + ## @doc broker.sys_msg_interval + ## ValueType: Duration | disabled + ## Default: 1m + sys_msg_interval: 1m -## System interval of publishing $SYS messages. -## -## Value: Duration -## Default: 1m, 1 minute -broker.sys_interval = 1m + ## System heartbeat interval of publishing following heart beat message: + ## - "$SYS/brokers//uptime" + ## - "$SYS/brokers//datetime" + ## + ## @doc broker.sys_heartbeat_interval + ## ValueType: Duration + ## Default: 30s | disabled + sys_heartbeat_interval: 30s -## System heartbeat interval of publishing following heart beat message: -## - "$SYS/brokers//uptime" -## - "$SYS/brokers//datetime" -## -## Value: Duration -## Default: 30s -broker.sys_heartbeat = 30s + ## Session locking strategy in a cluster. + ## + ## @doc broker.session_locking_strategy + ## ValueType: local | one | quorum | all + ## - local: only lock the session locally on the current node + ## - one: select only one remove node to lock the session + ## - quorum: select some nodes to lock the session + ## - all: lock the session on all of the nodes in the cluster + ## Default: quorum + session_locking_strategy: quorum -## Session locking strategy in a cluster. -## -## Value: Enum -## - local -## - leader -## - quorum -## - all -broker.session_locking_strategy = quorum + ## Dispatch strategy for shared subscription + ## + ## @doc broker.shared_subscription_strategy + ## ValueType: random | round_robin | sticky | hash + ## - random: dispatch the message to a random selected subscriber + ## - round_robin: select the subscribers in a round-robin manner + ## - sticky: always use the last selected subscriber to dispatch, + ## until the susbcriber disconnected. + ## - hash: select the subscribers by the hash of clientIds + ## Default: round_robin + shared_subscription_strategy: round_robin -## Dispatch strategy for shared subscription -## -## Value: Enum -## - random -## - round_robin -## - sticky -## - hash # same as hash_clientid -## - hash_clientid -## - hash_topic -broker.shared_subscription_strategy = random + ## Enable/disable shared dispatch acknowledgement for QoS1 and QoS2 messages + ## This should allow messages to be dispatched to a different subscriber in + ## the group in case the picked (based on shared_subscription_strategy) one # is offline + ## + ## @doc broker.shared_dispatch_ack_enabled + ## ValueType: Boolean + ## Default: false + shared_dispatch_ack_enabled: false -## Enable/disable shared dispatch acknowledgement for QoS1 and QoS2 messages -## This should allow messages to be dispatched to a different subscriber in -## the group in case the picked (based on shared_subscription_strategy) one # is offline -## -## Value: Enum -## - true -## - false -broker.shared_dispatch_ack_enabled = false + ## Enable batch clean for deleted routes. + ## + ## @doc broker.route_batch_clean + ## ValueType: Boolean + ## Default: true + route_batch_clean: true -## Enable batch clean for deleted routes. -## -## Value: Flag -broker.route_batch_clean = off + ## Performance toggle for subscribe/unsubscribe wildcard topic. + ## Change this toggle only when there are many wildcard topics. + ## + ## NOTE: when changing from/to 'global' lock, it requires all + ## nodes in the cluster to be stopped before the change. + ## + ## @doc broker.perf.route_lock_type + ## ValueType: key | tab | global + ## - key: mnesia translational updates with per-key locks. recommended for single node setup. + ## - tab: mnesia translational updates with table lock. recommended for multi-nodes setup. + ## - global: global lock protected updates. recommended for larger cluster. + ## Default: key + perf.route_lock_type: key -## Performance toggle for subscribe/unsubscribe wildcard topic. -## Change this toggle only when there are many wildcard topics. -## Value: Enum -## - key: mnesia translational updates with per-key locks. recommended for single node setup. -## - tab: mnesia translational updates with table lock. recommended for multi-nodes setup. -## - global: global lock protected updates. recommended for larger cluster. -## NOTE: when changing from/to 'global' lock, it requires all nodes in the cluster -## to be stopped before the change. -# broker.perf.route_lock_type = key + ## Enable trie path compaction. + ## Enabling it significantly improves wildcard topic subscribe + ## rate, if wildcard topics have unique prefixes like: + ## 'sensor/{{id}}/+/', where ID is unique per subscriber. + ## + ## Topic match performance (when publishing) may degrade if messages + ## are mostly published to topics with large number of levels. + ## + ## NOTE: This is a cluster-wide configuration. + ## It rquires all nodes to be stopped before changing it. + ## + ## @doc broker.perf.trie_compaction + ## ValueType: Boolean + ## Default: true + perf.trie_compaction: true +} -## Enable trie path compaction. -## Enabling it significantly improves wildcard topic subscribe rate, -## if wildcard topics have unique prefixes like: 'sensor/{{id}}/+/', -## where ID is unique per subscriber. +##================================================================== +## Zones and Listeners +##================================================================== +## A zone contains a set of configurations for listeners. ## -## Topic match performance (when publishing) may degrade if messages -## are mostly published to topics with large number of levels. +## The configurations defined in zone can be overridden by the ones +## defined in listeners with the same key. ## -## NOTE: This is a cluster-wide configuration. -## It rquires all nodes to be stopped before changing it. +## For example given the following config: +## ``` ## -## Value: Enum -## - true: enable trie path compaction -## - false: disable trie path compaction -# broker.perf.trie_compaction = true +## zone.x { +## a: {b:1, c: 1} +## listeners.y { +## a: {b: 2} +## } +## } +## ``` +## The config "a" in zone "x" is overridden by the configs inside +## the listener "y". So the value of config "a" in listener "y" +## is `a: {b:2, c: 1}`. +## +## All the configs that can be set in zones and be overridden in listenser are: +## - `auth.*` +## - `stats.*` +## - `mqtt.*` +## - `authorization.*` +## - `flapping_detect.*` +## - `force_shutdown.*` +## - `conn_congestion.*` +## +## Syntax: zones. {} +zones.default { + ## Enable authentication + ## + ## @doc zones..auth.enable + ## ValueType: Boolean + ## Default: false + auth.enable: false -## CONFIG_SECTION_BGN=sys_mon ================================================== + ## Enable per connection statistics. + ## + ## @doc zones..stats.enable + ## ValueType: Boolean + ## Default: true + stats.enable: true -## Enable Long GC monitoring. Disable if the value is 0. -## Notice: don't enable the monitor in production for: -## https://github.com/erlang/otp/blob/feb45017da36be78d4c5784d758ede619fa7bfd3/erts/emulator/beam/erl_gc.c#L421 -## -## Value: Duration -## - h: hour -## - m: minute -## - s: second -## - ms: milliseconds -## -## Examples: -## - 2h: 2 hours -## - 30m: 30 minutes -## - 0.1s: 0.1 seconds -## - 100ms : 100 milliseconds -## -## Default: 0ms -sysmon.long_gc = 0 + ## Maximum number of concurrent connections in this zone. + ## + ## This value must be larger than the sum of `max_connections` set + ## in the listeners under this zone. + ## + ## @doc zones..overall_max_connections + ## ValueType: Number | infinity + ## Default: infinity + overall_max_connections: infinity -## Enable Long Schedule(ms) monitoring. -## -## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 -## -## Value: Duration -## - h: hour -## - m: minute -## - s: second -## - ms: milliseconds -## -## Examples: -## - 2h: 2 hours -## - 30m: 30 minutes -## - 100ms: 100 milliseconds -## -## Default: 0ms -sysmon.long_schedule = 240ms + mqtt { + ## When publishing or subscribing, prefix all topics with a mountpoint string. + ## The prefixed string will be removed from the topic name when the message + ## is delivered to the subscriber. The mountpoint is a way that users can use + ## to implement isolation of message routing between different listeners. + ## + ## For example if a clientA subscribes to "t" with `zones..mqtt.mountpoint` + ## set to "some_tenant", then the client accually subscribes to the topic + ## "some_tenant/t". Similarly if another clientB (connected to the same listener + ## with the clientA) send a message to topic "t", the message is accually route + ## to all the clients subscribed "some_tenant/t", so clientA will receive the + ## message, with topic name "t". + ## + ## Set to "" to disable the feature. + ## + ## Variables in mountpoint string: + ## - %c: clientid + ## - %u: username + ## + ## @doc zones..listeners..mountpoint + ## ValueType: String + ## Default: "" + mountpoint: "" -## Enable Large Heap monitoring. -## -## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 -## -## Value: bytes -## -## Default: 8M words. 32MB on 32-bit VM, 64MB on 64-bit VM. -sysmon.large_heap = 8MB + ## How long time the MQTT connection will be disconnected if the + ## TCP connection is established but MQTT CONNECT has not been + ## received. + ## + ## @doc zones..mqtt.idle_timeout + ## ValueType: Duration + ## Default: 15s + idle_timeout: 15s -## Enable Busy Port monitoring. -## -## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 -## -## Value: true | false -sysmon.busy_port = false + ## Maximum MQTT packet size allowed. + ## + ## @doc zones..mqtt.max_packet_size + ## ValueType: Bytes + ## Default: 1MB + max_packet_size: 1MB -## Enable Busy Dist Port monitoring. -## -## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 -## -## Value: true | false -sysmon.busy_dist_port = true + ## Maximum length of MQTT clientId allowed. + ## + ## @doc zones..mqtt.max_clientid_len + ## ValueType: Integer + ## Range: [23, 65535] + ## Default: 65535 + max_clientid_len: 65535 -## The time interval for the periodic cpu check -## -## Value: Duration -## -h: hour, e.g. '2h' for 2 hours -## -m: minute, e.g. '5m' for 5 minutes -## -s: second, e.g. '30s' for 30 seconds -## -## Default: 60s -os_mon.cpu_check_interval = 60s + ## Maximum topic levels allowed. + ## + ## @doc zones..mqtt.max_topic_levels + ## ValueType: Integer + ## Range: [1, 65535] + ## Default: 65535 + max_topic_levels: 65535 -## The threshold, as percentage of system cpu, for how much system cpu can be used before the corresponding alarm is set. -## -## Default: 80% -os_mon.cpu_high_watermark = 80% + ## Maximum QoS allowed. + ## + ## @doc zones..mqtt.max_qos_allowed + ## ValueType: 0 | 1 | 2 + ## Default: 2 + max_qos_allowed: 2 -## The threshold, as percentage of system cpu, for how much system cpu can be used before the corresponding alarm is clear. -## -## Default: 60% -os_mon.cpu_low_watermark = 60% + ## Maximum Topic Alias, 0 means no topic alias supported. + ## + ## @doc zones..mqtt.max_topic_alias + ## ValueType: Integer + ## Range: [0, 65535] + ## Default: 65535 + max_topic_alias: 65535 -## The time interval for the periodic memory check -## -## Value: Duration -## -h: hour, e.g. '2h' for 2 hours -## -m: minute, e.g. '5m' for 5 minutes -## -s: second, e.g. '30s' for 30 seconds -## -## Default: 60s -os_mon.mem_check_interval = 60s + ## Whether the Server supports MQTT retained messages. + ## + ## @doc zones..mqtt.retain_available + ## ValueType: Boolean + ## Default: true + retain_available: true -## The threshold, as percentage of system memory, for how much system memory can be allocated before the corresponding alarm is set. -## -## Default: 70% -os_mon.sysmem_high_watermark = 70% + ## Whether the Server supports MQTT Wildcard Subscriptions + ## + ## @doc zones..mqtt.wildcard_subscription + ## ValueType: Boolean + ## Default: true + wildcard_subscription: true -## The threshold, as percentage of system memory, for how much system memory can be allocated by one Erlang process before the corresponding alarm is set. -## -## Default: 5% -os_mon.procmem_high_watermark = 5% + ## Whether the Server supports MQTT Shared Subscriptions. + ## + ## @doc zones..mqtt.shared_subscription + ## ValueType: Boolean + ## Default: true + shared_subscription: true -## The time interval for the periodic process limit check -## -## Value: Duration -## -## Default: 30s -vm_mon.check_interval = 30s + ## Whether to ignore loop delivery of messages.(for mqtt v3.1.1) + ## + ## @doc zones..mqtt.ignore_loop_deliver + ## ValueType: Boolean + ## Default: false + ignore_loop_deliver: false -## The threshold, as percentage of processes, for how many processes can simultaneously exist at the local node before the corresponding alarm is set. -## -## Default: 80% -vm_mon.process_high_watermark = 80% + ## Whether to parse the MQTT frame in strict mode + ## + ## @doc zones..mqtt.strict_mode + ## ValueType: Boolean + ## Default: false + strict_mode: false -## The threshold, as percentage of processes, for how many processes can simultaneously exist at the local node before the corresponding alarm is clear. -## -## Default: 60% -vm_mon.process_low_watermark = 60% + ## Specify the response information returned to the client + ## + ## This feature is disabled if is set to "" + ## + ## @doc zones..mqtt.response_information + ## ValueType: String + ## Default: "" + response_information: "" -## Specifies the actions to take when an alarm is activated -## -## Value: String -## - log -## - publish -## -## Default: "log,publish" -alarm.actions = "log,publish" + ## Server Keep Alive of MQTT 5.0 + ## + ## @doc zones..mqtt.server_keepalive + ## ValueType: Number | disabled + ## Default: disabled + server_keepalive: disabled -## The maximum number of deactivated alarms -## -## Value: Integer -## -## Default: 1000 -alarm.size_limit = 1000 + ## The backoff for MQTT keepalive timeout. The broker will kick a connection out + ## until 'Keepalive * backoff * 2' timeout. + ## + ## @doc zones..mqtt.keepalive_backoff + ## ValueType: Float + ## Range: (0.5, 1] + ## Default: 0.75 + keepalive_backoff: 0.75 -## Validity Period of deactivated alarms -## -## Value: Duration -## - h: hour -## - m: minute -## - s: second -## - ms: milliseconds -## -## Default: 24h -alarm.validity_period = 24h + ## Maximum number of subscriptions allowed. + ## + ## @doc zones..mqtt.max_subscriptions + ## ValueType: Integer | infinity + ## Range: [1, infinity) + ## Default: infinity + max_subscriptions: infinity -## CONFIG_SECTION_END=sys_mon ================================================== + ## Force to upgrade QoS according to subscription. + ## + ## @doc zones..mqtt.upgrade_qos + ## ValueType: Boolean + ## Default: false + upgrade_qos: false + + ## Maximum size of the Inflight Window storing QoS1/2 messages delivered but unacked. + ## + ## @doc zones..mqtt.max_inflight + ## ValueType: Integer + ## Range: [1, 65535] + ## Default: 32 + max_inflight: 32 + + ## Retry interval for QoS1/2 message delivering. + ## + ## @doc zones..mqtt.retry_interval + ## ValueType: Duration + ## Default: 30s + retry_interval: 30s + + ## Maximum QoS2 packets (Client -> Broker) awaiting PUBREL. + ## + ## @doc zones..mqtt.max_awaiting_rel + ## ValueType: Integer | infinity + ## Range: [1, infinity) + ## Default: 100 + max_awaiting_rel: 100 + + ## The QoS2 messages (Client -> Broker) will be dropped if awaiting PUBREL timeout. + ## + ## @doc zones..mqtt.await_rel_timeout + ## ValueType: Duration + ## Default: 300s + await_rel_timeout: 300s + + ## Default session expiry interval for MQTT V3.1.1 connections. + ## + ## @doc zones..mqtt.session_expiry_interval + ## ValueType: Duration + ## Default: 2h + session_expiry_interval: 2h + + ## Maximum queue length. Enqueued messages when persistent client disconnected, + ## or inflight window is full. + ## + ## @doc zones..mqtt.max_mqueue_len + ## ValueType: Integer | infinity + ## Range: [0, infinity) + ## Default: 1000 + max_mqueue_len: 1000 + + ## Topic priorities. + ## + ## There's no priority table by default, hence all messages + ## are treated equal. + ## + ## Priority number [1-255] + ## + ## NOTE: comma and equal signs are not allowed for priority topic names + ## NOTE: Messages for topics not in the priority table are treated as + ## either highest or lowest priority depending on the configured + ## value for mqtt.mqueue_default_priority + ## + ## @doc zones..mqtt.mqueue_priorities + ## ValueType: Map | disabled + ## Examples: + ## To configure "topic/1" > "topic/2": + ## mqueue_priorities: {"topic/1": 10, "topic/2": 8} + ## Default: disabled + mqueue_priorities: disabled + + ## Default to highest priority for topics not matching priority table + ## + ## @doc zones..mqtt.mqueue_default_priority + ## ValueType: highest | lowest + ## Default: lowest + mqueue_default_priority: lowest + + ## Whether to enqueue QoS0 messages. + ## + ## @doc zones..mqtt.mqueue_store_qos0 + ## ValueType: Boolean + ## Default: true + mqueue_store_qos0: true + + ## Whether use username replace client id + ## + ## @doc zones..mqtt.use_username_as_clientid + ## ValueType: Boolean + ## Default: false + use_username_as_clientid: false + + ## Use the CN, DN or CRT field from the client certificate as a username. + ## Only works for SSL connection. + ## + ## @doc zones..mqtt.peer_cert_as_username + ## ValueType: cn | dn | crt | disabled + ## Default: disabled + peer_cert_as_username: disabled + + ## Use the CN, DN or CRT field from the client certificate as a clientid. + ## Only works for SSL connection. + ## + ## @doc zones..mqtt.peer_cert_as_clientid + ## ValueType: cn | dn | crt | disabled + ## Default: disabled + peer_cert_as_clientid: disabled + + } + + authorization { + + ## Enable Authorization check. + ## + ## @doc zones..authorization.enable + ## ValueType: Boolean + ## Default: true + enable: true + + ## The action when authorization check reject current operation + ## + ## @doc zones..authorization.deny_action + ## ValueType: ignore | disconnect + ## Default: ignore + deny_action: ignore + + ## Whether to enable Authorization cache. + ## + ## If enabled, Authorization roles for each client will be cached in the memory + ## + ## @doc zones..authorization.cache.enable + ## ValueType: Boolean + ## Default: true + cache.enable: true + + ## The maximum count of Authorization entries can be cached for a client. + ## + ## @doc zones..authorization.cache.max_size + ## ValueType: Integer + ## Range: [0, 1048576] + ## Default: 32 + cache.max_size: 32 + + ## The time after which an Authorization cache entry will be deleted + ## + ## @doc zones..authorization.cache.ttl + ## ValueType: Duration + ## Default: 1m + cache.ttl: 1m + } + + flapping_detect { + ## Enable Flapping Detection. + ## + ## This config controls the allowed maximum number of CONNECT received + ## from the same clientid in a time frame defined by `window_time`. + ## After the limit is reached, successive CONNECT requests are forbidden + ## (banned) until the end of the time period defined by `ban_time`. + ## + ## @doc zones..flapping_detect.enable + ## ValueType: Boolean + ## Default: true + enable: false + + ## The max disconnect allowed of a MQTT Client in `window_time` + ## + ## @doc zones..flapping_detect.max_count + ## ValueType: Integer + ## Default: 15 + max_count: 15 + + ## The time window for flapping detect + ## + ## @doc zones..flapping_detect.window_time + ## ValueType: Duration + ## Default: 1m + window_time: 1m + + ## How long the clientid will be banned + ## + ## @doc zones..flapping_detect.ban_time + ## ValueType: Duration + ## Default: 5m + ban_time: 5m + + } + + force_shutdown: { + ## Enable force_shutdown + ## + ## @doc zones..force_shutdown.enable + ## ValueType: Boolean + ## Default: true + enable: true + + ## Max message queue length + ## @doc zones..force_shutdown.max_message_queue_len + ## ValueType: Integer + ## Range: (0, infinity) + ## Default: 1000 + max_message_queue_len: 1000 + + ## Total heap size + ## + ## @doc zones..force_shutdown.max_heap_size + ## ValueType: Size + ## Default: 32MB + max_heap_size: 32MB + } + + force_gc: { + ## Force the MQTT connection process GC after this number of + ## messages or bytes passed through. + ## + ## @doc zones..force_gc.enable + ## ValueType: Boolean + ## Default: true + enable: true + + ## GC the process after how many messages received + ## @doc zones..force_gc.max_message_queue_len + ## ValueType: Integer + ## Range: (0, infinity) + ## Default: 16000 + count: 16000 + + ## GC the process after how much bytes passed through + ## + ## @doc zones..force_gc.bytes + ## ValueType: Size + ## Default: 16MB + bytes: 16MB + } + + conn_congestion: { + ## Whether to alarm the congested connections. + ## + ## Sometimes the mqtt connection (usually an MQTT subscriber) may + ## get "congested" because there're too many packets to sent. + ## The socket trys to buffer the packets until the buffer is + ## full. If more packets comes after that, the packets will be + ## "pending" in a queue and we consider the connection is + ## "congested". + ## + ## Enable this to send an alarm when there's any bytes pending in + ## the queue. You could set the `sndbuf` to a larger value if the + ## alarm is triggered too often. + ## + ## The name of the alarm is of format "conn_congestion//". + ## Where the is the client-id of the congested MQTT connection. + ## And the is the username or "unknown_user" of not provided by the client. + ## + ## @doc zones..conn_congestion.enable_alarm + ## ValueType: Boolean + ## Default: true + enable_alarm: true + + ## Won't clear the congested alarm in how long time. + ## The alarm is cleared only when there're no pending bytes in + ## the queue, and also it has been `min_alarm_sustain_duration` + ## time since the last time we considered the connection is "congested". + ## + ## This is to avoid clearing and sending the alarm again too often. + ## + ## @doc zones..conn_congestion.min_alarm_sustain_duration + ## ValueType: Duration + ## Default: 1m + min_alarm_sustain_duration: 1m + } + + listeners.mqtt_tcp: + #${example_common_tcp_options} # common options can be written in a separate config entry and reference it from here. + { + + ## The type of the listener. + ## + ## @doc zones..listeners..type + ## ValueType: tcp | ws + ## - tcp: MQTT over TCP + ## - ws: MQTT over Websocket + ## - quic: MQTT over QUIC + ## Required: true + type: tcp + + ## The IP address and port that the listener will bind. + ## + ## @doc zones..listeners..bind + ## ValueType: IPAddress | Port | IPAddrPort + ## Required: true + ## Examples: 1883, 127.0.0.1:1883, ::1:1883 + bind: "0.0.0.0:1883" + + ## The size of the acceptor pool for this listener. + ## + ## @doc zones..listeners..acceptors + ## ValueType: Number + ## Default: 16 + acceptors: 16 + + ## Maximum number of concurrent connections. + ## + ## @doc zones..listeners..max_connections + ## ValueType: Number | infinity + ## Default: infinity + max_connections: 1024000 + + ## The access control rules for this listener. + ## + ## See: https://github.com/emqtt/esockd#allowdeny + ## + ## @doc zones..listeners..access_rules + ## ValueType: Array + ## Default: [] + ## Examples: + ## access_rules: [ + ## "deny 192.168.0.0/24", + ## "all all" + ## ] + access_rules: [ + "allow all" + ] + + ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed + ## behind HAProxy or Nginx. + ## + ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ + ## + ## @doc zones..listeners..proxy_protocol + ## ValueType: Boolean + ## Default: false + proxy_protocol: false + + ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection + ## if no proxy protocol packet recevied within the timeout. + ## + ## @doc zones..listeners..proxy_protocol_timeout + ## ValueType: Duration + ## Default: 3s + proxy_protocol_timeout: 3s + + rate_limit { + ## Maximum connections per second. + ## + ## @doc zones..max_conn_rate + ## ValueType: Number | infinity + ## Default: 1000 + ## Examples: + ## max_conn_rate: 1000 + max_conn_rate: 1000 + + ## Message limit for the a external MQTT connection. + ## + ## @doc zones..rate_limit.conn_messages_in + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 100 messages per 10 seconds. + ## conn_messages_in: "100,10s" + conn_messages_in: "100,10s" + + ## Limit the rate of receiving packets for a MQTT connection. + ## The rate is counted by bytes of packets per second. + ## + ## The connection won't accept more messages if the messages come + ## faster than the limit. + ## + ## @doc zones..rate_limit.conn_bytes_in + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 100KB incoming per 10 seconds. + ## conn_bytes_in: "100KB,10s" + ## + conn_bytes_in: "100KB,10s" + + ## Messages quota for the each of external MQTT connection. + ## This value consumed by the number of recipient on a message. + ## + ## @doc zones..rate_limit.quota.conn_messages_routing + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 100 messaegs per 1s: + ## quota.conn_messages_routing: "100,1s" + quota.conn_messages_routing: "100,1s" + + ## Messages quota for the all of external MQTT connections. + ## This value consumed by the number of recipient on a message. + ## + ## @doc zones..rate_limit.quota.overall_messages_routing + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 200000 messages per 1s: + ## quota.overall_messages_routing: "200000,1s" + ## + quota.overall_messages_routing: "200000,1s" + } + + ## TCP options + ## See ${example_common_tcp_options} for more information + tcp.backlog: 1024 + tcp.buffer: 4KB + } + + ## MQTT/SSL - SSL Listener for MQTT Protocol + listeners.mqtt_ssl: + #${example_common_tcp_options} ${example_common_ssl_options} # common options can be written in a separate config entry and reference it from here. + { + + ## The type of the listener. + ## + ## @doc zones..listeners..type + ## ValueType: tcp | ws + ## - tcp: MQTT over TCP + ## - ws: MQTT over Websocket + ## - quic: MQTT over QUIC + ## Required: true + type: tcp + + ## The IP address and port that the listener will bind. + ## + ## @doc zones..listeners..bind + ## ValueType: IPAddress | Port | IPAddrPort + ## Required: true + ## Examples: 8883, 127.0.0.1:8883, ::1:8883 + bind: "0.0.0.0:8883" + + ## The size of the acceptor pool for this listener. + ## + ## @doc zones..listeners..acceptors + ## ValueType: Number + ## Default: 16 + acceptors: 16 + + ## Maximum number of concurrent connections. + ## + ## @doc zones..listeners..max_connections + ## ValueType: Number | infinity + ## Default: infinity + max_connections: 512000 + + ## The access control rules for this listener. + ## + ## See: https://github.com/emqtt/esockd#allowdeny + ## + ## @doc zones..listeners..access_rules + ## ValueType: Array + ## Default: [] + ## Examples: + ## access_rules: [ + ## "deny 192.168.0.0/24", + ## "all all" + ## ] + access_rules: [ + "allow all" + ] + + ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed + ## behind HAProxy or Nginx. + ## + ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ + ## + ## @doc zones..listeners..proxy_protocol + ## ValueType: Boolean + ## Default: true + proxy_protocol: false + + ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection + ## if no proxy protocol packet recevied within the timeout. + ## + ## @doc zones..listeners..proxy_protocol_timeout + ## ValueType: Duration + ## Default: 3s + proxy_protocol_timeout: 3s + + rate_limit { + ## Maximum connections per second. + ## + ## @doc zones..max_conn_rate + ## ValueType: Number | infinity + ## Default: 1000 + ## Examples: + ## max_conn_rate: 1000 + max_conn_rate: 1000 + + ## Message limit for the a external MQTT connection. + ## + ## @doc zones..rate_limit.conn_messages_in + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 100 messages per 10 seconds. + ## conn_messages_in: "100,10s" + conn_messages_in: "100,10s" + + ## Limit the rate of receiving packets for a MQTT connection. + ## The rate is counted by bytes of packets per second. + ## + ## The connection won't accept more messages if the messages come + ## faster than the limit. + ## + ## @doc zones..rate_limit.conn_bytes_in + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 100KB incoming per 10 seconds. + ## conn_bytes_in: "100KB,10s" + ## + conn_bytes_in: "100KB,10s" + + ## Messages quota for the each of external MQTT connection. + ## This value consumed by the number of recipient on a message. + ## + ## @doc zones..rate_limit.quota.conn_messages_routing + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 100 messaegs per 1s: + ## quota.conn_messages_routing: "100,1s" + quota.conn_messages_routing: "100,1s" + + ## Messages quota for the all of external MQTT connections. + ## This value consumed by the number of recipient on a message. + ## + ## @doc zones..rate_limit.quota.overall_messages_routing + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 200000 messages per 1s: + ## quota.overall_messages_routing: "200000,1s" + ## + quota.overall_messages_routing: "200000,1s" + } + + ## SSL options + ## See ${example_common_ssl_options} for more information + ssl.enable: true + ssl.versions: ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] + ssl.keyfile: "{{ platform_etc_dir }}/certs/key.pem" + ssl.certfile: "{{ platform_etc_dir }}/certs/cert.pem" + ssl.cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem" + + ## TCP options + ## See ${example_common_tcp_options} for more information + tcp.backlog: 1024 + tcp.buffer: 4KB + } + + listeners.mqtt_quic: + { + ## The type of the listener. + ## + ## @doc zones..listeners..type + ## ValueType: tcp | ws + ## - tcp: MQTT over TCP + ## - ws: MQTT over Websocket + ## - quic: MQTT over QUIC + ## Required: true + type: quic + + ## The IP address and port that the listener will bind. + ## + ## @doc zones..listeners..bind + ## ValueType: IPAddress | Port | IPAddrPort + ## Required: true + ## Examples: 14567, 127.0.0.1:14567, ::1:14567 + bind: "0.0.0.0:14567" + + ## The size of the acceptor pool for this listener. + ## + ## @doc zones..listeners..acceptors + ## ValueType: Number + ## Default: 16 + acceptors: 16 + + ## Maximum number of concurrent connections. + ## + ## @doc zones..listeners..max_connections + ## ValueType: Number | infinity + ## Default: infinity + max_connections: 1024000 + + ## Path to the file containing the user's private PEM-encoded key. + ## + ## @doc zones..listeners..keyfile + ## ValueType: String + ## Default: "{{ platform_etc_dir }}/certs/key.pem" + keyfile: "{{ platform_etc_dir }}/certs/key.pem" + + ## Path to a file containing the user certificate. + ## + ## @doc zones..listeners..certfile + ## ValueType: String + ## Default: "{{ platform_etc_dir }}/certs/cert.pem" + certfile: "{{ platform_etc_dir }}/certs/cert.pem" + } + + listeners.mqtt_ws: + #${example_common_tcp_options} ${example_common_websocket_options} # common options can be written in a separate config entry and reference it from here. + { + + ## The type of the listener. + ## + ## @doc zones..listeners..type + ## ValueType: tcp | ws + ## - tcp: MQTT over TCP + ## - ws: MQTT over Websocket + ## - quic: MQTT over QUIC + ## Required: true + type: ws + + ## The IP address and port that the listener will bind. + ## + ## @doc zones..listeners..bind + ## ValueType: IPAddress | Port | IPAddrPort + ## Required: true + ## Examples: 8083, 127.0.0.1:8083, ::1:8083 + bind: "0.0.0.0:8083" + + ## The size of the acceptor pool for this listener. + ## + ## @doc zones..listeners..acceptors + ## ValueType: Number + ## Default: 16 + acceptors: 16 + + ## Maximum number of concurrent connections. + ## + ## @doc zones..listeners..max_connections + ## ValueType: Number | infinity + ## Default: infinity + max_connections: 1024000 + + ## The access control rules for this listener. + ## + ## See: https://github.com/emqtt/esockd#allowdeny + ## + ## @doc zones..listeners..access_rules + ## ValueType: Array + ## Default: [] + ## Examples: + ## access_rules: [ + ## "deny 192.168.0.0/24", + ## "all all" + ## ] + access_rules: [ + "allow all" + ] + + ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed + ## behind HAProxy or Nginx. + ## + ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ + ## + ## @doc zones..listeners..proxy_protocol + ## ValueType: Boolean + ## Default: true + proxy_protocol: false + + ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection + ## if no proxy protocol packet recevied within the timeout. + ## + ## @doc zones..listeners..proxy_protocol_timeout + ## ValueType: Duration + ## Default: 3s + proxy_protocol_timeout: 3s + + rate_limit { + ## Maximum connections per second. + ## + ## @doc zones..max_conn_rate + ## ValueType: Number | infinity + ## Default: 1000 + ## Examples: + ## max_conn_rate: 1000 + max_conn_rate: 1000 + + ## Message limit for the a external MQTT connection. + ## + ## @doc zones..rate_limit.conn_messages_in + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 100 messages per 10 seconds. + ## conn_messages_in: "100,10s" + conn_messages_in: "100,10s" + + ## Limit the rate of receiving packets for a MQTT connection. + ## The rate is counted by bytes of packets per second. + ## + ## The connection won't accept more messages if the messages come + ## faster than the limit. + ## + ## @doc zones..rate_limit.conn_bytes_in + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 100KB incoming per 10 seconds. + ## conn_bytes_in: "100KB,10s" + ## + conn_bytes_in: "100KB,10s" + + ## Messages quota for the each of external MQTT connection. + ## This value consumed by the number of recipient on a message. + ## + ## @doc zones..rate_limit.quota.conn_messages_routing + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 100 messaegs per 1s: + ## quota.conn_messages_routing: "100,1s" + quota.conn_messages_routing: "100,1s" + + ## Messages quota for the all of external MQTT connections. + ## This value consumed by the number of recipient on a message. + ## + ## @doc zones..rate_limit.quota.overall_messages_routing + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 200000 messages per 1s: + ## quota.overall_messages_routing: "200000,1s" + ## + quota.overall_messages_routing: "200000,1s" + } + + ## TCP options + ## See ${example_common_tcp_options} for more information + tcp.backlog: 1024 + tcp.buffer: 4KB + + ## Websocket options + ## See ${example_common_websocket_options} for more information + websocket.idle_timeout: 86400s + } + + listeners.mqtt_wss: + #${example_common_tcp_options} ${example_common_ssl_options} ${example_common_websocket_options} # common options can be written in a separate config entry and reference it from here. + { + + ## The type of the listener. + ## + ## @doc zones..listeners..type + ## ValueType: tcp | ws + ## - tcp: MQTT over TCP + ## - ws: MQTT over Websocket + ## - quic: MQTT over QUIC + ## Required: true + type: ws + + ## The IP address and port that the listener will bind. + ## + ## @doc zones..listeners..bind + ## ValueType: IPAddress | Port | IPAddrPort + ## Required: true + ## Examples: 8084, 127.0.0.1:8084, ::1:8084 + bind: "0.0.0.0:8084" + + ## The size of the acceptor pool for this listener. + ## + ## @doc zones..listeners..acceptors + ## ValueType: Number + ## Default: 16 + acceptors: 16 + + ## Maximum number of concurrent connections. + ## + ## @doc zones..listeners..max_connections + ## ValueType: Number | infinity + ## Default: infinity + max_connections: 512000 + + ## The access control rules for this listener. + ## + ## See: https://github.com/emqtt/esockd#allowdeny + ## + ## @doc zones..listeners..access_rules + ## ValueType: Array + ## Default: [] + ## Examples: + ## access_rules: [ + ## "deny 192.168.0.0/24", + ## "all all" + ## ] + access_rules: [ + "allow all" + ] + + ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed + ## behind HAProxy or Nginx. + ## + ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ + ## + ## @doc zones..listeners..proxy_protocol + ## ValueType: Boolean + ## Default: true + proxy_protocol: false + + ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection + ## if no proxy protocol packet recevied within the timeout. + ## + ## @doc zones..listeners..proxy_protocol_timeout + ## ValueType: Duration + ## Default: 3s + proxy_protocol_timeout: 3s + + rate_limit { + ## Maximum connections per second. + ## + ## @doc zones..max_conn_rate + ## ValueType: Number | infinity + ## Default: 1000 + ## Examples: + ## max_conn_rate: 1000 + max_conn_rate: 1000 + + ## Message limit for the a external MQTT connection. + ## + ## @doc zones..rate_limit.conn_messages_in + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 100 messages per 10 seconds. + ## conn_messages_in: "100,10s" + conn_messages_in: "100,10s" + + ## Limit the rate of receiving packets for a MQTT connection. + ## The rate is counted by bytes of packets per second. + ## + ## The connection won't accept more messages if the messages come + ## faster than the limit. + ## + ## @doc zones..rate_limit.conn_bytes_in + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 100KB incoming per 10 seconds. + ## conn_bytes_in: "100KB,10s" + ## + conn_bytes_in: "100KB,10s" + + ## Messages quota for the each of external MQTT connection. + ## This value consumed by the number of recipient on a message. + ## + ## @doc zones..rate_limit.quota.conn_messages_routing + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 100 messaegs per 1s: + ## quota.conn_messages_routing: "100,1s" + quota.conn_messages_routing: "100,1s" + + ## Messages quota for the all of external MQTT connections. + ## This value consumed by the number of recipient on a message. + ## + ## @doc zones..rate_limit.quota.overall_messages_routing + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 200000 messages per 1s: + ## quota.overall_messages_routing: "200000,1s" + ## + quota.overall_messages_routing: "200000,1s" + } + + ## SSL options + ## See ${example_common_ssl_options} for more information + ssl.enable: true + ssl.keyfile: "{{ platform_etc_dir }}/certs/key.pem" + ssl.certfile: "{{ platform_etc_dir }}/certs/cert.pem" + ssl.cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem" + + ## TCP options + ## See ${example_common_tcp_options} for more information + tcp.backlog: 1024 + tcp.buffer: 4KB + + ## Websocket options + ## See ${example_common_websocket_options} for more information + websocket.idle_timeout: 86400s + } + +} + +#This is an example zone which has less "strict" settings. +#It's useful to clients connecting the broker from trusted networks. +zones.internal { + authorization.enable: true + auth.enable: false + listeners.mqtt_internal: { + type: tcp + bind: "127.0.0.1:11883" + acceptors: 4 + max_connections: 1024000 + tcp.active_n: 1000 + tcp.backlog: 512 + } +} + +##================================================================== +## System Monitor +##================================================================== +sysmon { + ## The time interval for the periodic process limit check + ## + ## @doc sysmon.vm.process_check_interval + ## ValueType: Duration + ## Default: 30s + vm.process_check_interval: 30s + + ## The threshold, as percentage of processes, for how many processes can simultaneously exist at the local node before the corresponding alarm is set. + ## + ## @doc sysmon.vm.process_high_watermark + ## ValueType: Percentage + ## Default: 80% + vm.process_high_watermark: 80% + + ## The threshold, as percentage of processes, for how many processes can simultaneously exist at the local node before the corresponding alarm is clear. + ## + ## @doc sysmon.vm.process_low_watermark + ## ValueType: Percentage + ## Default: 60% + vm.process_low_watermark: 60% + + ## Enable Long GC monitoring. + ## Notice: don't enable the monitor in production for: + ## https://github.com/erlang/otp/blob/feb45017da36be78d4c5784d758ede619fa7bfd3/erts/emulator/beam/erl_gc.c#L421 + ## + ## @doc sysmon.vm.long_gc + ## ValueType: Duration | disabled + ## Default: disabled + vm.long_gc: disabled + + ## Enable Long Schedule(ms) monitoring. + ## + ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 + ## + ## @doc sysmon.vm.long_schedule + ## ValueType: Duration | disabled + ## Default: disabled + vm.long_schedule: 240ms + + ## Enable Large Heap monitoring. + ## + ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 + ## + ## @doc sysmon.vm.large_heap + ## ValueType: Size | disabled + ## Default: 32MB + vm.large_heap: 32MB + + ## Enable Busy Port monitoring. + ## + ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 + ## + ## @doc sysmon.vm.busy_port + ## ValueType: Boolean + ## Default: true + vm.busy_port: true + + ## Enable Busy Dist Port monitoring. + ## + ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 + ## + ## @doc sysmon.vm.busy_dist_port + ## ValueType: Boolean + ## Default: true + vm.busy_dist_port: true + + ## The time interval for the periodic cpu check + ## + ## @doc sysmon.os.cpu_check_interval + ## ValueType: Duration + ## Default: 60s + os.cpu_check_interval: 60s + + ## The threshold, as percentage of system cpu, for how much system cpu can be used before the corresponding alarm is set. + ## + ## @doc sysmon.os.cpu_high_watermark + ## ValueType: Percentage + ## Default: 80% + os.cpu_high_watermark: 80% + + ## The threshold, as percentage of system cpu, for how much system cpu can be used before the corresponding alarm is clear. + ## + ## @doc sysmon.os.cpu_low_watermark + ## ValueType: Percentage + ## Default: 60% + os.cpu_low_watermark: 60% + + ## The time interval for the periodic memory check + ## + ## @doc sysmon.os.mem_check_interval + ## ValueType: Duration | disabled + ## Default: 60s + os.mem_check_interval: 60s + + ## The threshold, as percentage of system memory, for how much system memory can be allocated before the corresponding alarm is set. + ## + ## @doc sysmon.os.sysmem_high_watermark + ## ValueType: Percentage + ## Default: 70% + os.sysmem_high_watermark: 70% + + ## The threshold, as percentage of system memory, for how much system memory can be allocated by one Erlang process before the corresponding alarm is set. + ## + ## @doc sysmon.os.procmem_high_watermark + ## ValueType: Percentage + ## Default: 5% + os.procmem_high_watermark: 5% +} + +##================================================================== +## Alarm +##================================================================== +alarm { + ## Specifies the actions to take when an alarm is activated + ## + ## @doc alarm.actions + ## ValueType: Array + ## Default: [log, publish] + actions: [log, publish] + + ## The maximum number of deactivated alarms + ## + ## @doc alarm.size_limit + ## ValueType: Integer + ## Default: 1000 + size_limit: 1000 + + ## Validity Period of deactivated alarms + ## + ## @doc alarm.validity_period + ## ValueType: Duration + ## Default: 24h + validity_period: 24h +} + +## Config references for listeners + +## Socket options for TCP connections +## See: http://erlang.org/doc/man/inet.html +example_common_tcp_options { + ## Specify the {active, N} option for this Socket. + ## + ## See: https://erlang.org/doc/man/inet.html#setopts-2 + ## + ## @doc listeners..tcp.active_n + ## ValueType: Number + ## Default: 100 + tcp.active_n: 100 + + ## TCP backlog defines the maximum length that the queue of + ## pending connections can grow to. + ## + ## @doc listeners..tcp.backlog + ## ValueType: Number + ## Range: [0, 1048576] + ## Default: 1024 + tcp.backlog: 1024 + + ## The TCP send timeout for the connections. + ## + ## @doc listeners..tcp.send_timeout + ## ValueType: Duration + ## Default: 15s + tcp.send_timeout: 15s + + ## Close the connection if send timeout. + ## + ## @doc listeners..tcp.send_timeout_close + ## ValueType: Boolean + ## Default: true + tcp.send_timeout_close: true + + ## The TCP receive buffer(os kernel) for the connections. + ## + ## @doc listeners..tcp.recbuf + ## ValueType: Size + ## Default: notset + #tcp.recbuf: 2KB + + ## The TCP send buffer(os kernel) for the connections. + ## + ## @doc listeners..tcp.sndbuf + ## ValueType: Size + ## Default: notset + #tcp.sndbuf: 4KB + + ## The size of the user-level software buffer used by the driver. + ## + ## @doc listeners..tcp.buffer + ## ValueType: Size + ## Default: notset + #tcp.buffer: 4KB + + ## The socket is set to a busy state when the amount of data queued internally + ## by the ERTS socket implementation reaches this limit. + ## + ## @doc listeners..tcp.high_watermark + ## ValueType: Size + ## Default: 1MB + tcp.high_watermark: 1MB + + ## The TCP_NODELAY flag for the connections. + ## + ## @doc listeners..tcp.nodelay + ## ValueType: Boolean + ## Default: false + tcp.nodelay: false + + ## The SO_REUSEADDR flag for the connections. + ## + ## @doc listeners..tcp.reuseaddr + ## ValueType: Boolean + ## Default: true + tcp.reuseaddr: true +} + +## Socket options for SSL connections +## See: http://erlang.org/doc/man/ssl.html +example_common_ssl_options { + + ## A performance optimization setting, it allows clients to reuse + ## pre-existing sessions, instead of initializing new ones. + ## Read more about it here. + ## + ## @doc listeners..ssl.reuse_sessions + ## ValueType: Boolean + ## Default: true + ssl.reuse_sessions: true + + ## SSL parameter renegotiation is a feature that allows a client and a server + ## to renegotiate the parameters of the SSL connection on the fly. + ## RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation, + ## you drop support for the insecure renegotiation, prone to MitM attacks. + ## + ## @doc listeners..ssl.secure_renegotiate + ## ValueType: Boolean + ## Default: true + ssl.secure_renegotiate: true + + ## An important security setting, it forces the cipher to be set based + ## on the server-specified order instead of the client-specified order, + ## hence enforcing the (usually more properly configured) security + ## ordering of the server administrator. + ## + ## @doc listeners..ssl.honor_cipher_order + ## ValueType: Boolean + ## Default: true + ssl.honor_cipher_order: true + + ## TLS versions only to protect from POODLE attack. + ## + ## @doc listeners..ssl.versions + ## ValueType: Array + ## Default: ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] + ssl.versions: ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] + + ## TLS Handshake timeout. + ## + ## @doc listeners..ssl.handshake_timeout + ## ValueType: Duration + ## Default: 15s + ssl.handshake_timeout: 15s + + ## Maximum number of non-self-issued intermediate certificates that + ## can follow the peer certificate in a valid certification path. + ## + ## @doc listeners..ssl.depth + ## ValueType: Integer + ## Default: 10 + ssl.depth: 10 + + ## Path to the file containing the user's private PEM-encoded key. + ## + ## @doc listeners..ssl.keyfile + ## ValueType: File + ## Default: "{{ platform_etc_dir }}/certs/key.pem" + ssl.keyfile: "{{ platform_etc_dir }}/certs/key.pem" + + ## Path to a file containing the user certificate. + ## + ## @doc listeners..ssl.certfile + ## ValueType: File + ## Default: "{{ platform_etc_dir }}/certs/cert.pem" + ssl.certfile: "{{ platform_etc_dir }}/certs/cert.pem" + + ## Path to the file containing PEM-encoded CA certificates. The CA certificates + ## are used during server authentication and when building the client certificate chain. + ## + ## @doc listeners..ssl.cacertfile + ## ValueType: File + ## Default: "{{ platform_etc_dir }}/certs/cacert.pem" + ssl.cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem" + + ## Maximum number of non-self-issued intermediate certificates that + ## can follow the peer certificate in a valid certification path. + ## + ## @doc listeners..ssl.depth + ## ValueType: Number + ## Default: 10 + ssl.depth: 10 + + ## String containing the user's password. Only used if the private keyfile + ## is password-protected. + ## + ## See: listener.ssl.$name.key_password + ## + ## @doc listeners..ssl.depth + ## ValueType: String + ## Default: "" + #ssl.key_password: "" + + ## The Ephemeral Diffie-Helman key exchange is a very effective way of + ## ensuring Forward Secrecy by exchanging a set of keys that never hit + ## the wire. Since the DH key is effectively signed by the private key, + ## it needs to be at least as strong as the private key. In addition, + ## the default DH groups that most of the OpenSSL installations have + ## are only a handful (since they are distributed with the OpenSSL + ## package that has been built for the operating system it’s running on) + ## and hence predictable (not to mention, 1024 bits only). + ## In order to escape this situation, first we need to generate a fresh, + ## strong DH group, store it in a file and then use the option above, + ## to force our SSL application to use the new DH group. Fortunately, + ## OpenSSL provides us with a tool to do that. Simply run: + ## openssl dhparam -out dh-params.pem 2048 + ## + ## @doc listeners..ssl.dhfile + ## ValueType: File + ## Default: "{{ platform_etc_dir }}/certs/dh-params.pem" + #ssl.dhfile: "{{ platform_etc_dir }}/certs/dh-params.pem" + + ## A server only does x509-path validation in mode verify_peer, + ## as it then sends a certificate request to the client (this + ## message is not sent if the verify option is verify_none). + ## You can then also want to specify option fail_if_no_peer_cert. + ## More information at: http://erlang.org/doc/man/ssl.html + ## + ## @doc listeners..ssl.verify + ## ValueType: verify_peer | verify_none + ## Default: verify_none + ssl.verify: verify_none + + ## Used together with {verify, verify_peer} by an SSL server. If set to true, + ## the server fails if the client does not have a certificate to send, that is, + ## sends an empty certificate. + ## + ## @doc listeners..ssl.fail_if_no_peer_cert + ## ValueType: Boolean + ## Default: true + ssl.fail_if_no_peer_cert: false + + ## This is the single most important configuration option of an Erlang SSL + ## application. Ciphers (and their ordering) define the way the client and + ## server encrypt information over the wire, from the initial Diffie-Helman + ## key exchange, the session key encryption ## algorithm and the message + ## digest algorithm. Selecting a good cipher suite is critical for the + ## application’s data security, confidentiality and performance. + ## + ## The cipher list above offers: + ## + ## A good balance between compatibility with older browsers. + ## It can get stricter for Machine-To-Machine scenarios. + ## Perfect Forward Secrecy. + ## No old/insecure encryption and HMAC algorithms + ## + ## Most of it was copied from Mozilla’s Server Side TLS article + ## + ## @doc listeners..ssl.ciphers + ## ValueType: Array + ## Default: [ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA,PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA] + ssl.ciphers: [ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA,PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA] + +} + +## Socket options for websocket connections +example_common_websocket_options { + ## The path of WebSocket MQTT endpoint + ## + ## @doc listeners..websocket.mqtt_path + ## ValueType: Path + ## Default: "/mqtt" + websocket.mqtt_path: "/mqtt" + + ## Whether a WebSocket message is allowed to contain multiple MQTT packets + ## + ## @doc listeners..websocket.mqtt_piggyback + ## ValueType: single | multiple + ## Default: multiple + websocket.mqtt_piggyback: multiple + + ## The compress flag for external WebSocket connections. + ## + ## If this Value is set true,the websocket message would be compressed + ## + ## @doc listeners..websocket.compress + ## ValueType: Boolean + ## Default: false + websocket.compress: false + + ## The idle timeout for external WebSocket connections. + ## + ## @doc listeners..websocket.idle_timeout + ## ValueType: Duration | infinity + ## Default: infinity + websocket.idle_timeout: infinity + + ## The max frame size for external WebSocket connections. + ## + ## @doc listeners..websocket.max_frame_size + ## ValueType: Size + ## Default: infinity + websocket.max_frame_size: infinity + + ## If set to true, the server fails if the client does not + ## have a Sec-WebSocket-Protocol to send. + ## Set to false for WeChat MiniApp. + ## + ## @doc listeners..websocket.fail_if_no_subprotocol + ## ValueType: Boolean + ## Default: true + websocket.fail_if_no_subprotocol: true + + ## Supported subprotocols + ## + ## @doc listeners..websocket.supported_subprotocols + ## ValueType: String + ## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 + websocket.supported_subprotocols: "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" + + ## Enable origin check in header for websocket connection + ## + ## @doc listeners..websocket.check_origin_enable + ## ValueType: Boolean + ## Default: false + websocket.check_origin_enable: false + + ## Allow origin to be absent in header in websocket connection + ## when check_origin_enable is true + ## + ## @doc listeners..websocket.allow_origin_absence + ## ValueType: Boolean + ## Default: true + websocket.allow_origin_absence: true + + ## Comma separated list of allowed origin in header for websocket connection + ## + ## @doc listeners..websocket.check_origins + ## ValueType: String + ## Examples: + ## local http dashboard url + ## check_origins: "http://localhost:18083, http://127.0.0.1:18083" + ## Default: "" + websocket.check_origins: "http://localhost:18083, http://127.0.0.1:18083" + + ## Specify which HTTP header for real source IP if the EMQ X cluster is + ## deployed behind NGINX or HAProxy. + ## + ## @doc listeners..websocket.proxy_address_header + ## ValueType: String + ## Default: X-Forwarded-For + websocket.proxy_address_header: X-Forwarded-For + + ## Specify which HTTP header for real source port if the EMQ X cluster is + ## deployed behind NGINX or HAProxy. + ## + ## @doc listeners..websocket.proxy_port_header + ## ValueType: String + ## Default: X-Forwarded-Port + websocket.proxy_port_header: X-Forwarded-Port + + websocket.deflate_opts { + ## The level of deflate options for external WebSocket connections. + ## + ## @doc listeners..websocket.deflate_opts.level + ## ValueType: none | default | best_compression | best_speed + ## Default: default + level: default + + ## The mem_level of deflate options for external WebSocket connections. + ## + ## @doc listeners..websocket.deflate_opts.mem_level + ## ValueType: Integer + ## Range: [1,9] + ## Default: 8 + mem_level: 8 + + ## The strategy of deflate options for external WebSocket connections. + ## + ## @doc listeners..websocket.deflate_opts.strategy + ## ValueType: default | filtered | huffman_only | rle + ## Default: default + strategy: default + + ## The deflate option for external WebSocket connections. + ## + ## @doc listeners..websocket.deflate_opts.server_context_takeover + ## ValueType: takeover | no_takeover + ## Default: takeover + server_context_takeover: takeover + + ## The deflate option for external WebSocket connections. + ## + ## @doc listeners..websocket.deflate_opts.client_context_takeover + ## ValueType: takeover | no_takeover + ## Default: takeover + client_context_takeover: takeover + + ## The deflate options for external WebSocket connections. + ## + ## + ## @doc listeners..websocket.deflate_opts.server_max_window_bits + ## ValueType: Integer + ## Range: [8,15] + ## Default: 15 + server_max_window_bits: 15 + + ## The deflate options for external WebSocket connections. + ## + ## @doc listeners..websocket.deflate_opts.client_max_window_bits + ## ValueType: Integer + ## Range: [8,15] + ## Default: 15 + client_max_window_bits: 15 + } +} diff --git a/apps/emqx/include/emqx.hrl b/apps/emqx/include/emqx.hrl index ba72a47b5..60dccd9a3 100644 --- a/apps/emqx/include/emqx.hrl +++ b/apps/emqx/include/emqx.hrl @@ -23,6 +23,10 @@ -define(Otherwise, true). +-define(COMMON_SHARD, emqx_common_shard). +-define(SHARED_SUB_SHARD, emqx_shared_sub_shard). +-define(MOD_DELAYED_SHARD, emqx_delayed_shard). + %%-------------------------------------------------------------------- %% Banner %%-------------------------------------------------------------------- @@ -31,12 +35,6 @@ -define(ERTS_MINIMUM_REQUIRED, "10.0"). -%%-------------------------------------------------------------------- -%% Configs -%%-------------------------------------------------------------------- - --define(NO_PRIORITY_TABLE, none). - %%-------------------------------------------------------------------- %% Topics' prefix: $SYS | $queue | $share %%-------------------------------------------------------------------- @@ -86,6 +84,9 @@ -define(ROUTE_SHARD, route_shard). + +-define(RULE_ENGINE_SHARD, emqx_rule_engine_shard). + -record(route, { topic :: binary(), dest :: node() | {binary(), node()} @@ -101,8 +102,7 @@ descr :: string(), vendor :: string() | undefined, active = false :: boolean(), - info = #{} :: map(), - type :: atom() + info = #{} :: map() }). %%-------------------------------------------------------------------- @@ -134,4 +134,3 @@ }). -endif. - diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index e69b07558..61444224c 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "5.0-pre"}). +-define(EMQX_RELEASE, {opensource, "5.0-alpha.3"}). -else. diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 22f31a345..55bad7471 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -12,12 +12,11 @@ [ {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} - , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.2"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.4"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v4.0.1"}}} %% todo delete when plugins use hocon - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.9.0"}}} - , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.10.5"}}} + , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} ]}. @@ -30,7 +29,7 @@ [ meck , {bbmustache,"1.10.0"} , {emqx_ct_helpers, {git,"https://github.com/emqx/emqx-ct-helpers", {branch,"hocon"}}} - , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3.1"}}} + , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.2"}}} ]}, {extra_src_dirs, [{"test",[recursive]}]} ]} diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index eae18f106..9d4e443ed 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -1,11 +1,30 @@ -Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {branch, "0.6.0"}}}, -AddBcrypt = fun(C) -> - {deps, Deps0} = lists:keyfind(deps, 1, C), - Deps = [Bcrypt | Deps0], - lists:keystore(deps, 1, C, {deps, Deps}) -end, +IsCentos6 = fun() -> + case file:read_file("/etc/centos-release") of + {ok, <<"CentOS release 6", _/binary >>} -> + true; + _ -> + false + end + end, -case os:type() of - {win32, _} -> CONFIG; - _ -> AddBcrypt(CONFIG) -end. +IsWin32 = fun() -> + win32 =:= element(1, os:type()) + end, + +IsQuicSupp = fun() -> + not (IsCentos6() orelse IsWin32() orelse + false =/= os:getenv("BUILD_WITHOUT_QUIC") + ) + end, + +Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {branch, "0.6.0"}}}, +Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {branch, "main"}}}, + +ExtraDeps = fun(C) -> + {deps, Deps0} = lists:keyfind(deps, 1, C), + Deps = Deps0 ++ [Bcrypt || not IsWin32()] ++ + [ Quicer || IsQuicSupp()], + lists:keystore(deps, 1, C, {deps, Deps}) + end, + +ExtraDeps(CONFIG). diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index e909702ae..546b70f14 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -4,7 +4,7 @@ {vsn, "5.0.0"}, % strict semver, bump manually! {modules, []}, {registered, []}, - {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon]}, + {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon,jiffy]}, {mod, {emqx_app,[]}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx/src/emqx.appup.src b/apps/emqx/src/emqx.appup.src deleted file mode 100644 index 4f9f00673..000000000 --- a/apps/emqx/src/emqx.appup.src +++ /dev/null @@ -1,111 +0,0 @@ -%% -*- mode: erlang -*- -{VSN, - [ - {"4.3.4", - [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]} - ]}, - {"4.3.3", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]} - ]}, - {"4.3.2", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]} - ]}, - {"4.3.1", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, - {load_module,emqx_node_dump,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_trie,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_node_dump,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,upgrade_retained_delayed_counter_type,[]}}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}], - [ - {"4.3.4", - [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]} - ]}, - {"4.3.3", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]} - ]}, - {"4.3.2", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]} - ]}, - {"4.3.1", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, - {load_module,emqx_node_dump,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_trie,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_node_dump,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}]}. diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl index 9c42e96a3..3530f0dfb 100644 --- a/apps/emqx/src/emqx.erl +++ b/apps/emqx/src/emqx.erl @@ -29,10 +29,6 @@ , stop/0 ]). --export([ get_env/1 - , get_env/2 - ]). - %% PubSub API -export([ subscribe/1 , subscribe/2 @@ -126,15 +122,6 @@ is_running(Node) -> Pid when is_pid(Pid) -> true end. -%% @doc Get environment --spec(get_env(Key :: atom()) -> maybe(term())). -get_env(Key) -> - get_env(Key, undefined). - --spec(get_env(Key :: atom(), Default :: term()) -> term()). -get_env(Key, Default) -> - application:get_env(?APP, Key, Default). - %%-------------------------------------------------------------------- %% PubSub API %%-------------------------------------------------------------------- @@ -227,7 +214,6 @@ shutdown() -> shutdown(Reason) -> ?LOG(critical, "emqx shutdown for ~s", [Reason]), _ = emqx_alarm_handler:unload(), - _ = emqx_plugins:unload(), lists:foreach(fun application:stop/1 , lists:reverse(default_started_applications()) ). @@ -235,13 +221,8 @@ shutdown(Reason) -> reboot() -> lists:foreach(fun application:start/1 , default_started_applications()). --ifdef(EMQX_ENTERPRISE). default_started_applications() -> - [gproc, esockd, ranch, cowboy, ekka, emqx]. --else. -default_started_applications() -> - [gproc, esockd, ranch, cowboy, ekka, emqx, emqx_modules]. --endif. + [gproc, esockd, ranch, cowboy, ekka, quicer, emqx] ++ emqx_feature(). %%-------------------------------------------------------------------- %% Internal functions @@ -253,3 +234,9 @@ reload_config(ConfFile) -> [application:set_env(App, Par, Val) || {Par, Val} <- Vals] end, Conf). +-ifndef(EMQX_DEP_APPS). +emqx_feature() -> []. +-else. +emqx_feature() -> + ?EMQX_DEP_APPS. +-endif. diff --git a/apps/emqx/src/emqx_access_control.erl b/apps/emqx/src/emqx_access_control.erl index 1ef885ed5..65991d222 100644 --- a/apps/emqx/src/emqx_access_control.erl +++ b/apps/emqx/src/emqx_access_control.erl @@ -18,66 +18,43 @@ -include("emqx.hrl"). --export([authenticate/1]). - --export([ check_acl/3 +-export([ authenticate/1 + , authorize/3 ]). --type(result() :: #{auth_result := emqx_types:auth_result(), - anonymous := boolean() - }). - %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- --spec(authenticate(emqx_types:clientinfo()) -> {ok, result()} | {error, term()}). -authenticate(ClientInfo = #{zone := Zone}) -> - AuthResult = default_auth_result(Zone), - case emqx_zone:get_env(Zone, bypass_auth_plugins, false) of - true -> - return_auth_result(AuthResult); - false -> - return_auth_result(run_hooks('client.authenticate', [ClientInfo], AuthResult)) +-spec(authenticate(emqx_types:clientinfo()) -> + ok | {ok, binary()} | {continue, map()} | {continue, binary(), map()} | {error, term()}). +authenticate(Credential) -> + run_hooks('client.authenticate', [Credential], ok). + +%% @doc Check Authorization +-spec authorize(emqx_types:clientinfo(), emqx_types:pubsub(), emqx_types:topic()) + -> allow | deny. +authorize(ClientInfo = #{zone := Zone}, PubSub, Topic) -> + case emqx_authz_cache:is_enabled(Zone) of + true -> check_authorization_cache(ClientInfo, PubSub, Topic); + false -> do_authorize(ClientInfo, PubSub, Topic) end. -%% @doc Check ACL --spec(check_acl(emqx_types:clientinfo(), emqx_types:pubsub(), emqx_types:topic()) - -> allow | deny). -check_acl(ClientInfo, PubSub, Topic) -> - case emqx_acl_cache:is_enabled() of - true -> check_acl_cache(ClientInfo, PubSub, Topic); - false -> do_check_acl(ClientInfo, PubSub, Topic) - end. - -check_acl_cache(ClientInfo, PubSub, Topic) -> - case emqx_acl_cache:get_acl_cache(PubSub, Topic) of +check_authorization_cache(ClientInfo = #{zone := Zone}, PubSub, Topic) -> + case emqx_authz_cache:get_authz_cache(Zone, PubSub, Topic) of not_found -> - AclResult = do_check_acl(ClientInfo, PubSub, Topic), - emqx_acl_cache:put_acl_cache(PubSub, Topic, AclResult), - AclResult; - AclResult -> AclResult + AuthzResult = do_authorize(ClientInfo, PubSub, Topic), + emqx_authz_cache:put_authz_cache(Zone, PubSub, Topic, AuthzResult), + AuthzResult; + AuthzResult -> AuthzResult end. -do_check_acl(ClientInfo = #{zone := Zone}, PubSub, Topic) -> - Default = emqx_zone:get_env(Zone, acl_nomatch, deny), - case run_hooks('client.check_acl', [ClientInfo, PubSub, Topic], Default) of +do_authorize(ClientInfo, PubSub, Topic) -> + case run_hooks('client.authorize', [ClientInfo, PubSub, Topic], allow) of allow -> allow; _Other -> deny end. -default_auth_result(Zone) -> - case emqx_zone:get_env(Zone, allow_anonymous, false) of - true -> #{auth_result => success, anonymous => true}; - false -> #{auth_result => not_authorized, anonymous => false} - end. - -compile({inline, [run_hooks/3]}). run_hooks(Name, Args, Acc) -> ok = emqx_metrics:inc(Name), emqx_hooks:run_fold(Name, Args, Acc). - --compile({inline, [return_auth_result/1]}). -return_auth_result(Result = #{auth_result := success}) -> - {ok, Result}; -return_auth_result(Result) -> - {error, maps:get(auth_result, Result, unknown_error)}. diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl index 62ce1af8b..223d5aa50 100644 --- a/apps/emqx/src/emqx_alarm.erl +++ b/apps/emqx/src/emqx_alarm.erl @@ -17,6 +17,7 @@ -module(emqx_alarm). -behaviour(gen_server). +-behaviour(emqx_config_handler). -include("emqx.hrl"). -include("logger.hrl"). @@ -29,10 +30,14 @@ -boot_mnesia({mnesia, [boot]}). -copy_mnesia({mnesia, [copy]}). --export([ start_link/1 +-export([pre_config_update/2]). + +-export([ start_link/0 , stop/0 ]). +-export([format/1]). + %% API -export([ activate/1 , activate/2 @@ -75,21 +80,16 @@ }). -record(state, { - actions :: [action()], - - size_limit :: non_neg_integer(), - - validity_period :: non_neg_integer(), - - timer = undefined :: undefined | reference() + timer :: reference() }). --type action() :: log | publish | event. - -define(ACTIVATED_ALARM, emqx_activated_alarm). -define(DEACTIVATED_ALARM, emqx_deactivated_alarm). +-rlog_shard({?COMMON_SHARD, ?ACTIVATED_ALARM}). +-rlog_shard({?COMMON_SHARD, ?DEACTIVATED_ALARM}). + -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). @@ -120,8 +120,8 @@ mnesia(copy) -> %% API %%-------------------------------------------------------------------- -start_link(Opts) -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [Opts], []). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). stop() -> gen_server:stop(?MODULE). @@ -153,27 +153,50 @@ get_alarms(activated) -> get_alarms(deactivated) -> gen_server:call(?MODULE, {get_alarms, deactivated}). +pre_config_update(#{<<"validity_period">> := Period0} = NewConf, OldConf) -> + ?MODULE ! {update_timer, hocon_postprocess:duration(Period0)}, + merge(OldConf, NewConf); +pre_config_update(NewConf, OldConf) -> + merge(OldConf, NewConf). + +merge(undefined, New) -> New; +merge(Old, New) -> maps:merge(Old, New). + +format(#activated_alarm{name = Name, message = Message, activate_at = At, details = Details}) -> + Now = erlang:system_time(microsecond), + #{ + node => node(), + name => Name, + message => Message, + duration => Now - At, + details => Details + }; +format(#deactivated_alarm{name = Name, message = Message, activate_at = At, details = Details, + deactivate_at = DAt}) -> + #{ + node => node(), + name => Name, + message => Message, + duration => DAt - At, + details => Details + }; +format(_) -> + {error, unknow_alarm}. + %%-------------------------------------------------------------------- %% gen_server callbacks %%-------------------------------------------------------------------- init([]) -> - Opts = [{actions, [log, publish]}], - init([Opts]); -init([Opts]) -> deactivate_all_alarms(), - Actions = proplists:get_value(actions, Opts), - SizeLimit = proplists:get_value(size_limit, Opts), - ValidityPeriod = timer:seconds(proplists:get_value(validity_period, Opts)), - {ok, ensure_delete_timer(#state{actions = Actions, - size_limit = SizeLimit, - validity_period = ValidityPeriod})}. + emqx_config_handler:add_handler([alarm], ?MODULE), + {ok, #state{timer = ensure_timer(undefined, get_validity_period())}}. %% suppress dialyzer warning due to dirty read/write race condition. %% TODO: change from dirty_read/write to transactional. %% TODO: handle mnesia write errors. -dialyzer([{nowarn_function, [handle_call/3]}]). -handle_call({activate_alarm, Name, Details}, _From, State = #state{actions = Actions}) -> +handle_call({activate_alarm, Name, Details}, _From, State) -> case mnesia:dirty_read(?ACTIVATED_ALARM, Name) of [#activated_alarm{name = Name}] -> {reply, {error, already_existed}, State}; @@ -182,18 +205,17 @@ handle_call({activate_alarm, Name, Details}, _From, State = #state{actions = Act details = Details, message = normalize_message(Name, Details), activate_at = erlang:system_time(microsecond)}, - mnesia:dirty_write(?ACTIVATED_ALARM, Alarm), - do_actions(activate, Alarm, Actions), + ekka_mnesia:dirty_write(?ACTIVATED_ALARM, Alarm), + do_actions(activate, Alarm, emqx_config:get([alarm, actions])), {reply, ok, State} end; -handle_call({deactivate_alarm, Name, Details}, _From, State = #state{ - actions = Actions, size_limit = SizeLimit}) -> +handle_call({deactivate_alarm, Name, Details}, _From, State) -> case mnesia:dirty_read(?ACTIVATED_ALARM, Name) of [] -> {reply, {error, not_found}, State}; [Alarm] -> - deactivate_alarm(Details, SizeLimit, Actions, Alarm), + deactivate_alarm(Details, Alarm), {reply, ok, State} end; @@ -202,9 +224,14 @@ handle_call(delete_all_deactivated_alarms, _From, State) -> {reply, ok, State}; handle_call({get_alarms, all}, _From, State) -> - Alarms = [normalize(Alarm) || - Alarm <- ets:tab2list(?ACTIVATED_ALARM) - ++ ets:tab2list(?DEACTIVATED_ALARM)], + {atomic, Alarms} = + ekka_mnesia:ro_transaction( + ?COMMON_SHARD, + fun() -> + [normalize(Alarm) || + Alarm <- ets:tab2list(?ACTIVATED_ALARM) + ++ ets:tab2list(?DEACTIVATED_ALARM)] + end), {reply, Alarms, State}; handle_call({get_alarms, activated}, _From, State) -> @@ -223,11 +250,15 @@ handle_cast(Msg, State) -> ?LOG(error, "Unexpected msg: ~p", [Msg]), {noreply, State}. -handle_info({timeout, TRef, delete_expired_deactivated_alarm}, - State = #state{timer = TRef, - validity_period = ValidityPeriod}) -> - delete_expired_deactivated_alarms(erlang:system_time(microsecond) - ValidityPeriod * 1000), - {noreply, ensure_delete_timer(State)}; +handle_info({timeout, _TRef, delete_expired_deactivated_alarm}, + #state{timer = TRef} = State) -> + Period = get_validity_period(), + delete_expired_deactivated_alarms(erlang:system_time(microsecond) - Period * 1000), + {noreply, State#state{timer = ensure_timer(TRef, Period)}}; + +handle_info({update_timer, Period}, #state{timer = TRef} = State) -> + ?LOG(warning, "update the 'validity_period' timer to ~p", [Period]), + {noreply, State#state{timer = ensure_timer(TRef, Period)}}; handle_info(Info, State) -> ?LOG(error, "Unexpected info: ~p", [Info]), @@ -243,16 +274,18 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%------------------------------------------------------------------------------ -deactivate_alarm(Details, SizeLimit, Actions, #activated_alarm{ - activate_at = ActivateAt, name = Name, details = Details0, - message = Msg0}) -> - case SizeLimit > 0 andalso - (mnesia:table_info(?DEACTIVATED_ALARM, size) >= SizeLimit) of +get_validity_period() -> + emqx_config:get([alarm, validity_period]). + +deactivate_alarm(Details, #activated_alarm{activate_at = ActivateAt, name = Name, + details = Details0, message = Msg0}) -> + SizeLimit = emqx_config:get([alarm, size_limit]), + case SizeLimit > 0 andalso (mnesia:table_info(?DEACTIVATED_ALARM, size) >= SizeLimit) of true -> case mnesia:dirty_first(?DEACTIVATED_ALARM) of '$end_of_table' -> ok; ActivateAt2 -> - mnesia:dirty_delete(?DEACTIVATED_ALARM, ActivateAt2) + ekka_mnesia:dirty_delete(?DEACTIVATED_ALARM, ActivateAt2) end; false -> ok end, @@ -261,9 +294,9 @@ deactivate_alarm(Details, SizeLimit, Actions, #activated_alarm{ DeActAlarm = make_deactivated_alarm(ActivateAt, Name, Details, normalize_message(Name, Details), erlang:system_time(microsecond)), - mnesia:dirty_write(?DEACTIVATED_ALARM, HistoryAlarm), - mnesia:dirty_delete(?ACTIVATED_ALARM, Name), - do_actions(deactivate, DeActAlarm, Actions). + ekka_mnesia:dirty_write(?DEACTIVATED_ALARM, HistoryAlarm), + ekka_mnesia:dirty_delete(?ACTIVATED_ALARM, Name), + do_actions(deactivate, DeActAlarm, emqx_config:get([alarm, actions])). make_deactivated_alarm(ActivateAt, Name, Details, Message, DeActivateAt) -> #deactivated_alarm{ @@ -279,7 +312,7 @@ deactivate_all_alarms() -> details = Details, message = Message, activate_at = ActivateAt}) -> - mnesia:dirty_write(?DEACTIVATED_ALARM, + ekka_mnesia:dirty_write(?DEACTIVATED_ALARM, #deactivated_alarm{ activate_at = ActivateAt, name = Name, @@ -291,7 +324,7 @@ deactivate_all_alarms() -> %% Delete all records from the given table, ignore result. clear_table(TableName) -> - case mnesia:clear_table(TableName) of + case ekka_mnesia:clear_table(TableName) of {aborted, Reason} -> ?LOG(warning, "Faile to clear table ~p reason: ~p", [TableName, Reason]); @@ -299,9 +332,12 @@ clear_table(TableName) -> ok end. -ensure_delete_timer(State = #state{validity_period = ValidityPeriod}) -> - TRef = emqx_misc:start_timer(ValidityPeriod, delete_expired_deactivated_alarm), - State#state{timer = TRef}. +ensure_timer(OldTRef, Period) -> + _ = case is_reference(OldTRef) of + true -> erlang:cancel_timer(OldTRef); + false -> ok + end, + emqx_misc:start_timer(Period, delete_expired_deactivated_alarm). delete_expired_deactivated_alarms(Checkpoint) -> delete_expired_deactivated_alarms(mnesia:dirty_first(?DEACTIVATED_ALARM), Checkpoint). @@ -311,7 +347,7 @@ delete_expired_deactivated_alarms('$end_of_table', _Checkpoint) -> delete_expired_deactivated_alarms(ActivatedAt, Checkpoint) -> case ActivatedAt =< Checkpoint of true -> - mnesia:dirty_delete(?DEACTIVATED_ALARM, ActivatedAt), + ekka_mnesia:dirty_delete(?DEACTIVATED_ALARM, ActivatedAt), NActivatedAt = mnesia:dirty_next(?DEACTIVATED_ALARM, ActivatedAt), delete_expired_deactivated_alarms(NActivatedAt, Checkpoint); false -> @@ -372,9 +408,9 @@ normalize_message(high_system_memory_usage, #{high_watermark := HighWatermark}) normalize_message(high_process_memory_usage, #{high_watermark := HighWatermark}) -> list_to_binary(io_lib:format("Process memory usage is higher than ~p%", [HighWatermark])); normalize_message(high_cpu_usage, #{usage := Usage}) -> - list_to_binary(io_lib:format("~p% cpu usage", [Usage])); + list_to_binary(io_lib:format("~s cpu usage", [Usage])); normalize_message(too_many_processes, #{usage := Usage}) -> - list_to_binary(io_lib:format("~p% process usage", [Usage])); + list_to_binary(io_lib:format("~s process usage", [Usage])); normalize_message(partition, #{occurred := Node}) -> list_to_binary(io_lib:format("Partition occurs at node ~s", [Node])); normalize_message(<<"resource", _/binary>>, #{type := Type, id := ID}) -> diff --git a/apps/emqx/src/emqx_alarm_handler.erl b/apps/emqx/src/emqx_alarm_handler.erl index a69913afd..2307b79db 100644 --- a/apps/emqx/src/emqx_alarm_handler.erl +++ b/apps/emqx/src/emqx_alarm_handler.erl @@ -56,20 +56,22 @@ init({_Args, {alarm_handler, _ExistingAlarms}}) -> init(_) -> {ok, []}. -handle_event({set_alarm, {system_memory_high_watermark, []}}, State) -> - emqx_alarm:activate(high_system_memory_usage, #{high_watermark => emqx_os_mon:get_sysmem_high_watermark()}), +handle_event({set_alarm, {system_memory_high_watermark, []}}, State) -> + emqx_alarm:activate(high_system_memory_usage, + #{high_watermark => emqx_os_mon:get_sysmem_high_watermark()}), {ok, State}; -handle_event({set_alarm, {process_memory_high_watermark, Pid}}, State) -> - emqx_alarm:activate(high_process_memory_usage, #{pid => Pid, - high_watermark => emqx_os_mon:get_procmem_high_watermark()}), +handle_event({set_alarm, {process_memory_high_watermark, Pid}}, State) -> + emqx_alarm:activate(high_process_memory_usage, + #{pid => list_to_binary(pid_to_list(Pid)), + high_watermark => emqx_os_mon:get_procmem_high_watermark()}), {ok, State}; -handle_event({clear_alarm, system_memory_high_watermark}, State) -> +handle_event({clear_alarm, system_memory_high_watermark}, State) -> emqx_alarm:deactivate(high_system_memory_usage), {ok, State}; -handle_event({clear_alarm, process_memory_high_watermark}, State) -> +handle_event({clear_alarm, process_memory_high_watermark}, State) -> emqx_alarm:deactivate(high_process_memory_usage), {ok, State}; diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl index 3041ca59d..f2ca61c15 100644 --- a/apps/emqx/src/emqx_app.erl +++ b/apps/emqx/src/emqx_app.erl @@ -19,16 +19,23 @@ -behaviour(application). -export([ start/2 + , prep_stop/1 , stop/1 , get_description/0 , get_release/0 + , set_init_config_load_done/0 ]). -include("emqx.hrl"). -define(APP, emqx). --define(EMQX_SHARDS, [route_shard]). +-define(EMQX_SHARDS, [ ?ROUTE_SHARD + , ?COMMON_SHARD + , ?SHARED_SUB_SHARD + , ?RULE_ENGINE_SHARD + , ?MOD_DELAYED_SHARD + ]). -include("emqx_release.hrl"). @@ -37,13 +44,16 @@ %%-------------------------------------------------------------------- start(_Type, _Args) -> - set_backtrace_depth(), + ok = maybe_load_config(), + ok = set_backtrace_depth(), print_otp_version_warning(), print_banner(), %% Load application first for ekka_mnesia scanner _ = load_ce_modules(), ekka:start(), ok = ekka_rlog:wait_for_shards(?EMQX_SHARDS, infinity), + false == os:getenv("EMQX_NO_QUIC") + andalso application:ensure_all_started(quicer), {ok, Sup} = emqx_sup:start_link(), ok = start_autocluster(), %% ok = emqx_plugins:init(), @@ -55,14 +65,31 @@ start(_Type, _Args) -> print_vsn(), {ok, Sup}. --spec(stop(State :: term()) -> term()). -stop(_State) -> +prep_stop(_State) -> ok = emqx_alarm_handler:unload(), emqx_boot:is_enabled(listeners) andalso emqx_listeners:stop(). +stop(_State) -> ok. + +%% @doc Call this function to make emqx boot without loading config, +%% in case we want to delegate the config load to a higher level app +%% which manages emqx app. +set_init_config_load_done() -> + application:set_env(emqx, init_config_load_done, true). + +maybe_load_config() -> + case application:get_env(emqx, init_config_load_done, false) of + true -> + ok; + false -> + %% the app env 'config_files' should be set before emqx get started. + ConfFiles = application:get_env(emqx, config_files, []), + emqx_config:init_load(emqx_schema, ConfFiles) + end. + set_backtrace_depth() -> - Depth = application:get_env(?APP, backtrace_depth, 16), + Depth = emqx_config:get([node, backtrace_depth]), _ = erlang:system_flag(backtrace_depth, Depth), ok. diff --git a/apps/emqx/src/emqx_acl_cache.erl b/apps/emqx/src/emqx_authz_cache.erl similarity index 55% rename from apps/emqx/src/emqx_acl_cache.erl rename to apps/emqx/src/emqx_authz_cache.erl index 4cbe6b06a..a13294da2 100644 --- a/apps/emqx/src/emqx_acl_cache.erl +++ b/apps/emqx/src/emqx_authz_cache.erl @@ -14,19 +14,19 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_acl_cache). +-module(emqx_authz_cache). -include("emqx.hrl"). --export([ list_acl_cache/0 - , get_acl_cache/2 - , put_acl_cache/3 - , cleanup_acl_cache/0 - , empty_acl_cache/0 - , dump_acl_cache/0 - , get_cache_max_size/0 - , get_cache_ttl/0 - , is_enabled/0 +-export([ list_authz_cache/1 + , get_authz_cache/3 + , put_authz_cache/4 + , cleanup_authz_cache/1 + , empty_authz_cache/0 + , dump_authz_cache/0 + , get_cache_max_size/1 + , get_cache_ttl/1 + , is_enabled/1 , drain_cache/0 ]). @@ -38,93 +38,95 @@ , get_oldest_key/0 ]). --type(acl_result() :: allow | deny). +-type(authz_result() :: allow | deny). -type(system_time() :: integer()). -type(cache_key() :: {emqx_types:pubsub(), emqx_types:topic()}). --type(cache_val() :: {acl_result(), system_time()}). +-type(cache_val() :: {authz_result(), system_time()}). --type(acl_cache_entry() :: {cache_key(), cache_val()}). +-type(authz_cache_entry() :: {cache_key(), cache_val()}). %% Wrappers for key and value cache_k(PubSub, Topic)-> {PubSub, Topic}. -cache_v(AclResult)-> {AclResult, time_now()}. +cache_v(AuthzResult)-> {AuthzResult, time_now()}. drain_k() -> {?MODULE, drain_timestamp}. --spec(is_enabled() -> boolean()). -is_enabled() -> - application:get_env(emqx, enable_acl_cache, true). +-spec(is_enabled(atom()) -> boolean()). +is_enabled(Zone) -> + emqx_config:get_zone_conf(Zone, [authorization, cache, enable]). --spec(get_cache_max_size() -> integer()). -get_cache_max_size() -> - application:get_env(emqx, acl_cache_max_size, 32). +-spec(get_cache_max_size(atom()) -> integer()). +get_cache_max_size(Zone) -> + emqx_config:get_zone_conf(Zone, [authorization, cache, max_size]). --spec(get_cache_ttl() -> integer()). -get_cache_ttl() -> - application:get_env(emqx, acl_cache_ttl, 60000). +-spec(get_cache_ttl(atom()) -> integer()). +get_cache_ttl(Zone) -> + emqx_config:get_zone_conf(Zone, [authorization, cache, ttl]). --spec(list_acl_cache() -> [acl_cache_entry()]). -list_acl_cache() -> - cleanup_acl_cache(), - map_acl_cache(fun(Cache) -> Cache end). +-spec(list_authz_cache(atom()) -> [authz_cache_entry()]). +list_authz_cache(Zone) -> + cleanup_authz_cache(Zone), + map_authz_cache(fun(Cache) -> Cache end). -%% We'll cleanup the cache before replacing an expired acl. --spec(get_acl_cache(emqx_types:pubsub(), emqx_topic:topic()) -> (acl_result() | not_found)). -get_acl_cache(PubSub, Topic) -> +%% We'll cleanup the cache before replacing an expired authz. +-spec get_authz_cache(atom(), emqx_types:pubsub(), emqx_topic:topic()) -> + authz_result() | not_found. +get_authz_cache(Zone, PubSub, Topic) -> case erlang:get(cache_k(PubSub, Topic)) of undefined -> not_found; - {AclResult, CachedAt} -> - if_expired(CachedAt, + {AuthzResult, CachedAt} -> + if_expired(get_cache_ttl(Zone), CachedAt, fun(false) -> - AclResult; + AuthzResult; (true) -> - cleanup_acl_cache(), + cleanup_authz_cache(Zone), not_found end) end. %% If the cache get full, and also the latest one %% is expired, then delete all the cache entries --spec(put_acl_cache(emqx_types:pubsub(), emqx_topic:topic(), acl_result()) -> ok). -put_acl_cache(PubSub, Topic, AclResult) -> - MaxSize = get_cache_max_size(), true = (MaxSize =/= 0), +-spec put_authz_cache(atom(), emqx_types:pubsub(), emqx_topic:topic(), authz_result()) + -> ok. +put_authz_cache(Zone, PubSub, Topic, AuthzResult) -> + MaxSize = get_cache_max_size(Zone), true = (MaxSize =/= 0), Size = get_cache_size(), case Size < MaxSize of true -> - add_acl(PubSub, Topic, AclResult); + add_authz(PubSub, Topic, AuthzResult); false -> NewestK = get_newest_key(), - {_AclResult, CachedAt} = erlang:get(NewestK), - if_expired(CachedAt, + {_AuthzResult, CachedAt} = erlang:get(NewestK), + if_expired(get_cache_ttl(Zone), CachedAt, fun(true) -> % all cache expired, cleanup first - empty_acl_cache(), - add_acl(PubSub, Topic, AclResult); + empty_authz_cache(), + add_authz(PubSub, Topic, AuthzResult); (false) -> % cache full, perform cache replacement - evict_acl_cache(), - add_acl(PubSub, Topic, AclResult) + evict_authz_cache(), + add_authz(PubSub, Topic, AuthzResult) end) end. -%% delete all the acl entries --spec(empty_acl_cache() -> ok). -empty_acl_cache() -> - foreach_acl_cache(fun({CacheK, _CacheV}) -> erlang:erase(CacheK) end), +%% delete all the authz entries +-spec(empty_authz_cache() -> ok). +empty_authz_cache() -> + foreach_authz_cache(fun({CacheK, _CacheV}) -> erlang:erase(CacheK) end), set_cache_size(0), keys_queue_set(queue:new()). -%% delete the oldest acl entry --spec(evict_acl_cache() -> ok). -evict_acl_cache() -> +%% delete the oldest authz entry +-spec(evict_authz_cache() -> ok). +evict_authz_cache() -> OldestK = keys_queue_out(), erlang:erase(OldestK), decr_cache_size(). %% cleanup all the expired cache entries --spec(cleanup_acl_cache() -> ok). -cleanup_acl_cache() -> +-spec(cleanup_authz_cache(atom()) -> ok). +cleanup_authz_cache(Zone) -> keys_queue_set( - cleanup_acl(keys_queue_get())). + cleanup_authz(get_cache_ttl(Zone), keys_queue_get())). get_oldest_key() -> keys_queue_pick(queue_front()). @@ -132,22 +134,22 @@ get_newest_key() -> keys_queue_pick(queue_rear()). get_cache_size() -> - case erlang:get(acl_cache_size) of + case erlang:get(authz_cache_size) of undefined -> 0; Size -> Size end. -dump_acl_cache() -> - map_acl_cache(fun(Cache) -> Cache end). +dump_authz_cache() -> + map_authz_cache(fun(Cache) -> Cache end). -map_acl_cache(Fun) -> - [Fun(R) || R = {{SubPub, _T}, _Acl} <- get(), SubPub =:= publish - orelse SubPub =:= subscribe]. -foreach_acl_cache(Fun) -> - _ = map_acl_cache(Fun), +map_authz_cache(Fun) -> + [Fun(R) || R = {{SubPub, _T}, _Authz} <- get(), SubPub =:= publish + orelse SubPub =:= subscribe]. +foreach_authz_cache(Fun) -> + _ = map_authz_cache(Fun), ok. -%% All acl cache entries added before `drain_cache()` invocation will become expired +%% All authz cache entries added before `drain_cache()` invocation will become expired drain_cache() -> _ = persistent_term:put(drain_k(), time_now()), ok. @@ -156,52 +158,52 @@ drain_cache() -> %% Internal functions %%-------------------------------------------------------------------- -add_acl(PubSub, Topic, AclResult) -> +add_authz(PubSub, Topic, AuthzResult) -> K = cache_k(PubSub, Topic), - V = cache_v(AclResult), + V = cache_v(AuthzResult), case erlang:get(K) of - undefined -> add_new_acl(K, V); - {_AclResult, _CachedAt} -> - update_acl(K, V) + undefined -> add_new_authz(K, V); + {_AuthzResult, _CachedAt} -> + update_authz(K, V) end. -add_new_acl(K, V) -> +add_new_authz(K, V) -> erlang:put(K, V), keys_queue_in(K), incr_cache_size(). -update_acl(K, V) -> +update_authz(K, V) -> erlang:put(K, V), keys_queue_update(K). -cleanup_acl(KeysQ) -> +cleanup_authz(TTL, KeysQ) -> case queue:out(KeysQ) of {{value, OldestK}, KeysQ2} -> - {_AclResult, CachedAt} = erlang:get(OldestK), - if_expired(CachedAt, + {_AuthzResult, CachedAt} = erlang:get(OldestK), + if_expired(TTL, CachedAt, fun(false) -> KeysQ; (true) -> erlang:erase(OldestK), decr_cache_size(), - cleanup_acl(KeysQ2) + cleanup_authz(TTL, KeysQ2) end); {empty, KeysQ} -> KeysQ end. incr_cache_size() -> - erlang:put(acl_cache_size, get_cache_size() + 1), ok. + erlang:put(authz_cache_size, get_cache_size() + 1), ok. decr_cache_size() -> Size = get_cache_size(), case Size > 1 of true -> - erlang:put(acl_cache_size, Size-1); + erlang:put(authz_cache_size, Size-1); false -> - erlang:put(acl_cache_size, 0) + erlang:put(authz_cache_size, 0) end, ok. set_cache_size(N) -> - erlang:put(acl_cache_size, N), ok. + erlang:put(authz_cache_size, N), ok. %%% Ordered Keys Q %%% keys_queue_in(Key) -> @@ -234,9 +236,9 @@ keys_queue_remove(Key, KeysQ) -> end, KeysQ). keys_queue_set(KeysQ) -> - erlang:put(acl_keys_q, KeysQ), ok. + erlang:put(authz_keys_q, KeysQ), ok. keys_queue_get() -> - case erlang:get(acl_keys_q) of + case erlang:get(authz_keys_q) of undefined -> queue:new(); KeysQ -> KeysQ end. @@ -246,8 +248,7 @@ queue_rear() -> fun queue:get_r/1. time_now() -> erlang:system_time(millisecond). -if_expired(CachedAt, Fun) -> - TTL = get_cache_ttl(), +if_expired(TTL, CachedAt, Fun) -> Now = time_now(), CurrentEvictTimestamp = persistent_term:get(drain_k(), 0), case CachedAt =< CurrentEvictTimestamp orelse (CachedAt + TTL) =< Now of diff --git a/apps/emqx/src/emqx_banned.erl b/apps/emqx/src/emqx_banned.erl index 762a2b61b..16804d329 100644 --- a/apps/emqx/src/emqx_banned.erl +++ b/apps/emqx/src/emqx_banned.erl @@ -51,6 +51,8 @@ -define(BANNED_TAB, ?MODULE). +-rlog_shard({?COMMON_SHARD, ?BANNED_TAB}). + %%-------------------------------------------------------------------- %% Mnesia bootstrap %%-------------------------------------------------------------------- @@ -96,19 +98,19 @@ create(#{who := Who, reason := Reason, at := At, until := Until}) -> - mnesia:dirty_write(?BANNED_TAB, #banned{who = Who, - by = By, - reason = Reason, - at = At, - until = Until}); + ekka_mnesia:dirty_write(?BANNED_TAB, #banned{who = Who, + by = By, + reason = Reason, + at = At, + until = Until}); create(Banned) when is_record(Banned, banned) -> - mnesia:dirty_write(?BANNED_TAB, Banned). + ekka_mnesia:dirty_write(?BANNED_TAB, Banned). -spec(delete({clientid, emqx_types:clientid()} | {username, emqx_types:username()} | {peerhost, emqx_types:peerhost()}) -> ok). delete(Who) -> - mnesia:dirty_delete(?BANNED_TAB, Who). + ekka_mnesia:dirty_delete(?BANNED_TAB, Who). info(InfoKey) -> mnesia:table_info(?BANNED_TAB, InfoKey). @@ -129,7 +131,7 @@ handle_cast(Msg, State) -> {noreply, State}. handle_info({timeout, TRef, expire}, State = #{expiry_timer := TRef}) -> - mnesia:async_dirty(fun expire_banned_items/1, [erlang:system_time(second)]), + ekka_mnesia:transaction(?COMMON_SHARD, fun expire_banned_items/1, [erlang:system_time(second)]), {noreply, ensure_expiry_timer(State), hibernate}; handle_info(Info, State) -> @@ -160,4 +162,3 @@ expire_banned_items(Now) -> mnesia:delete_object(?BANNED_TAB, B, sticky_write); (_, _Acc) -> ok end, ok, ?BANNED_TAB). - diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index d3ad128bb..eb8023d34 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -243,7 +243,7 @@ route(Routes, Delivery) -> do_route({To, Node}, Delivery) when Node =:= node() -> {Node, To, dispatch(To, Delivery)}; do_route({To, Node}, Delivery) when is_atom(Node) -> - {Node, To, forward(Node, To, Delivery, emqx:get_env(rpc_mode, async))}; + {Node, To, forward(Node, To, Delivery, emqx_config:get([rpc, mode]))}; do_route({To, Group}, Delivery) when is_tuple(Group); is_binary(Group) -> {share, To, emqx_shared_sub:dispatch(Group, To, Delivery)}. diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index e3cbff692..21e2a516c 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -31,6 +31,8 @@ -export([ info/1 , info/2 + , get_mqtt_conf/2 + , get_mqtt_conf/3 , set_conn_state/2 , get_session/1 , set_session/2 @@ -63,7 +65,7 @@ , maybe_apply/2 ]). --export_type([channel/0]). +-export_type([channel/0, opts/0]). -record(channel, { %% MQTT ConnInfo @@ -98,7 +100,9 @@ -type(channel() :: #channel{}). --type(conn_state() :: idle | connecting | connected | disconnected). +-type(opts() :: #{zone := atom(), listener := atom(), atom() => term()}). + +-type(conn_state() :: idle | connecting | connected | reauthenticating | disconnected). -type(reply() :: {outgoing, emqx_types:packet()} | {outgoing, [emqx_types:packet()]} @@ -151,7 +155,9 @@ info(connected_at, #channel{conninfo = ConnInfo}) -> info(clientinfo, #channel{clientinfo = ClientInfo}) -> ClientInfo; info(zone, #channel{clientinfo = ClientInfo}) -> - maps:get(zone, ClientInfo, undefined); + maps:get(zone, ClientInfo); +info(listener, #channel{clientinfo = ClientInfo}) -> + maps:get(listener, ClientInfo); info(clientid, #channel{clientinfo = ClientInfo}) -> maps:get(clientid, ClientInfo, undefined); info(username, #channel{clientinfo = ClientInfo}) -> @@ -195,17 +201,20 @@ caps(#channel{clientinfo = #{zone := Zone}}) -> %% Init the channel %%-------------------------------------------------------------------- --spec(init(emqx_types:conninfo(), proplists:proplist()) -> channel()). +-spec(init(emqx_types:conninfo(), opts()) -> channel()). init(ConnInfo = #{peername := {PeerHost, _Port}, - sockname := {_Host, SockPort}}, Options) -> - Zone = proplists:get_value(zone, Options), + sockname := {_Host, SockPort}}, #{zone := Zone, listener := Listener}) -> Peercert = maps:get(peercert, ConnInfo, undefined), Protocol = maps:get(protocol, ConnInfo, mqtt), - MountPoint = emqx_zone:mountpoint(Zone), - QuotaPolicy = emqx_zone:quota_policy(Zone), - ClientInfo = setting_peercert_infos( + MountPoint = case get_mqtt_conf(Zone, mountpoint) of + <<>> -> undefined; + MP -> MP + end, + QuotaPolicy = emqx_config:get_listener_conf(Zone, Listener,[rate_limit, quota], []), + ClientInfo = set_peercert_infos( Peercert, #{zone => Zone, + listener => Listener, protocol => Protocol, peerhost => PeerHost, sockport => SockPort, @@ -214,7 +223,7 @@ init(ConnInfo = #{peername := {PeerHost, _Port}, mountpoint => MountPoint, is_bridge => false, is_superuser => false - }, Options), + }, Zone, Listener), {NClientInfo, NConnInfo} = take_ws_cookie(ClientInfo, ConnInfo), #channel{conninfo = NConnInfo, clientinfo = NClientInfo, @@ -222,7 +231,7 @@ init(ConnInfo = #{peername := {PeerHost, _Port}, outbound => #{} }, auth_cache = #{}, - quota = emqx_limiter:init(Zone, QuotaPolicy), + quota = emqx_limiter:init(Zone, quota_policy(QuotaPolicy)), timers = #{}, conn_state = idle, takeover = false, @@ -230,30 +239,32 @@ init(ConnInfo = #{peername := {PeerHost, _Port}, pendings = [] }. -setting_peercert_infos(NoSSL, ClientInfo, _Options) +quota_policy(RawPolicy) -> + [{Name, {list_to_integer(StrCount), + erlang:trunc(hocon_postprocess:duration(StrWind) / 1000)}} + || {Name, [StrCount, StrWind]} <- maps:to_list(RawPolicy)]. + +set_peercert_infos(NoSSL, ClientInfo, _, _) when NoSSL =:= nossl; NoSSL =:= undefined -> ClientInfo#{username => undefined}; -setting_peercert_infos(Peercert, ClientInfo, Options) -> +set_peercert_infos(Peercert, ClientInfo, Zone, _Listener) -> {DN, CN} = {esockd_peercert:subject(Peercert), esockd_peercert:common_name(Peercert)}, - Username = peer_cert_as(peer_cert_as_username, Options, Peercert, DN, CN), - ClientId = peer_cert_as(peer_cert_as_clientid, Options, Peercert, DN, CN), - ClientInfo#{username => Username, clientid => ClientId, dn => DN, cn => CN}. - --dialyzer([{nowarn_function, [peer_cert_as/5]}]). -% esockd_peercert:peercert is opaque -% https://github.com/emqx/esockd/blob/master/src/esockd_peercert.erl -peer_cert_as(Key, Options, Peercert, DN, CN) -> - case proplists:get_value(Key, Options) of + PeercetAs = fun(Key) -> + case get_mqtt_conf(Zone, Key) of cn -> CN; dn -> DN; crt -> Peercert; - pem -> base64:encode(Peercert); - md5 -> emqx_passwd:hash(md5, Peercert); + pem when is_binary(Peercert) -> base64:encode(Peercert); + md5 when is_binary(Peercert) -> emqx_passwd:hash(md5, Peercert); _ -> undefined - end. + end + end, + Username = PeercetAs(peer_cert_as_username), + ClientId = PeercetAs(peer_cert_as_clientid), + ClientInfo#{username => Username, clientid => ClientId, dn => DN, cn => CN}. take_ws_cookie(ClientInfo, ConnInfo) -> case maps:take(ws_cookie, ConnInfo) of @@ -272,65 +283,77 @@ take_ws_cookie(ClientInfo, ConnInfo) -> | {ok, replies(), channel()} | {shutdown, Reason :: term(), channel()} | {shutdown, Reason :: term(), replies(), channel()}). -handle_in(?CONNECT_PACKET(), Channel = #channel{conn_state = connected}) -> +handle_in(?CONNECT_PACKET(), Channel = #channel{conn_state = ConnState}) + when ConnState =:= connected orelse ConnState =:= reauthenticating -> handle_out(disconnect, ?RC_PROTOCOL_ERROR, Channel); +handle_in(?CONNECT_PACKET(), Channel = #channel{conn_state = connecting}) -> + handle_out(connack, ?RC_PROTOCOL_ERROR, Channel); + handle_in(?CONNECT_PACKET(ConnPkt), Channel) -> case pipeline([fun enrich_conninfo/2, fun run_conn_hooks/2, fun check_connect/2, fun enrich_client/2, fun set_log_meta/2, - fun check_banned/2, - fun auth_connect/2 + fun check_banned/2 ], ConnPkt, Channel#channel{conn_state = connecting}) of {ok, NConnPkt, NChannel = #channel{clientinfo = ClientInfo}} -> NChannel1 = NChannel#channel{ will_msg = emqx_packet:will_msg(NConnPkt), alias_maximum = init_alias_maximum(NConnPkt, ClientInfo) }, - case enhanced_auth(?CONNECT_PACKET(NConnPkt), NChannel1) of + case authenticate(?CONNECT_PACKET(NConnPkt), NChannel1) of {ok, Properties, NChannel2} -> process_connect(Properties, ensure_connected(NChannel2)); {continue, Properties, NChannel2} -> handle_out(auth, {?RC_CONTINUE_AUTHENTICATION, Properties}, NChannel2); - {error, ReasonCode, NChannel2} -> - handle_out(connack, ReasonCode, NChannel2) + {error, ReasonCode} -> + handle_out(connack, ReasonCode, NChannel1) end; {error, ReasonCode, NChannel} -> handle_out(connack, ReasonCode, NChannel) end; -handle_in(Packet = ?AUTH_PACKET(?RC_CONTINUE_AUTHENTICATION, _Properties), +handle_in(Packet = ?AUTH_PACKET(ReasonCode, _Properties), Channel = #channel{conn_state = ConnState}) -> - case enhanced_auth(Packet, Channel) of - {ok, NProperties, NChannel} -> + try + case {ReasonCode, ConnState} of + {?RC_CONTINUE_AUTHENTICATION, connecting} -> ok; + {?RC_CONTINUE_AUTHENTICATION, reauthenticating} -> ok; + {?RC_RE_AUTHENTICATE, connected} -> ok; + _ -> error(protocol_error) + end, + case authenticate(Packet, Channel) of + {ok, NProperties, NChannel} -> + case ConnState of + connecting -> + process_connect(NProperties, ensure_connected(NChannel)); + _ -> + handle_out(auth, {?RC_SUCCESS, NProperties}, NChannel#channel{conn_state = connected}) + end; + {continue, NProperties, NChannel} -> + handle_out(auth, {?RC_CONTINUE_AUTHENTICATION, NProperties}, NChannel#channel{conn_state = reauthenticating}); + {error, NReasonCode} -> + case ConnState of + connecting -> + handle_out(connack, NReasonCode, Channel); + _ -> + handle_out(disconnect, NReasonCode, Channel) + end + end + catch + _Class:_Reason -> case ConnState of connecting -> - process_connect(NProperties, ensure_connected(NChannel)); - connected -> - handle_out(auth, {?RC_SUCCESS, NProperties}, NChannel); + handle_out(connack, ?RC_PROTOCOL_ERROR, Channel); _ -> handle_out(disconnect, ?RC_PROTOCOL_ERROR, Channel) - end; - {continue, NProperties, NChannel} -> - handle_out(auth, {?RC_CONTINUE_AUTHENTICATION, NProperties}, NChannel); - {error, NReasonCode, NChannel} -> - handle_out(connack, NReasonCode, NChannel) + end end; -handle_in(Packet = ?AUTH_PACKET(?RC_RE_AUTHENTICATE, _Properties), - Channel = #channel{conn_state = connected}) -> - case enhanced_auth(Packet, Channel) of - {ok, NProperties, NChannel} -> - handle_out(auth, {?RC_SUCCESS, NProperties}, NChannel); - {continue, NProperties, NChannel} -> - handle_out(auth, {?RC_CONTINUE_AUTHENTICATION, NProperties}, NChannel); - {error, NReasonCode, NChannel} -> - handle_out(disconnect, NReasonCode, NChannel) - end; - -handle_in(?PACKET(_), Channel = #channel{conn_state = ConnState}) when ConnState =/= connected -> +handle_in(?PACKET(_), Channel = #channel{conn_state = ConnState}) + when ConnState =/= connected andalso ConnState =/= reauthenticating -> handle_out(disconnect, ?RC_PROTOCOL_ERROR, Channel); handle_in(Packet = ?PUBLISH_PACKET(_QoS), Channel) -> @@ -408,11 +431,12 @@ handle_in(Packet = ?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters), ok -> TopicFilters0 = parse_topic_filters(TopicFilters), TopicFilters1 = put_subid_in_subopts(Properties, TopicFilters0), - TupleTopicFilters0 = check_sub_acls(TopicFilters1, Channel), - case emqx_zone:get_env(Zone, acl_deny_action, ignore) =:= disconnect andalso - lists:any(fun({_TopicFilter, ReasonCode}) -> - ReasonCode =:= ?RC_NOT_AUTHORIZED - end, TupleTopicFilters0) of + TupleTopicFilters0 = check_sub_authzs(TopicFilters1, Channel), + HasAuthzDeny = lists:any(fun({_TopicFilter, ReasonCode}) -> + ReasonCode =:= ?RC_NOT_AUTHORIZED + end, TupleTopicFilters0), + DenyAction = emqx_config:get_zone_conf(Zone, [authorization, deny_action]), + case DenyAction =:= disconnect andalso HasAuthzDeny of true -> handle_out(disconnect, ?RC_NOT_AUTHORIZED, Channel); false -> Replace = fun @@ -469,9 +493,11 @@ handle_in({frame_error, frame_too_large}, Channel = #channel{conn_state = connec handle_in({frame_error, Reason}, Channel = #channel{conn_state = connecting}) -> shutdown(Reason, ?CONNACK_PACKET(?RC_MALFORMED_PACKET), Channel); -handle_in({frame_error, frame_too_large}, Channel = #channel{conn_state = connected}) -> +handle_in({frame_error, frame_too_large}, Channel = #channel{conn_state = ConnState}) + when ConnState =:= connected orelse ConnState =:= reauthenticating -> handle_out(disconnect, {?RC_PACKET_TOO_LARGE, frame_too_large}, Channel); -handle_in({frame_error, Reason}, Channel = #channel{conn_state = connected}) -> +handle_in({frame_error, Reason}, Channel = #channel{conn_state = ConnState}) + when ConnState =:= connected orelse ConnState =:= reauthenticating -> handle_out(disconnect, {?RC_MALFORMED_PACKET, Reason}, Channel); handle_in({frame_error, Reason}, Channel = #channel{conn_state = disconnected}) -> @@ -516,7 +542,7 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), case pipeline([fun check_quota_exceeded/2, fun process_alias/2, fun check_pub_alias/2, - fun check_pub_acl/2, + fun check_pub_authz/2, fun check_pub_caps/2 ], Packet, Channel) of {ok, NPacket, NChannel} -> @@ -525,7 +551,7 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), {error, Rc = ?RC_NOT_AUTHORIZED, NChannel} -> ?LOG(warning, "Cannot publish message to ~s due to ~s.", [Topic, emqx_reason_codes:text(Rc)]), - case emqx_zone:get_env(Zone, acl_deny_action, ignore) of + case emqx_config:get_zone_conf(Zone, [authorization, deny_action]) of ignore -> case QoS of ?QOS_0 -> {ok, NChannel}; @@ -711,7 +737,7 @@ process_disconnect(ReasonCode, Properties, Channel) -> maybe_update_expiry_interval(#{'Session-Expiry-Interval' := Interval}, Channel = #channel{conninfo = ConnInfo}) -> - Channel#channel{conninfo = ConnInfo#{expiry_interval => Interval}}; + Channel#channel{conninfo = ConnInfo#{expiry_interval => timer:seconds(Interval)}}; maybe_update_expiry_interval(_Properties, Channel) -> Channel. %%-------------------------------------------------------------------- @@ -930,8 +956,9 @@ handle_call({takeover, 'end'}, Channel = #channel{session = Session, AllPendings = lists:append(Delivers, Pendings), disconnect_and_shutdown(takeovered, AllPendings, Channel); -handle_call(list_acl_cache, Channel) -> - {reply, emqx_acl_cache:list_acl_cache(), Channel}; +handle_call(list_authz_cache, #channel{clientinfo = #{zone := Zone}} + = Channel) -> + {reply, emqx_authz_cache:list_authz_cache(Zone), Channel}; handle_call({quota, Policy}, Channel) -> Zone = info(zone, Channel), @@ -967,9 +994,10 @@ handle_info({sock_closed, Reason}, Channel = #channel{conn_state = connecting}) shutdown(Reason, Channel); handle_info({sock_closed, Reason}, Channel = - #channel{conn_state = connected, - clientinfo = ClientInfo = #{zone := Zone}}) -> - emqx_zone:enable_flapping_detect(Zone) + #channel{conn_state = ConnState, + clientinfo = ClientInfo = #{zone := Zone}}) + when ConnState =:= connected orelse ConnState =:= reauthenticating -> + emqx_config:get_zone_conf(Zone, [flapping_detect, enable]) andalso emqx_flapping:detect(ClientInfo), Channel1 = ensure_disconnected(Reason, mabye_publish_will_msg(Channel)), case maybe_shutdown(Reason, Channel1) of @@ -981,8 +1009,8 @@ handle_info({sock_closed, Reason}, Channel = #channel{conn_state = disconnected} ?LOG(error, "Unexpected sock_closed: ~p", [Reason]), {ok, Channel}; -handle_info(clean_acl_cache, Channel) -> - ok = emqx_acl_cache:empty_acl_cache(), +handle_info(clean_authz_cache, Channel) -> + ok = emqx_authz_cache:empty_authz_cache(), {ok, Channel}; handle_info(Info, Channel) -> @@ -1086,11 +1114,11 @@ clean_timer(Name, Channel = #channel{timers = Timers}) -> interval(alive_timer, #channel{keepalive = KeepAlive}) -> emqx_keepalive:info(interval, KeepAlive); interval(retry_timer, #channel{session = Session}) -> - timer:seconds(emqx_session:info(retry_interval, Session)); + emqx_session:info(retry_interval, Session); interval(await_timer, #channel{session = Session}) -> - timer:seconds(emqx_session:info(await_rel_timeout, Session)); + emqx_session:info(await_rel_timeout, Session); interval(expire_timer, #channel{conninfo = ConnInfo}) -> - timer:seconds(maps:get(expiry_interval, ConnInfo)); + maps:get(expiry_interval, ConnInfo); interval(will_timer, #channel{will_msg = WillMsg}) -> timer:seconds(will_delay_interval(WillMsg)). @@ -1146,17 +1174,16 @@ enrich_conninfo(ConnPkt = #mqtt_packet_connect{ {ok, Channel#channel{conninfo = NConnInfo}}. %% If the Session Expiry Interval is absent the value 0 is used. --compile({inline, [expiry_interval/2]}). -expiry_interval(_Zone, #mqtt_packet_connect{proto_ver = ?MQTT_PROTO_V5, +expiry_interval(_, #mqtt_packet_connect{proto_ver = ?MQTT_PROTO_V5, properties = ConnProps}) -> - emqx_mqtt_props:get('Session-Expiry-Interval', ConnProps, 0); + timer:seconds(emqx_mqtt_props:get('Session-Expiry-Interval', ConnProps, 0)); expiry_interval(Zone, #mqtt_packet_connect{clean_start = false}) -> - emqx_zone:session_expiry_interval(Zone); -expiry_interval(_Zone, #mqtt_packet_connect{clean_start = true}) -> + get_mqtt_conf(Zone, session_expiry_interval); +expiry_interval(_, #mqtt_packet_connect{clean_start = true}) -> 0. receive_maximum(Zone, ConnProps) -> - MaxInflightConfig = case emqx_zone:max_inflight(Zone) of + MaxInflightConfig = case get_mqtt_conf(Zone, max_inflight) of 0 -> ?RECEIVE_MAXIMUM_LIMIT; N -> N end, @@ -1205,8 +1232,9 @@ set_bridge_mode(_ConnPkt, _ClientInfo) -> ok. maybe_username_as_clientid(_ConnPkt, ClientInfo = #{username := undefined}) -> {ok, ClientInfo}; -maybe_username_as_clientid(_ConnPkt, ClientInfo = #{zone := Zone, username := Username}) -> - case emqx_zone:use_username_as_clientid(Zone) of +maybe_username_as_clientid(_ConnPkt, ClientInfo = #{zone := Zone, + username := Username}) -> + case get_mqtt_conf(Zone, use_username_as_clientid) of true -> {ok, ClientInfo#{clientid => Username}}; false -> ok end. @@ -1234,82 +1262,67 @@ set_log_meta(_ConnPkt, #channel{clientinfo = #{clientid := ClientId}}) -> %%-------------------------------------------------------------------- %% Check banned -check_banned(_ConnPkt, #channel{clientinfo = ClientInfo = #{zone := Zone}}) -> - case emqx_zone:enable_ban(Zone) andalso emqx_banned:check(ClientInfo) of +check_banned(_ConnPkt, #channel{clientinfo = ClientInfo}) -> + case emqx_banned:check(ClientInfo) of true -> {error, ?RC_BANNED}; false -> ok end. %%-------------------------------------------------------------------- -%% Auth Connect +%% Authenticate -auth_connect(#mqtt_packet_connect{password = Password}, +authenticate(?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{'Authentication-Method' := AuthMethod} = Properties}), + #channel{clientinfo = ClientInfo, + auth_cache = AuthCache} = Channel) -> + AuthData = emqx_mqtt_props:get('Authentication-Data', Properties, undefined), + do_authenticate(ClientInfo#{auth_method => AuthMethod, + auth_data => AuthData, + auth_cache => AuthCache}, Channel); + +authenticate(?CONNECT_PACKET(#mqtt_packet_connect{password = Password}), #channel{clientinfo = ClientInfo} = Channel) -> - #{clientid := ClientId, - username := Username} = ClientInfo, - case emqx_access_control:authenticate(ClientInfo#{password => Password}) of - {ok, AuthResult} -> - is_anonymous(AuthResult) andalso - emqx_metrics:inc('client.auth.anonymous'), - NClientInfo = maps:merge(ClientInfo, AuthResult), - {ok, Channel#channel{clientinfo = NClientInfo}}; - {error, Reason} -> - ?LOG(warning, "Client ~s (Username: '~s') login failed for ~0p", - [ClientId, Username, Reason]), - {error, emqx_reason_codes:connack_error(Reason)} + do_authenticate(ClientInfo#{password => Password}, Channel); + +authenticate(?AUTH_PACKET(_, #{'Authentication-Method' := AuthMethod} = Properties), + #channel{clientinfo = ClientInfo, + conninfo = #{conn_props := ConnProps}, + auth_cache = AuthCache} = Channel) -> + case emqx_mqtt_props:get('Authentication-Method', ConnProps, undefined) of + AuthMethod -> + AuthData = emqx_mqtt_props:get('Authentication-Data', Properties, undefined), + do_authenticate(ClientInfo#{auth_method => AuthMethod, + auth_data => AuthData, + auth_cache => AuthCache}, Channel); + _ -> + {error, ?RC_BAD_AUTHENTICATION_METHOD} end. -is_anonymous(#{anonymous := true}) -> true; -is_anonymous(_AuthResult) -> false. - -%%-------------------------------------------------------------------- -%% Enhanced Authentication - -enhanced_auth(?CONNECT_PACKET(#mqtt_packet_connect{ - proto_ver = Protover, - properties = Properties - }), Channel) -> - case Protover of - ?MQTT_PROTO_V5 -> - AuthMethod = emqx_mqtt_props:get('Authentication-Method', Properties, undefined), - AuthData = emqx_mqtt_props:get('Authentication-Data', Properties, undefined), - do_enhanced_auth(AuthMethod, AuthData, Channel); - _ -> - {ok, #{}, Channel} +do_authenticate(#{auth_method := AuthMethod} = Credential, Channel) -> + Properties = #{'Authentication-Method' => AuthMethod}, + case emqx_access_control:authenticate(Credential) of + ok -> + {ok, Properties, Channel#channel{auth_cache = #{}}}; + {ok, AuthData} -> + {ok, Properties#{'Authentication-Data' => AuthData}, + Channel#channel{auth_cache = #{}}}; + {continue, AuthCache} -> + {continue, Properties, Channel#channel{auth_cache = AuthCache}}; + {continue, AuthData, AuthCache} -> + {continue, Properties#{'Authentication-Data' => AuthData}, + Channel#channel{auth_cache = AuthCache}}; + {error, Reason} -> + {error, emqx_reason_codes:connack_error(Reason)} end; -enhanced_auth(?AUTH_PACKET(_ReasonCode, Properties), Channel = #channel{conninfo = ConnInfo}) -> - AuthMethod = emqx_mqtt_props:get('Authentication-Method', - emqx_mqtt_props:get(conn_props, ConnInfo, #{}), - undefined - ), - NAuthMethod = emqx_mqtt_props:get('Authentication-Method', Properties, undefined), - AuthData = emqx_mqtt_props:get('Authentication-Data', Properties, undefined), - case NAuthMethod =:= undefined orelse NAuthMethod =/= AuthMethod of - true -> - {error, emqx_reason_codes:connack_error(bad_authentication_method), Channel}; - false -> - do_enhanced_auth(AuthMethod, AuthData, Channel) - end. - -do_enhanced_auth(undefined, undefined, Channel) -> - {ok, #{}, Channel}; -do_enhanced_auth(undefined, _AuthData, Channel) -> - {error, emqx_reason_codes:connack_error(not_authorized), Channel}; -do_enhanced_auth(_AuthMethod, undefined, Channel) -> - {error, emqx_reason_codes:connack_error(not_authorized), Channel}; -do_enhanced_auth(AuthMethod, AuthData, Channel = #channel{auth_cache = Cache}) -> - case run_hooks('client.enhanced_authenticate', [AuthMethod, AuthData], Cache) of - {ok, NAuthData, NCache} -> - NProperties = #{'Authentication-Method' => AuthMethod, - 'Authentication-Data' => NAuthData}, - {ok, NProperties, Channel#channel{auth_cache = NCache}}; - {continue, NAuthData, NCache} -> - NProperties = #{'Authentication-Method' => AuthMethod, - 'Authentication-Data' => NAuthData}, - {continue, NProperties, Channel#channel{auth_cache = NCache}}; - _ -> - {error, emqx_reason_codes:connack_error(not_authorized), Channel} +do_authenticate(Credential, Channel) -> + case emqx_access_control:authenticate(Credential) of + ok -> + {ok, #{}, Channel}; + {error, Reason} -> + {error, emqx_reason_codes:connack_error(Reason)} end. %%-------------------------------------------------------------------- @@ -1401,12 +1414,12 @@ check_pub_alias(#mqtt_packet{ check_pub_alias(_Packet, _Channel) -> ok. %%-------------------------------------------------------------------- -%% Check Pub ACL +%% Check Pub Authorization -check_pub_acl(#mqtt_packet{variable = #mqtt_packet_publish{topic_name = Topic}}, +check_pub_authz(#mqtt_packet{variable = #mqtt_packet_publish{topic_name = Topic}}, #channel{clientinfo = ClientInfo}) -> - case is_acl_enabled(ClientInfo) andalso - emqx_access_control:check_acl(ClientInfo, publish, Topic) of + case is_authz_enabled(ClientInfo) andalso + emqx_access_control:authorize(ClientInfo, publish, Topic) of false -> ok; allow -> ok; deny -> {error, ?RC_NOT_AUTHORIZED} @@ -1423,24 +1436,24 @@ check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic => Topic}). %%-------------------------------------------------------------------- -%% Check Sub ACL +%% Check Sub Authorization -check_sub_acls(TopicFilters, Channel) -> - check_sub_acls(TopicFilters, Channel, []). +check_sub_authzs(TopicFilters, Channel) -> + check_sub_authzs(TopicFilters, Channel, []). -check_sub_acls([ TopicFilter = {Topic, _} | More] , Channel, Acc) -> - case check_sub_acl(Topic, Channel) of +check_sub_authzs([ TopicFilter = {Topic, _} | More] , Channel, Acc) -> + case check_sub_authz(Topic, Channel) of allow -> - check_sub_acls(More, Channel, [ {TopicFilter, 0} | Acc]); + check_sub_authzs(More, Channel, [ {TopicFilter, 0} | Acc]); deny -> - check_sub_acls(More, Channel, [ {TopicFilter, ?RC_NOT_AUTHORIZED} | Acc]) + check_sub_authzs(More, Channel, [ {TopicFilter, ?RC_NOT_AUTHORIZED} | Acc]) end; -check_sub_acls([], _Channel, Acc) -> +check_sub_authzs([], _Channel, Acc) -> lists:reverse(Acc). -check_sub_acl(TopicFilter, #channel{clientinfo = ClientInfo}) -> - case is_acl_enabled(ClientInfo) andalso - emqx_access_control:check_acl(ClientInfo, subscribe, TopicFilter) of +check_sub_authz(TopicFilter, #channel{clientinfo = ClientInfo}) -> + case is_authz_enabled(ClientInfo) andalso + emqx_access_control:authorize(ClientInfo, subscribe, TopicFilter) of false -> allow; Result -> Result end. @@ -1464,13 +1477,14 @@ put_subid_in_subopts(_Properties, TopicFilters) -> TopicFilters. enrich_subopts(SubOpts, _Channel = ?IS_MQTT_V5) -> SubOpts; enrich_subopts(SubOpts, #channel{clientinfo = #{zone := Zone, is_bridge := IsBridge}}) -> - NL = flag(emqx_zone:ignore_loop_deliver(Zone)), + NL = flag(get_mqtt_conf(Zone, ignore_loop_deliver)), SubOpts#{rap => flag(IsBridge), nl => NL}. %%-------------------------------------------------------------------- %% Enrich ConnAck Caps -enrich_connack_caps(AckProps, ?IS_MQTT_V5 = #channel{clientinfo = #{zone := Zone}}) -> +enrich_connack_caps(AckProps, ?IS_MQTT_V5 = #channel{clientinfo = #{ + zone := Zone}}) -> #{max_packet_size := MaxPktSize, max_qos_allowed := MaxQoS, retain_available := Retain, @@ -1500,8 +1514,8 @@ enrich_connack_caps(AckProps, _Channel) -> AckProps. %% Enrich server keepalive enrich_server_keepalive(AckProps, #channel{clientinfo = #{zone := Zone}}) -> - case emqx_zone:server_keepalive(Zone) of - undefined -> AckProps; + case get_mqtt_conf(Zone, server_keepalive) of + disabled -> AckProps; Keepalive -> AckProps#{'Server-Keep-Alive' => Keepalive} end. @@ -1509,10 +1523,14 @@ enrich_server_keepalive(AckProps, #channel{clientinfo = #{zone := Zone}}) -> %% Enrich response information enrich_response_information(AckProps, #channel{conninfo = #{conn_props := ConnProps}, - clientinfo = #{zone := Zone}}) -> + clientinfo = #{zone := Zone}}) -> case emqx_mqtt_props:get('Request-Response-Information', ConnProps, 0) of 0 -> AckProps; - 1 -> AckProps#{'Response-Information' => emqx_zone:response_information(Zone)} + 1 -> AckProps#{'Response-Information' => + case get_mqtt_conf(Zone, response_information, "") of + "" -> undefined; + RspInfo -> RspInfo + end} end. %%-------------------------------------------------------------------- @@ -1544,7 +1562,7 @@ init_alias_maximum(#mqtt_packet_connect{proto_ver = ?MQTT_PROTO_V5, properties = Properties}, #{zone := Zone} = _ClientInfo) -> #{outbound => emqx_mqtt_props:get('Topic-Alias-Maximum', Properties, 0), - inbound => emqx_mqtt_caps:get_caps(Zone, max_topic_alias, ?MAX_TOPIC_AlIAS) + inbound => maps:get(max_topic_alias, emqx_mqtt_caps:get_caps(Zone)) }; init_alias_maximum(_ConnPkt, _ClientInfo) -> undefined. @@ -1560,8 +1578,9 @@ ensure_keepalive(_AckProps, Channel = #channel{conninfo = ConnInfo}) -> ensure_keepalive_timer(maps:get(keepalive, ConnInfo), Channel). ensure_keepalive_timer(0, Channel) -> Channel; +ensure_keepalive_timer(disabled, Channel) -> Channel; ensure_keepalive_timer(Interval, Channel = #channel{clientinfo = #{zone := Zone}}) -> - Backoff = emqx_zone:keepalive_backoff(Zone), + Backoff = get_mqtt_conf(Zone, keepalive_backoff), Keepalive = emqx_keepalive:init(round(timer:seconds(Interval) * Backoff)), ensure_timer(alive_timer, Channel#channel{keepalive = Keepalive}). @@ -1596,16 +1615,14 @@ maybe_shutdown(Reason, Channel = #channel{conninfo = ConnInfo}) -> case maps:get(expiry_interval, ConnInfo) of ?UINT_MAX -> {ok, Channel}; I when I > 0 -> - {ok, ensure_timer(expire_timer, timer:seconds(I), Channel)}; + {ok, ensure_timer(expire_timer, I, Channel)}; _ -> shutdown(Reason, Channel) end. %%-------------------------------------------------------------------- -%% Is ACL enabled? - --compile({inline, [is_acl_enabled/1]}). -is_acl_enabled(#{zone := Zone, is_superuser := IsSuperuser}) -> - (not IsSuperuser) andalso emqx_zone:enable_acl(Zone). +%% Is Authorization enabled? +is_authz_enabled(#{zone := Zone, is_superuser := IsSuperuser}) -> + (not IsSuperuser) andalso emqx_config:get_zone_conf(Zone, [authorization, enable]). %%-------------------------------------------------------------------- %% Parse Topic Filters @@ -1703,7 +1720,8 @@ shutdown(Reason, Reply, Packet, Channel) -> {shutdown, Reason, Reply, Packet, Channel}. disconnect_and_shutdown(Reason, Reply, Channel = ?IS_MQTT_V5 - = #channel{conn_state = connected}) -> + = #channel{conn_state = ConnState}) + when ConnState =:= connected orelse ConnState =:= reauthenticating -> shutdown(Reason, Reply, ?DISCONNECT_PACKET(reason_code(Reason)), Channel); disconnect_and_shutdown(Reason, Reply, Channel) -> @@ -1715,6 +1733,12 @@ sp(false) -> 0. flag(true) -> 1; flag(false) -> 0. +get_mqtt_conf(Zone, Key) -> + emqx_config:get_zone_conf(Zone, [mqtt, Key]). + +get_mqtt_conf(Zone, Key, Default) -> + emqx_config:get_zone_conf(Zone, [mqtt, Key], Default). + %%-------------------------------------------------------------------- %% For CT tests %%-------------------------------------------------------------------- @@ -1722,4 +1746,3 @@ flag(false) -> 0. set_field(Name, Value, Channel) -> Pos = emqx_misc:index_of(Name, record_info(fields, channel)), setelement(Pos+1, Channel, Value). - diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index 6eb375aba..e2c8438a2 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -241,11 +241,31 @@ open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> emqx_cm_locker:trans(ClientId, ResumeStart). create_session(ClientInfo, ConnInfo) -> - Session = emqx_session:init(ClientInfo, ConnInfo), + Options = get_session_confs(ClientInfo, ConnInfo), + Session = emqx_session:init(Options), ok = emqx_metrics:inc('session.created'), ok = emqx_hooks:run('session.created', [ClientInfo, emqx_session:info(Session)]), Session. +get_session_confs(#{zone := Zone}, #{receive_maximum := MaxInflight}) -> + #{max_subscriptions => get_mqtt_conf(Zone, max_subscriptions), + upgrade_qos => get_mqtt_conf(Zone, upgrade_qos), + max_inflight => MaxInflight, + retry_interval => get_mqtt_conf(Zone, retry_interval), + await_rel_timeout => get_mqtt_conf(Zone, await_rel_timeout), + mqueue => mqueue_confs(Zone) + }. + +mqueue_confs(Zone) -> + #{max_len => get_mqtt_conf(Zone, max_mqueue_len), + store_qos0 => get_mqtt_conf(Zone, mqueue_store_qos0), + priorities => get_mqtt_conf(Zone, mqueue_priorities), + default_priority => get_mqtt_conf(Zone, mqueue_default_priority) + }. + +get_mqtt_conf(Zone, Key) -> + emqx_config:get_zone_conf(Zone, [mqtt, Key]). + %% @doc Try to takeover a session. -spec(takeover_session(emqx_types:clientid()) -> {error, term()} diff --git a/apps/emqx/src/emqx_cm_locker.erl b/apps/emqx/src/emqx_cm_locker.erl index 4ce6a9279..c1a85d6c9 100644 --- a/apps/emqx/src/emqx_cm_locker.erl +++ b/apps/emqx/src/emqx_cm_locker.erl @@ -62,5 +62,5 @@ unlock(ClientId) -> -spec(strategy() -> local | leader | quorum | all). strategy() -> - emqx:get_env(session_locking_strategy, quorum). + emqx_config:get([broker, session_locking_strategy]). diff --git a/apps/emqx/src/emqx_cm_registry.erl b/apps/emqx/src/emqx_cm_registry.erl index d8095b445..da716ca29 100644 --- a/apps/emqx/src/emqx_cm_registry.erl +++ b/apps/emqx/src/emqx_cm_registry.erl @@ -48,7 +48,9 @@ -define(TAB, emqx_channel_registry). -define(LOCK, {?MODULE, cleanup_down}). --rlog_shard({?ROUTE_SHARD, ?TAB}). +-define(CM_SHARD, emqx_cm_shard). + +-rlog_shard({?CM_SHARD, ?TAB}). -record(channel, {chid, pid}). @@ -64,7 +66,7 @@ start_link() -> %% @doc Is the global registry enabled? -spec(is_enabled() -> boolean()). is_enabled() -> - emqx:get_env(enable_session_registry, true). + emqx_config:get([broker, enable_session_registry]). %% @doc Register a global channel. -spec(register_channel(emqx_types:clientid() @@ -111,6 +113,7 @@ init([]) -> {storage_properties, [{ets, [{read_concurrency, true}, {write_concurrency, true}]}]}]), ok = ekka_mnesia:copy_table(?TAB, ram_copies), + ok = ekka_rlog:wait_for_shards([?CM_SHARD], infinity), ok = ekka:monitor(membership), {ok, #{}}. @@ -125,7 +128,7 @@ handle_cast(Msg, State) -> handle_info({membership, {mnesia, down, Node}}, State) -> global:trans({?LOCK, self()}, fun() -> - ekka_mnesia:transaction(?ROUTE_SHARD, fun cleanup_channels/1, [Node]) + ekka_mnesia:transaction(?CM_SHARD, fun cleanup_channels/1, [Node]) end), {noreply, State}; diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 213eb7a8b..43a41cdc6 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -15,132 +15,271 @@ %%-------------------------------------------------------------------- -module(emqx_config). --compile({no_auto_import, [get/0, get/1]}). +-compile({no_auto_import, [get/0, get/1, put/2]}). --export([ get/0 - , get/1 +-export([ init_load/2 + , read_override_conf/0 + , check_config/2 + , save_configs/4 + , save_to_app_env/1 + , save_to_config_map/2 + , save_to_override_conf/1 + ]). + +-export([get_root/1, + get_root_raw/1]). + +-export([ get/1 , get/2 + , find/1 , put/1 , put/2 ]). --export([ update_config/2 +-export([ get_zone_conf/2 + , get_zone_conf/3 + , put_zone_conf/3 + , find_zone_conf/2 ]). -%% raw configs is the config that is now parsed and tranlated by hocon schema --export([ get_raw/0 - , get_raw/1 +-export([ get_listener_conf/3 + , get_listener_conf/4 + , put_listener_conf/4 + , find_listener_conf/3 + ]). + +-export([ update/2 + , update/3 + , remove/1 + , remove/2 + ]). + +-export([ get_raw/1 , get_raw/2 , put_raw/1 , put_raw/2 ]). --export([ deep_get/2 - , deep_get/3 - , deep_put/3 - , safe_atom_key_map/1 - , unsafe_atom_key_map/1 - ]). +-define(CONF, fun(ROOT) -> {?MODULE, bin(ROOT)} end). +-define(RAW_CONF, fun(ROOT) -> {?MODULE, raw, bin(ROOT)} end). +-define(ZONE_CONF_PATH(ZONE, PATH), [zones, ZONE | PATH]). +-define(LISTENER_CONF_PATH(ZONE, LISTENER, PATH), [zones, ZONE, listeners, LISTENER | PATH]). --define(CONF, ?MODULE). --define(RAW_CONF, {?MODULE, raw}). - --export_type([update_request/0, raw_config/0, config_key/0, config_key_path/0]). +-export_type([update_request/0, raw_config/0, config/0]). -type update_request() :: term(). --type raw_config() :: hocon:config() | undefined. --type config_key() :: atom() | binary(). --type config_key_path() :: [config_key()]. +%% raw_config() is the config that is NOT parsed and tranlated by hocon schema +-type raw_config() :: #{binary() => term()} | undefined. +%% config() is the config that is parsed and tranlated by hocon schema +-type config() :: #{atom() => term()} | undefined. +-type app_envs() :: [proplists:property()]. --spec get() -> map(). -get() -> - persistent_term:get(?CONF, #{}). +%% @doc For the given path, get root value enclosed in a single-key map. +-spec get_root(emqx_map_lib:config_key_path()) -> map(). +get_root([RootName | _]) -> + #{RootName => do_get(?CONF, [RootName], #{})}. --spec get(config_key_path()) -> term(). -get(KeyPath) -> - deep_get(KeyPath, get()). +%% @doc For the given path, get raw root value enclosed in a single-key map. +%% key is ensured to be binary. +get_root_raw([RootName | _]) -> + #{bin(RootName) => do_get(?RAW_CONF, [RootName], #{})}. --spec get(config_key_path(), term()) -> term(). -get(KeyPath, Default) -> - deep_get(KeyPath, get(), Default). +%% @doc Get a config value for the given path. +%% The path should at least include root config name. +-spec get(emqx_map_lib:config_key_path()) -> term(). +get(KeyPath) -> do_get(?CONF, KeyPath). + +-spec get(emqx_map_lib:config_key_path(), term()) -> term(). +get(KeyPath, Default) -> do_get(?CONF, KeyPath, Default). + +-spec find(emqx_map_lib:config_key_path()) -> + {ok, term()} | {not_found, emqx_map_lib:config_key_path(), term()}. +find(KeyPath) -> + emqx_map_lib:deep_find(KeyPath, get_root(KeyPath)). + +-spec get_zone_conf(atom(), emqx_map_lib:config_key_path()) -> term(). +get_zone_conf(Zone, KeyPath) -> + ?MODULE:get(?ZONE_CONF_PATH(Zone, KeyPath)). + +-spec get_zone_conf(atom(), emqx_map_lib:config_key_path(), term()) -> term(). +get_zone_conf(Zone, KeyPath, Default) -> + ?MODULE:get(?ZONE_CONF_PATH(Zone, KeyPath), Default). + +-spec put_zone_conf(atom(), emqx_map_lib:config_key_path(), term()) -> ok. +put_zone_conf(Zone, KeyPath, Conf) -> + ?MODULE:put(?ZONE_CONF_PATH(Zone, KeyPath), Conf). + +-spec find_zone_conf(atom(), emqx_map_lib:config_key_path()) -> + {ok, term()} | {not_found, emqx_map_lib:config_key_path(), term()}. +find_zone_conf(Zone, KeyPath) -> + find(?ZONE_CONF_PATH(Zone, KeyPath)). + +-spec get_listener_conf(atom(), atom(), emqx_map_lib:config_key_path()) -> term(). +get_listener_conf(Zone, Listener, KeyPath) -> + ?MODULE:get(?LISTENER_CONF_PATH(Zone, Listener, KeyPath)). + +-spec get_listener_conf(atom(), atom(), emqx_map_lib:config_key_path(), term()) -> term(). +get_listener_conf(Zone, Listener, KeyPath, Default) -> + ?MODULE:get(?LISTENER_CONF_PATH(Zone, Listener, KeyPath), Default). + +-spec put_listener_conf(atom(), atom(), emqx_map_lib:config_key_path(), term()) -> ok. +put_listener_conf(Zone, Listener, KeyPath, Conf) -> + ?MODULE:put(?LISTENER_CONF_PATH(Zone, Listener, KeyPath), Conf). + +-spec find_listener_conf(atom(), atom(), emqx_map_lib:config_key_path()) -> + {ok, term()} | {not_found, emqx_map_lib:config_key_path(), term()}. +find_listener_conf(Zone, Listener, KeyPath) -> + find(?LISTENER_CONF_PATH(Zone, Listener, KeyPath)). -spec put(map()) -> ok. put(Config) -> - persistent_term:put(?CONF, Config). + maps:fold(fun(RootName, RootValue, _) -> + ?MODULE:put([RootName], RootValue) + end, [], Config). --spec put(config_key_path(), term()) -> ok. -put(KeyPath, Config) -> - put(deep_put(KeyPath, get(), Config)). +-spec put(emqx_map_lib:config_key_path(), term()) -> ok. +put(KeyPath, Config) -> do_put(?CONF, KeyPath, Config). --spec update_config(config_key_path(), update_request()) -> +-spec update(emqx_map_lib:config_key_path(), update_request()) -> ok | {error, term()}. -update_config(ConfKeyPath, UpdateReq) -> - emqx_config_handler:update_config(ConfKeyPath, UpdateReq, get_raw()). +update(ConfKeyPath, UpdateReq) -> + update(emqx_schema, ConfKeyPath, UpdateReq). --spec get_raw() -> map(). -get_raw() -> - persistent_term:get(?RAW_CONF, #{}). +-spec update(module(), emqx_map_lib:config_key_path(), update_request()) -> + ok | {error, term()}. +update(SchemaModule, ConfKeyPath, UpdateReq) -> + emqx_config_handler:update_config(SchemaModule, ConfKeyPath, UpdateReq). --spec get_raw(config_key_path()) -> term(). -get_raw(KeyPath) -> - deep_get(KeyPath, get_raw()). +-spec remove(emqx_map_lib:config_key_path()) -> ok | {error, term()}. +remove(ConfKeyPath) -> + remove(emqx_schema, ConfKeyPath). --spec get_raw(config_key_path(), term()) -> term(). -get_raw(KeyPath, Default) -> - deep_get(KeyPath, get_raw(), Default). +remove(SchemaModule, ConfKeyPath) -> + emqx_config_handler:remove_config(SchemaModule, ConfKeyPath). + +-spec get_raw(emqx_map_lib:config_key_path()) -> term(). +get_raw(KeyPath) -> do_get(?RAW_CONF, KeyPath). + +-spec get_raw(emqx_map_lib:config_key_path(), term()) -> term(). +get_raw(KeyPath, Default) -> do_get(?RAW_CONF, KeyPath, Default). -spec put_raw(map()) -> ok. put_raw(Config) -> - persistent_term:put(?RAW_CONF, Config). + maps:fold(fun(RootName, RootV, _) -> + ?MODULE:put_raw([RootName], RootV) + end, [], hocon_schema:get_value([], Config)). --spec put_raw(config_key_path(), term()) -> ok. -put_raw(KeyPath, Config) -> - put_raw(deep_put(KeyPath, get_raw(), Config)). +-spec put_raw(emqx_map_lib:config_key_path(), term()) -> ok. +put_raw(KeyPath, Config) -> do_put(?RAW_CONF, KeyPath, Config). -%%----------------------------------------------------------------- --dialyzer([{nowarn_function, [deep_get/2]}]). --spec deep_get(config_key_path(), map()) -> term(). -deep_get(ConfKeyPath, Map) -> - do_deep_get(ConfKeyPath, Map, fun(KeyPath, Data) -> - error({not_found, KeyPath, Data}) end). +%%============================================================================ +%% Load/Update configs From/To files +%%============================================================================ --spec deep_get(config_key_path(), map(), term()) -> term(). -deep_get(ConfKeyPath, Map, Default) -> - do_deep_get(ConfKeyPath, Map, fun(_, _) -> Default end). - --spec deep_put(config_key_path(), map(), term()) -> map(). -deep_put([], Map, Config) when is_map(Map) -> - Config; -deep_put([Key | KeyPath], Map, Config) -> - SubMap = deep_put(KeyPath, maps:get(Key, Map, #{}), Config), - Map#{Key => SubMap}. - -unsafe_atom_key_map(Map) -> - covert_keys_to_atom(Map, fun(K) -> binary_to_atom(K, utf8) end). - -safe_atom_key_map(Map) -> - covert_keys_to_atom(Map, fun(K) -> binary_to_existing_atom(K, utf8) end). - -%%--------------------------------------------------------------------------- - --spec do_deep_get(config_key_path(), map(), fun((config_key(), term()) -> any())) -> term(). -do_deep_get([], Map, _) -> - Map; -do_deep_get([Key | KeyPath], Map, OnNotFound) when is_map(Map) -> - case maps:find(Key, Map) of - {ok, SubMap} -> do_deep_get(KeyPath, SubMap, OnNotFound); - error -> OnNotFound(Key, Map) +%% @doc Initial load of the given config files. +%% NOTE: The order of the files is significant, configs from files orderd +%% in the rear of the list overrides prior values. +-spec init_load(module(), [string()] | binary() | hocon:config()) -> ok. +init_load(SchemaModule, Conf) when is_list(Conf) orelse is_binary(Conf) -> + ParseOptions = #{format => richmap}, + Parser = case is_binary(Conf) of + true -> fun hocon:binary/2; + false -> fun hocon:files/2 + end, + case Parser(Conf, ParseOptions) of + {ok, RawRichConf} -> + init_load(SchemaModule, RawRichConf); + {error, Reason} -> + logger:error(#{msg => failed_to_load_hocon_conf, + reason => Reason + }), + error(failed_to_load_hocon_conf) end; -do_deep_get([Key | _KeyPath], Data, OnNotFound) -> - OnNotFound(Key, Data). +init_load(SchemaModule, RawRichConf) when is_map(RawRichConf) -> + %% check with richmap for line numbers in error reports (future enhancement) + Opts = #{return_plain => true, + nullable => true + }, + %% this call throws exception in case of check failure + {_AppEnvs, CheckedConf} = hocon_schema:map_translate(SchemaModule, RawRichConf, Opts), + ok = save_to_config_map(emqx_map_lib:unsafe_atom_key_map(CheckedConf), + hocon_schema:richmap_to_map(RawRichConf)). -covert_keys_to_atom(BinKeyMap, Conv) when is_map(BinKeyMap) -> - maps:fold( - fun(K, V, Acc) when is_binary(K) -> - Acc#{Conv(K) => covert_keys_to_atom(V, Conv)}; - (K, V, Acc) when is_atom(K) -> - %% richmap keys - Acc#{K => covert_keys_to_atom(V, Conv)} - end, #{}, BinKeyMap); -covert_keys_to_atom(ListV, Conv) when is_list(ListV) -> - [covert_keys_to_atom(V, Conv) || V <- ListV]; -covert_keys_to_atom(Val, _) -> Val. +-spec check_config(module(), raw_config()) -> {AppEnvs, CheckedConf} + when AppEnvs :: app_envs(), CheckedConf :: config(). +check_config(SchemaModule, RawConf) -> + Opts = #{return_plain => true, + nullable => true, + is_richmap => false + }, + {AppEnvs, CheckedConf} = + hocon_schema:map_translate(SchemaModule, RawConf, Opts), + Conf = maps:with(maps:keys(RawConf), CheckedConf), + {AppEnvs, emqx_map_lib:unsafe_atom_key_map(Conf)}. + +-spec read_override_conf() -> raw_config(). +read_override_conf() -> + load_hocon_file(emqx_override_conf_name(), map). + +-spec save_configs(app_envs(), config(), raw_config(), raw_config()) -> ok | {error, term()}. +save_configs(_AppEnvs, Conf, RawConf, OverrideConf) -> + %% We may need also support hot config update for the apps that use application envs. + %% If that is the case uncomment the following line to update the configs to app env + %save_to_app_env(AppEnvs), + save_to_config_map(Conf, RawConf), + %% TODO: merge RawConf to OverrideConf can be done here + save_to_override_conf(OverrideConf). + +-spec save_to_app_env([tuple()]) -> ok. +save_to_app_env(AppEnvs) -> + lists:foreach(fun({AppName, Envs}) -> + [application:set_env(AppName, Par, Val) || {Par, Val} <- Envs] + end, AppEnvs). + +-spec save_to_config_map(config(), raw_config()) -> ok. +save_to_config_map(Conf, RawConf) -> + ?MODULE:put(Conf), + ?MODULE:put_raw(RawConf). + +-spec save_to_override_conf(raw_config()) -> ok | {error, term()}. +save_to_override_conf(RawConf) -> + FileName = emqx_override_conf_name(), + ok = filelib:ensure_dir(FileName), + case file:write_file(FileName, jsx:prettify(jsx:encode(RawConf))) of + ok -> ok; + {error, Reason} -> + logger:error("write to ~s failed, ~p", [FileName, Reason]), + {error, Reason} + end. + +load_hocon_file(FileName, LoadType) -> + case filelib:is_regular(FileName) of + true -> + {ok, Raw0} = hocon:load(FileName, #{format => LoadType}), + Raw0; + false -> #{} + end. + +emqx_override_conf_name() -> + filename:join([?MODULE:get([node, data_dir]), "emqx_override.conf"]). + +bin(Bin) when is_binary(Bin) -> Bin; +bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). + +do_get(PtKey, KeyPath) -> + Ref = make_ref(), + Res = do_get(PtKey, KeyPath, Ref), + case Res =:= Ref of + true -> error({config_not_found, KeyPath}); + false -> Res + end. + +do_get(PtKey, [RootName], Default) -> + persistent_term:get(PtKey(RootName), Default); +do_get(PtKey, [RootName | KeyPath], Default) -> + RootV = persistent_term:get(PtKey(RootName), #{}), + emqx_map_lib:deep_get(KeyPath, RootV, Default). + +do_put(PtKey, [RootName | KeyPath], DeepValue) -> + OldValue = do_get(PtKey, [RootName], #{}), + NewValue = emqx_map_lib:deep_put(KeyPath, OldValue, DeepValue), + persistent_term:put(PtKey(RootName), NewValue). diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index bc915d778..9a830bf4d 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -25,13 +25,10 @@ -export([ start_link/0 , add_handler/2 , update_config/3 + , remove_config/2 , merge_to_old_config/2 ]). -%% emqx_config_handler callbacks --export([ handle_update_config/2 - ]). - %% gen_server callbacks -export([init/1, handle_call/3, @@ -41,15 +38,21 @@ code_change/3]). -define(MOD, {mod}). +-define(REMOVE_CONF, '$remove_config'). -type handler_name() :: module(). -type handlers() :: #{emqx_config:config_key() => handlers(), ?MOD => handler_name()}. --optional_callbacks([handle_update_config/2]). +-optional_callbacks([ pre_config_update/2 + , post_config_update/3 + ]). --callback handle_update_config(emqx_config:update_request(), emqx_config:raw_config()) -> +-callback pre_config_update(emqx_config:update_request(), emqx_config:raw_config()) -> emqx_config:update_request(). +-callback post_config_update(emqx_config:update_request(), emqx_config:config(), + emqx_config:config()) -> any(). + -type state() :: #{ handlers := handlers(), atom() => term() @@ -58,11 +61,15 @@ start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, {}, []). --spec update_config(emqx_config:config_key_path(), emqx_config:update_request(), - emqx_config:raw_config()) -> +-spec update_config(module(), emqx_config:config_key_path(), emqx_config:update_request()) -> ok | {error, term()}. -update_config(ConfKeyPath, UpdateReq, RawConfig) -> - gen_server:call(?MODULE, {update_config, ConfKeyPath, UpdateReq, RawConfig}). +update_config(SchemaModule, ConfKeyPath, UpdateReq) when UpdateReq =/= ?REMOVE_CONF -> + gen_server:call(?MODULE, {change_config, SchemaModule, ConfKeyPath, UpdateReq}). + +-spec remove_config(module(), emqx_config:config_key_path()) -> + ok | {error, term()}. +remove_config(SchemaModule, ConfKeyPath) -> + gen_server:call(?MODULE, {change_config, SchemaModule, ConfKeyPath, ?REMOVE_CONF}). -spec add_handler(emqx_config:config_key_path(), handler_name()) -> ok. add_handler(ConfKeyPath, HandlerName) -> @@ -72,26 +79,28 @@ add_handler(ConfKeyPath, HandlerName) -> -spec init(term()) -> {ok, state()}. init(_) -> - {ok, RawConf} = hocon:load(emqx_conf_name(), #{format => richmap}), - {_MappedEnvs, Conf} = hocon_schema:map_translate(emqx_schema, RawConf, #{}), - ok = save_config_to_emqx(to_plainmap(Conf), to_plainmap(RawConf)), {ok, #{handlers => #{?MOD => ?MODULE}}}. + handle_call({add_child, ConfKeyPath, HandlerName}, _From, State = #{handlers := Handlers}) -> {reply, ok, State#{handlers => - emqx_config:deep_put(ConfKeyPath, Handlers, #{?MOD => HandlerName})}}; + emqx_map_lib:deep_put(ConfKeyPath, Handlers, #{?MOD => HandlerName})}}; -handle_call({update_config, ConfKeyPath, UpdateReq, RawConf}, _From, +handle_call({change_config, SchemaModule, ConfKeyPath, UpdateReq}, _From, #{handlers := Handlers} = State) -> - try {RootKeys, Conf} = do_update_config(ConfKeyPath, Handlers, RawConf, UpdateReq), - {reply, save_configs(RootKeys, Conf), State} - catch - throw: Reason -> - {reply, {error, Reason}, State}; - Error : Reason : ST -> - ?LOG(error, "update config failed: ~p", [{Error, Reason, ST}]), - {reply, {error, Reason}, State} - end; + OldConf = emqx_config:get_root(ConfKeyPath), + OldRawConf = emqx_config:get_root_raw(ConfKeyPath), + Result = try + {NewRawConf, OverrideConf} = process_upadate_request(ConfKeyPath, OldRawConf, + Handlers, UpdateReq), + {AppEnvs, CheckedConf} = emqx_config:check_config(SchemaModule, NewRawConf), + _ = do_post_config_update(ConfKeyPath, Handlers, OldConf, CheckedConf, UpdateReq), + emqx_config:save_configs(AppEnvs, CheckedConf, NewRawConf, OverrideConf) + catch Error:Reason:ST -> + ?LOG(error, "change_config failed: ~p", [{Error, Reason, ST}]), + {error, Reason} + end, + {reply, Result, State}; handle_call(_Request, _From, State) -> Reply = ok, @@ -109,104 +118,67 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. -do_update_config([], Handlers, OldConf, UpdateReq) -> - call_handle_update_config(Handlers, OldConf, UpdateReq); -do_update_config([ConfKey | ConfKeyPath], Handlers, OldConf, UpdateReq) -> +process_upadate_request(ConfKeyPath, OldRawConf, _Handlers, ?REMOVE_CONF) -> + BinKeyPath = bin_path(ConfKeyPath), + NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf), + OverrideConf = emqx_map_lib:deep_remove(BinKeyPath, emqx_config:read_override_conf()), + {NewRawConf, OverrideConf}; +process_upadate_request(ConfKeyPath, OldRawConf, Handlers, UpdateReq) -> + NewRawConf = do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq), + OverrideConf = update_override_config(NewRawConf), + {NewRawConf, OverrideConf}. + +do_update_config([], Handlers, OldRawConf, UpdateReq) -> + call_pre_config_update(Handlers, OldRawConf, UpdateReq); +do_update_config([ConfKey | ConfKeyPath], Handlers, OldRawConf, UpdateReq) -> + SubOldRawConf = get_sub_config(bin(ConfKey), OldRawConf), + SubHandlers = maps:get(ConfKey, Handlers, #{}), + NewUpdateReq = do_update_config(ConfKeyPath, SubHandlers, SubOldRawConf, UpdateReq), + call_pre_config_update(Handlers, OldRawConf, #{bin(ConfKey) => NewUpdateReq}). + +do_post_config_update([], Handlers, OldConf, NewConf, UpdateReq) -> + call_post_config_update(Handlers, OldConf, NewConf, UpdateReq); +do_post_config_update([ConfKey | ConfKeyPath], Handlers, OldConf, NewConf, UpdateReq) -> SubOldConf = get_sub_config(ConfKey, OldConf), - case maps:find(ConfKey, Handlers) of - error -> throw({handler_not_found, ConfKey}); - {ok, SubHandlers} -> - NewUpdateReq = do_update_config(ConfKeyPath, SubHandlers, SubOldConf, UpdateReq), - call_handle_update_config(Handlers, OldConf, #{bin(ConfKey) => NewUpdateReq}) - end. + SubNewConf = get_sub_config(ConfKey, NewConf), + SubHandlers = maps:get(ConfKey, Handlers, #{}), + _ = do_post_config_update(ConfKeyPath, SubHandlers, SubOldConf, SubNewConf, UpdateReq), + call_post_config_update(Handlers, OldConf, NewConf, UpdateReq). -get_sub_config(_, undefined) -> - undefined; -get_sub_config(ConfKey, OldConf) when is_map(OldConf) -> - maps:get(bin(ConfKey), OldConf, undefined); -get_sub_config(_, OldConf) -> - OldConf. +get_sub_config(ConfKey, Conf) when is_map(Conf) -> + maps:get(ConfKey, Conf, undefined); +get_sub_config(_, _Conf) -> %% the Conf is a primitive + undefined. -call_handle_update_config(Handlers, OldConf, UpdateReq) -> +call_pre_config_update(Handlers, OldRawConf, UpdateReq) -> HandlerName = maps:get(?MOD, Handlers, undefined), - case erlang:function_exported(HandlerName, handle_update_config, 2) of - true -> HandlerName:handle_update_config(UpdateReq, OldConf); - false -> UpdateReq %% the default behaviour is overwriting the old config + case erlang:function_exported(HandlerName, pre_config_update, 2) of + true -> HandlerName:pre_config_update(UpdateReq, OldRawConf); + false -> merge_to_old_config(UpdateReq, OldRawConf) end. -%% callbacks for the top-level handler -handle_update_config(UpdateReq, OldConf) -> - FullRawConf = merge_to_old_config(UpdateReq, OldConf), - {maps:keys(UpdateReq), FullRawConf}. - -%% default callback of config handlers -merge_to_old_config(UpdateReq, undefined) -> - merge_to_old_config(UpdateReq, #{}); -merge_to_old_config(UpdateReq, RawConf) -> - maps:merge(RawConf, UpdateReq). - -%%============================================================================ -save_configs(RootKeys, RawConf) -> - {_MappedEnvs, Conf} = hocon_schema:map_translate(emqx_schema, to_richmap(RawConf), #{}), - %% We may need also support hot config update for the apps that use application envs. - %% If so uncomment the following line to update the configs to application env - %save_config_to_app_env(_MappedEnvs), - save_config_to_emqx(to_plainmap(Conf), RawConf), - save_config_to_disk(RootKeys, RawConf). - -% save_config_to_app_env(MappedEnvs) -> -% lists:foreach(fun({AppName, Envs}) -> -% [application:set_env(AppName, Par, Val) || {Par, Val} <- Envs] -% end, MappedEnvs). - -save_config_to_emqx(Conf, RawConf) -> - emqx_config:put(emqx_config:unsafe_atom_key_map(Conf)), - emqx_config:put_raw(RawConf). - -save_config_to_disk(RootKeys, Conf) -> - FileName = emqx_override_conf_name(), - OldConf = read_old_config(FileName), - %% We don't save the overall config to file, but only the sub configs - %% under RootKeys - write_new_config(FileName, - maps:merge(OldConf, maps:with(RootKeys, Conf))). - -write_new_config(FileName, Conf) -> - case file:write_file(FileName, jsx:prettify(jsx:encode(Conf))) of - ok -> ok; - {error, Reason} -> - logger:error("write to ~s failed, ~p", [FileName, Reason]), - {error, Reason} +call_post_config_update(Handlers, OldConf, NewConf, UpdateReq) -> + HandlerName = maps:get(?MOD, Handlers, undefined), + case erlang:function_exported(HandlerName, post_config_update, 3) of + true -> HandlerName:post_config_update(UpdateReq, NewConf, OldConf); + false -> ok end. -read_old_config(FileName) -> - case file:read_file(FileName) of - {ok, Text} -> - try jsx:decode(Text, [{return_maps, true}]) of - Conf when is_map(Conf) -> Conf; - _ -> #{} - catch _Err : _Reason -> - #{} - end; - _ -> #{} - end. +%% The default callback of config handlers +%% the behaviour is overwriting the old config if: +%% 1. the old config is undefined +%% 2. either the old or the new config is not of map type +%% the behaviour is merging the new the config to the old config if they are maps. +merge_to_old_config(UpdateReq, RawConf) when is_map(UpdateReq), is_map(RawConf) -> + maps:merge(RawConf, UpdateReq); +merge_to_old_config(UpdateReq, _RawConf) -> + UpdateReq. -emqx_conf_name() -> - filename:join([etc_dir(), "emqx.conf"]). +update_override_config(RawConf) -> + OldConf = emqx_config:read_override_conf(), + maps:merge(OldConf, RawConf). -emqx_override_conf_name() -> - filename:join([emqx:get_env(data_dir), "emqx_override.conf"]). +bin_path(ConfKeyPath) -> [bin(Key) || Key <- ConfKeyPath]. -etc_dir() -> - emqx:get_env(etc_dir). - -to_richmap(Map) -> - {ok, RichMap} = hocon:binary(jsx:encode(Map), #{format => richmap}), - RichMap. - -to_plainmap(RichMap) -> - hocon_schema:richmap_to_map(RichMap). - -bin(A) when is_atom(A) -> list_to_binary(atom_to_list(A)); -bin(B) when is_binary(B) -> B; -bin(S) when is_list(S) -> list_to_binary(S). +bin(A) when is_atom(A) -> atom_to_binary(A, utf8); +bin(B) when is_binary(B) -> B. diff --git a/apps/emqx/src/emqx_congestion.erl b/apps/emqx/src/emqx_congestion.erl index 4ec20034d..f0db5415f 100644 --- a/apps/emqx/src/emqx_congestion.erl +++ b/apps/emqx/src/emqx_congestion.erl @@ -55,8 +55,8 @@ cancel_alarms(Socket, Transport, Channel) -> end, ?ALL_ALARM_REASONS). is_alarm_enabled(Channel) -> - emqx_zone:get_env(emqx_channel:info(zone, Channel), - conn_congestion_alarm_enabled, false). + Zone = emqx_channel:info(zone, Channel), + emqx_config:get_zone_conf(Zone, [conn_congestion, enable_alarm]). alarm_congestion(Socket, Transport, Channel, Reason) -> case has_alarm_sent(Reason) of @@ -68,8 +68,8 @@ alarm_congestion(Socket, Transport, Channel, Reason) -> cancel_alarm_congestion(Socket, Transport, Channel, Reason) -> Zone = emqx_channel:info(zone, Channel), - WontClearIn = emqx_zone:get_env(Zone, conn_congestion_min_alarm_sustain_duration, - ?WONT_CLEAR_IN), + WontClearIn = emqx_config:get_zone_conf(Zone, [conn_congestion, + min_alarm_sustain_duration]), case has_alarm_sent(Reason) andalso long_time_since_last_alarm(Reason, WontClearIn) of true -> do_cancel_alarm_congestion(Socket, Transport, Channel, Reason); false -> ok diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index ab91c02b4..21ba5231e 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -83,8 +83,6 @@ sockname :: emqx_types:peername(), %% Sock State sockstate :: emqx_types:sockstate(), - %% The {active, N} option - active_n :: pos_integer(), %% Limiter limiter :: maybe(emqx_limiter:limiter()), %% Limit Timer @@ -102,13 +100,17 @@ %% Idle Timeout idle_timeout :: integer(), %% Idle Timer - idle_timer :: maybe(reference()) + idle_timer :: maybe(reference()), + %% Zone name + zone :: atom(), + %% Listener Name + listener :: atom() }). -type(state() :: #state{}). -define(ACTIVE_N, 100). --define(INFO_KEYS, [socktype, peername, sockname, sockstate, active_n]). +-define(INFO_KEYS, [socktype, peername, sockname, sockstate]). -define(CONN_STATS, [recv_pkt, recv_msg, send_pkt, send_msg]). -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). @@ -134,7 +136,7 @@ , system_code_change/4 ]}). --spec(start_link(esockd:transport(), esockd:socket(), proplists:proplist()) +-spec(start_link(esockd:transport(), esockd:socket(), emqx_channel:opts()) -> {ok, pid()}). start_link(Transport, Socket, Options) -> Args = [self(), Transport, Socket, Options], @@ -165,8 +167,6 @@ info(sockname, #state{sockname = Sockname}) -> Sockname; info(sockstate, #state{sockstate = SockSt}) -> SockSt; -info(active_n, #state{active_n = ActiveN}) -> - ActiveN; info(stats_timer, #state{stats_timer = StatsTimer}) -> StatsTimer; info(limit_timer, #state{limit_timer = LimitTimer}) -> @@ -243,7 +243,7 @@ init(Parent, Transport, RawSocket, Options) -> exit_on_sock_error(Reason) end. -init_state(Transport, Socket, Options) -> +init_state(Transport, Socket, #{zone := Zone, listener := Listener} = Opts) -> {ok, Peername} = Transport:ensure_ok_or_exit(peername, [Socket]), {ok, Sockname} = Transport:ensure_ok_or_exit(sockname, [Socket]), Peercert = Transport:ensure_ok_or_exit(peercert, [Socket]), @@ -253,26 +253,29 @@ init_state(Transport, Socket, Options) -> peercert => Peercert, conn_mod => ?MODULE }, - Zone = proplists:get_value(zone, Options), - ActiveN = proplists:get_value(active_n, Options, ?ACTIVE_N), - PubLimit = emqx_zone:publish_limit(Zone), - BytesIn = proplists:get_value(rate_limit, Options), - RateLimit = emqx_zone:ratelimit(Zone), - Limiter = emqx_limiter:init(Zone, PubLimit, BytesIn, RateLimit), - FrameOpts = emqx_zone:mqtt_frame_options(Zone), + Limiter = emqx_limiter:init(Zone, undefined, undefined, []), + FrameOpts = #{ + strict_mode => emqx_config:get_zone_conf(Zone, [mqtt, strict_mode]), + max_size => emqx_config:get_zone_conf(Zone, [mqtt, max_packet_size]) + }, ParseState = emqx_frame:initial_parse_state(FrameOpts), Serialize = emqx_frame:serialize_opts(), - Channel = emqx_channel:init(ConnInfo, Options), - GcState = emqx_zone:init_gc_state(Zone), - StatsTimer = emqx_zone:stats_timer(Zone), - IdleTimeout = emqx_zone:idle_timeout(Zone), + Channel = emqx_channel:init(ConnInfo, Opts), + GcState = case emqx_config:get_zone_conf(Zone, [force_gc]) of + #{enable := false} -> undefined; + GcPolicy -> emqx_gc:init(GcPolicy) + end, + StatsTimer = case emqx_config:get_zone_conf(Zone, [stats, enable]) of + true -> undefined; + false -> disabled + end, + IdleTimeout = emqx_channel:get_mqtt_conf(Zone, idle_timeout), IdleTimer = start_timer(IdleTimeout, idle_timeout), #state{transport = Transport, socket = Socket, peername = Peername, sockname = Sockname, sockstate = idle, - active_n = ActiveN, limiter = Limiter, parse_state = ParseState, serialize = Serialize, @@ -280,7 +283,9 @@ init_state(Transport, Socket, Options) -> gc_state = GcState, stats_timer = StatsTimer, idle_timeout = IdleTimeout, - idle_timer = IdleTimer + idle_timer = IdleTimer, + zone = Zone, + listener = Listener }. run_loop(Parent, State = #state{transport = Transport, @@ -288,8 +293,9 @@ run_loop(Parent, State = #state{transport = Transport, peername = Peername, channel = Channel}) -> emqx_logger:set_metadata_peername(esockd:format(Peername)), - emqx_misc:tune_heap_size(emqx_zone:oom_policy( - emqx_channel:info(zone, Channel))), + ShutdownPolicy = emqx_config:get_zone_conf(emqx_channel:info(zone, Channel), + [force_shutdown]), + emqx_misc:tune_heap_size(ShutdownPolicy), case activate_socket(State) of {ok, NState} -> hibernate(Parent, NState); {error, Reason} -> @@ -416,6 +422,13 @@ handle_msg({Inet, _Sock, Data}, State) when Inet == tcp; Inet == ssl -> ok = emqx_metrics:inc('bytes.received', Oct), parse_incoming(Data, State); +handle_msg({quic, Data, _Sock, _, _, _}, State) -> + ?LOG(debug, "RECV ~0p", [Data]), + Oct = iolist_size(Data), + inc_counter(incoming_bytes, Oct), + ok = emqx_metrics:inc('bytes.received', Oct), + parse_incoming(Data, State); + handle_msg({incoming, Packet = ?CONNECT_PACKET(ConnPkt)}, State = #state{idle_timer = IdleTimer}) -> ok = emqx_misc:cancel_timer(IdleTimer), @@ -440,7 +453,7 @@ handle_msg({Closed, _Sock}, State) handle_info({sock_closed, Closed}, close_socket(State)); handle_msg({Passive, _Sock}, State) - when Passive == tcp_passive; Passive == ssl_passive -> + when Passive == tcp_passive; Passive == ssl_passive; Passive =:= quic_passive -> %% In Stats Pubs = emqx_pd:reset_counter(incoming_pubs), Bytes = emqx_pd:reset_counter(incoming_bytes), @@ -451,14 +464,15 @@ handle_msg({Passive, _Sock}, State) NState1 = check_oom(run_gc(InStats, NState)), handle_info(activate_socket, NState1); -handle_msg(Deliver = {deliver, _Topic, _Msg}, - #state{active_n = ActiveN} = State) -> +handle_msg(Deliver = {deliver, _Topic, _Msg}, #state{zone = Zone, + listener = Listener} = State) -> + ActiveN = get_active_n(Zone, Listener), Delivers = [Deliver|emqx_misc:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); %% Something sent -handle_msg({inet_reply, _Sock, ok}, State = #state{active_n = ActiveN}) -> - case emqx_pd:get_counter(outgoing_pubs) > ActiveN of +handle_msg({inet_reply, _Sock, ok}, State = #state{zone = Zone, listener = Listener}) -> + case emqx_pd:get_counter(outgoing_pubs) > get_active_n(Zone, Listener) of true -> Pubs = emqx_pd:reset_counter(outgoing_pubs), Bytes = emqx_pd:reset_counter(outgoing_bytes), @@ -731,6 +745,15 @@ handle_info({sock_error, Reason}, State) -> end, handle_info({sock_closed, Reason}, close_socket(State)); +handle_info({quic, peer_send_shutdown, _Stream}, State) -> + handle_info({sock_closed, force}, close_socket(State)); + +handle_info({quic, closed, _Channel, ReasonFlag}, State) -> + handle_info({sock_closed, ReasonFlag}, State); + +handle_info({quic, closed, _Stream}, State) -> + handle_info({sock_closed, force}, State); + handle_info(Info, State) -> with_channel(handle_info, [Info], State). @@ -778,15 +801,14 @@ run_gc(Stats, State = #state{gc_state = GcSt}) -> end. check_oom(State = #state{channel = Channel}) -> - Zone = emqx_channel:info(zone, Channel), - OomPolicy = emqx_zone:oom_policy(Zone), - ?tp(debug, check_oom, #{policy => OomPolicy}), - case ?ENABLED(OomPolicy) andalso emqx_misc:check_oom(OomPolicy) of + ShutdownPolicy = emqx_config:get_zone_conf( + emqx_channel:info(zone, Channel), [force_shutdown]), + ?tp(debug, check_oom, #{policy => ShutdownPolicy}), + case emqx_misc:check_oom(ShutdownPolicy) of {shutdown, Reason} -> %% triggers terminate/2 callback immediately erlang:exit({shutdown, Reason}); - _Other -> - ok + _ -> ok end, State. @@ -798,10 +820,10 @@ activate_socket(State = #state{sockstate = closed}) -> {ok, State}; activate_socket(State = #state{sockstate = blocked}) -> {ok, State}; -activate_socket(State = #state{transport = Transport, - socket = Socket, - active_n = N}) -> - case Transport:setopts(Socket, [{active, N}]) of +activate_socket(State = #state{transport = Transport, socket = Socket, + zone = Zone, listener = Listener}) -> + ActiveN = get_active_n(Zone, Listener), + case Transport:setopts(Socket, [{active, ActiveN}]) of ok -> {ok, State#state{sockstate = running}}; Error -> Error end. @@ -882,3 +904,9 @@ get_state(Pid) -> State = sys:get_state(Pid), maps:from_list(lists:zip(record_info(fields, state), tl(tuple_to_list(State)))). + +get_active_n(Zone, Listener) -> + case emqx_config:get([zones, Zone, listeners, Listener, type]) of + quic -> 100; + _ -> emqx_config:get_listener_conf(Zone, Listener, [tcp, active_n]) + end. diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl index a0eab9c18..0b852d88d 100644 --- a/apps/emqx/src/emqx_flapping.erl +++ b/apps/emqx/src/emqx_flapping.erl @@ -45,16 +45,16 @@ -define(FLAPPING_DURATION, 60000). -define(FLAPPING_BANNED_INTERVAL, 300000). -define(DEFAULT_DETECT_POLICY, - #{threshold => ?FLAPPING_THRESHOLD, - duration => ?FLAPPING_DURATION, - banned_interval => ?FLAPPING_BANNED_INTERVAL + #{max_count => ?FLAPPING_THRESHOLD, + window_time => ?FLAPPING_DURATION, + ban_time => ?FLAPPING_BANNED_INTERVAL }). -record(flapping, { clientid :: emqx_types:clientid(), peerhost :: emqx_types:peerhost(), started_at :: pos_integer(), - detect_cnt :: pos_integer() + detect_cnt :: integer() }). -opaque(flapping() :: #flapping{}). @@ -69,33 +69,28 @@ stop() -> gen_server:stop(?MODULE). %% @doc Detect flapping when a MQTT client disconnected. -spec(detect(emqx_types:clientinfo()) -> boolean()). -detect(Client) -> detect(Client, get_policy()). - -detect(#{clientid := ClientId, peerhost := PeerHost}, Policy = #{threshold := Threshold}) -> - try ets:update_counter(?FLAPPING_TAB, ClientId, {#flapping.detect_cnt, 1}) of +detect(#{clientid := ClientId, peerhost := PeerHost, zone := Zone}) -> + Policy = #{max_count := Threshold} = get_policy(Zone), + %% The initial flapping record sets the detect_cnt to 0. + InitVal = #flapping{ + clientid = ClientId, + peerhost = PeerHost, + started_at = erlang:system_time(millisecond), + detect_cnt = 0 + }, + case ets:update_counter(?FLAPPING_TAB, ClientId, {#flapping.detect_cnt, 1}, InitVal) of Cnt when Cnt < Threshold -> false; - _Cnt -> case ets:take(?FLAPPING_TAB, ClientId) of - [Flapping] -> - ok = gen_server:cast(?MODULE, {detected, Flapping, Policy}), - true; - [] -> false - end - catch - error:badarg -> - %% Create a flapping record. - Flapping = #flapping{clientid = ClientId, - peerhost = PeerHost, - started_at = erlang:system_time(millisecond), - detect_cnt = 1 - }, - true = ets:insert(?FLAPPING_TAB, Flapping), - false + _Cnt -> + case ets:take(?FLAPPING_TAB, ClientId) of + [Flapping] -> + ok = gen_server:cast(?MODULE, {detected, Flapping, Policy}), + true; + [] -> false + end end. --compile({inline, [get_policy/0, now_diff/1]}). - -get_policy() -> - emqx:get_env(flapping_detect_policy, ?DEFAULT_DETECT_POLICY). +get_policy(Zone) -> + emqx_config:get_zone_conf(Zone, [flapping_detect]). now_diff(TS) -> erlang:system_time(millisecond) - TS. @@ -105,11 +100,12 @@ now_diff(TS) -> erlang:system_time(millisecond) - TS. init([]) -> ok = emqx_tables:new(?FLAPPING_TAB, [public, set, - {keypos, 2}, + {keypos, #flapping.clientid}, {read_concurrency, true}, {write_concurrency, true} ]), - {ok, ensure_timer(#{}), hibernate}. + start_timers(), + {ok, #{}, hibernate}. handle_call(Req, _From, State) -> ?LOG(error, "Unexpected call: ~p", [Req]), @@ -119,17 +115,17 @@ handle_cast({detected, #flapping{clientid = ClientId, peerhost = PeerHost, started_at = StartedAt, detect_cnt = DetectCnt}, - #{duration := Duration, banned_interval := Interval}}, State) -> - case now_diff(StartedAt) < Duration of + #{window_time := WindTime, ban_time := Interval}}, State) -> + case now_diff(StartedAt) < WindTime of true -> %% Flapping happened:( ?LOG(error, "Flapping detected: ~s(~s) disconnected ~w times in ~wms", - [ClientId, inet:ntoa(PeerHost), DetectCnt, Duration]), + [ClientId, inet:ntoa(PeerHost), DetectCnt, WindTime]), Now = erlang:system_time(second), Banned = #banned{who = {clientid, ClientId}, by = <<"flapping detector">>, reason = <<"flapping is detected">>, at = Now, - until = Now + Interval}, + until = Now + (Interval div 1000)}, emqx_banned:create(Banned); false -> ?LOG(warning, "~s(~s) disconnected ~w times in ~wms", @@ -141,11 +137,13 @@ handle_cast(Msg, State) -> ?LOG(error, "Unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({timeout, TRef, expired_detecting}, State = #{expired_timer := TRef}) -> - Timestamp = erlang:system_time(millisecond) - maps:get(duration, get_policy()), +handle_info({timeout, _TRef, {garbage_collect, Zone}}, State) -> + Timestamp = erlang:system_time(millisecond) + - maps:get(window_time, get_policy(Zone)), MatchSpec = [{{'_', '_', '_', '$1', '_'},[{'<', '$1', Timestamp}], [true]}], ets:select_delete(?FLAPPING_TAB, MatchSpec), - {noreply, ensure_timer(State), hibernate}; + start_timer(Zone), + {noreply, State, hibernate}; handle_info(Info, State) -> ?LOG(error, "Unexpected info: ~p", [Info]), @@ -157,7 +155,11 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. -ensure_timer(State) -> - Timeout = maps:get(duration, get_policy()), - TRef = emqx_misc:start_timer(Timeout, expired_detecting), - State#{expired_timer => TRef}. \ No newline at end of file +start_timer(Zone) -> + WindTime = maps:get(window_time, get_policy(Zone)), + emqx_misc:start_timer(WindTime, {garbage_collect, Zone}). + +start_timers() -> + lists:foreach(fun({Zone, _ZoneConf}) -> + start_timer(Zone) + end, maps:to_list(emqx_config:get([zones], #{}))). \ No newline at end of file diff --git a/apps/emqx/src/emqx_frame.erl b/apps/emqx/src/emqx_frame.erl index 37063c65f..082801bad 100644 --- a/apps/emqx/src/emqx_frame.erl +++ b/apps/emqx/src/emqx_frame.erl @@ -81,11 +81,7 @@ initial_parse_state() -> -spec(initial_parse_state(options()) -> {none, options()}). initial_parse_state(Options) when is_map(Options) -> - ?none(merge_opts(Options)). - -%% @pivate -merge_opts(Options) -> - maps:merge(?DEFAULT_OPTIONS, Options). + ?none(maps:merge(?DEFAULT_OPTIONS, Options)). %%-------------------------------------------------------------------- %% Parse MQTT Frame @@ -643,7 +639,7 @@ serialize_properties(Props) when is_map(Props) -> Bin = << <<(serialize_property(Prop, Val))/binary>> || {Prop, Val} <- maps:to_list(Props) >>, [serialize_variable_byte_integer(byte_size(Bin)), Bin]. -serialize_property(_, undefined) -> +serialize_property(_, Disabled) when Disabled =:= disabled; Disabled =:= undefined -> <<>>; serialize_property('Payload-Format-Indicator', Val) -> <<16#01, Val>>; diff --git a/apps/emqx/src/emqx_global_gc.erl b/apps/emqx/src/emqx_global_gc.erl index 51741ab1c..9449efe9a 100644 --- a/apps/emqx/src/emqx_global_gc.erl +++ b/apps/emqx/src/emqx_global_gc.erl @@ -85,9 +85,9 @@ code_change(_OldVsn, State, _Extra) -> %%-------------------------------------------------------------------- ensure_timer(State) -> - case emqx:get_env(global_gc_interval) of + case emqx_config:get([node, global_gc_interval]) of undefined -> State; - Interval -> TRef = emqx_misc:start_timer(timer:seconds(Interval), run), + Interval -> TRef = emqx_misc:start_timer(Interval, run), State#{timer := TRef} end. diff --git a/apps/emqx/src/emqx_hooks.erl b/apps/emqx/src/emqx_hooks.erl index 3709c64d3..eb2da4276 100644 --- a/apps/emqx/src/emqx_hooks.erl +++ b/apps/emqx/src/emqx_hooks.erl @@ -20,6 +20,7 @@ -include("logger.hrl"). -include("types.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). -logger_header("[Hooks]"). @@ -32,6 +33,8 @@ , add/3 , add/4 , put/2 + , put/3 + , put/4 , del/2 , run/2 , run_fold/3 @@ -130,11 +133,25 @@ add(HookPoint, Action, Filter, Priority) when is_integer(Priority) -> %% @doc Like add/2, it register a callback, discard 'already_exists' error. -spec(put(hookpoint(), action() | #callback{}) -> ok). -put(HookPoint, Callback) -> +put(HookPoint, Callback) when is_record(Callback, callback) -> case add(HookPoint, Callback) of ok -> ok; - {error, already_exists} -> ok - end. + {error, already_exists} -> + gen_server:call(?SERVER, {put, HookPoint, Callback}, infinity) + end; +put(HookPoint, Action) when is_function(Action); is_tuple(Action) -> + ?MODULE:put(HookPoint, #callback{action = Action, priority = 0}). + +-spec(put(hookpoint(), action(), filter() | integer() | list()) -> ok). +put(HookPoint, Action, {_M, _F, _A} = Filter) -> + ?MODULE:put(HookPoint, #callback{action = Action, filter = Filter, priority = 0}); +put(HookPoint, Action, Priority) when is_integer(Priority) -> + ?MODULE:put(HookPoint, #callback{action = Action, priority = Priority}). + +-spec(put(hookpoint(), action(), filter(), integer()) -> ok). +put(HookPoint, Action, Filter, Priority) when is_integer(Priority) -> + ?MODULE:put(HookPoint, #callback{action = Action, filter = Filter, priority = Priority}). + %% @doc Unregister a callback. -spec(del(hookpoint(), action() | {module(), atom()}) -> ok). @@ -215,15 +232,20 @@ init([]) -> ok = emqx_tables:new(?TAB, [{keypos, #hook.name}, {read_concurrency, true}]), {ok, #{}}. -handle_call({add, HookPoint, Callback = #callback{action = Action}}, _From, State) -> - Reply = case lists:keymember(Action, #callback.action, Callbacks = lookup(HookPoint)) of - true -> - {error, already_exists}; - false -> - insert_hook(HookPoint, add_callback(Callback, Callbacks)) +handle_call({add, HookPoint, Callback = #callback{action = {M, F, _}}}, _From, State) -> + Reply = case lists:any(fun (#callback{action = {M0, F0, _}}) -> + M0 =:= M andalso F0 =:= F + end, Callbacks = lookup(HookPoint)) of + true -> {error, already_exists}; + false -> insert_hook(HookPoint, add_callback(Callback, Callbacks)) end, {reply, Reply, State}; +handle_call({put, HookPoint, Callback = #callback{action = {M, F, _}}}, _From, State) -> + Callbacks = del_callback({M, F}, lookup(HookPoint)), + Reply = update_hook(HookPoint, add_callback(Callback, Callbacks)), + {reply, Reply, State}; + handle_call(Req, _From, State) -> ?LOG(error, "Unexpected call: ~p", [Req]), {reply, ignored, State}. @@ -257,6 +279,10 @@ code_change(_OldVsn, State, _Extra) -> insert_hook(HookPoint, Callbacks) -> ets:insert(?TAB, #hook{name = HookPoint, callbacks = Callbacks}), ok. +update_hook(HookPoint, Callbacks) -> + Ms = ets:fun2ms(fun ({hook, K, V}) when K =:= HookPoint -> {hook, K, Callbacks} end), + ets:select_replace(emqx_hooks, Ms), + ok. add_callback(C, Callbacks) -> add_callback(C, Callbacks, []). diff --git a/apps/emqx/src/emqx_kernel_sup.erl b/apps/emqx/src/emqx_kernel_sup.erl index 4e29431e2..5ca283481 100644 --- a/apps/emqx/src/emqx_kernel_sup.erl +++ b/apps/emqx/src/emqx_kernel_sup.erl @@ -27,14 +27,14 @@ start_link() -> init([]) -> {ok, {{one_for_one, 10, 100}, - [child_spec(emqx_global_gc, worker), - child_spec(emqx_pool_sup, supervisor), - child_spec(emqx_hooks, worker), - child_spec(emqx_stats, worker), - child_spec(emqx_metrics, worker), - child_spec(emqx_ctl, worker), - child_spec(emqx_zone, worker), - child_spec(emqx_config_handler, worker) + %% always start emqx_config_handler first to load the emqx.conf to emqx_config + [ child_spec(emqx_config_handler, worker) + , child_spec(emqx_global_gc, worker) + , child_spec(emqx_pool_sup, supervisor) + , child_spec(emqx_hooks, worker) + , child_spec(emqx_stats, worker) + , child_spec(emqx_metrics, worker) + , child_spec(emqx_ctl, worker) ]}}. child_spec(M, Type) -> diff --git a/apps/emqx/src/emqx_limiter.erl b/apps/emqx/src/emqx_limiter.erl index 181e5c6bf..b4cf745ff 100644 --- a/apps/emqx/src/emqx_limiter.erl +++ b/apps/emqx/src/emqx_limiter.erl @@ -27,7 +27,7 @@ -record(limiter, { %% Zone - zone :: emqx_zone:zone(), + zone :: atom(), %% Checkers checkers :: [checker()] }). @@ -35,7 +35,7 @@ -type(checker() :: #{ name := name() , capacity := non_neg_integer() , interval := non_neg_integer() - , consumer := esockd_rate_limit:bucket() | emqx_zone:zone() + , consumer := esockd_rate_limit:bucket() | atom() }). -type(name() :: conn_bytes_in @@ -59,7 +59,7 @@ %% APIs %%-------------------------------------------------------------------- --spec(init(emqx_zone:zone(), +-spec(init(atom(), maybe(esockd_rate_limit:config()), maybe(esockd_rate_limit:config()), policy()) -> maybe(limiter())). @@ -69,7 +69,7 @@ init(Zone, PubLimit, BytesIn, Specs) -> Filtered = maps:filter(fun(_, V) -> V /= undefined end, Merged), init(Zone, maps:to_list(Filtered)). --spec(init(emqx_zone:zone(), policy()) -> maybe(limiter())). +-spec(init(atom(), policy()) -> maybe(limiter())). init(_Zone, []) -> undefined; init(Zone, Specs) -> diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 1f3d1776b..8c7334cbe 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -20,97 +20,96 @@ -include("emqx_mqtt.hrl"). %% APIs --export([ start/0 - , ensure_all_started/0 +-export([ list/0 + , start/0 , restart/0 , stop/0 + , is_running/1 ]). -export([ start_listener/1 , start_listener/3 , stop_listener/1 + , stop_listener/3 , restart_listener/1 , restart_listener/3 ]). --export([ find_id_by_listen_on/1 - , find_by_listen_on/1 - , find_by_id/1 - , identifier/1 - , format_listen_on/1 - ]). +-spec(list() -> [{ListenerId :: atom(), ListenerConf :: map()}]). +list() -> + Zones = maps:to_list(emqx_config:get([zones], #{})), + lists:append([list(ZoneName, ZoneConf) || {ZoneName, ZoneConf} <- Zones]). --type(listener() :: #{ name := binary() - , proto := esockd:proto() - , listen_on := esockd:listen_on() - , opts := [esockd:option()] - }). +list(ZoneName, ZoneConf) -> + Listeners = maps:to_list(maps:get(listeners, ZoneConf, #{})), + [ + begin + ListenerId = listener_id(ZoneName, LName), + Running = is_running(ListenerId), + Conf = merge_zone_and_listener_confs(ZoneConf, LConf), + {ListenerId, maps:put(running, Running, Conf)} + end + || {LName, LConf} <- Listeners]. -%% @doc Find listener identifier by listen-on. -%% Return empty string (binary) if listener is not found in config. --spec(find_id_by_listen_on(esockd:listen_on()) -> binary() | false). -find_id_by_listen_on(ListenOn) -> - case find_by_listen_on(ListenOn) of - false -> false; - L -> identifier(L) +-spec is_running(ListenerId :: atom()) -> boolean() | {error, no_found}. +is_running(ListenerId) -> + Zones = maps:to_list(emqx_config:get([zones], #{})), + Listeners = lists:append( + [ + [{listener_id(ZoneName, LName),merge_zone_and_listener_confs(ZoneConf, LConf)} + || {LName, LConf} <- maps:to_list(maps:get(listeners, ZoneConf, #{}))] + || {ZoneName, ZoneConf} <- Zones]), + case proplists:get_value(ListenerId, Listeners, undefined) of + undefined -> + {error, no_found}; + Conf -> + is_running(ListenerId, Conf) end. -%% @doc Find listener by listen-on. -%% Return 'false' if not found. --spec(find_by_listen_on(esockd:listen_on()) -> listener() | false). -find_by_listen_on(ListenOn) -> - find_by_listen_on(ListenOn, emqx:get_env(listeners, [])). +is_running(ListenerId, #{type := tcp, bind := ListenOn})-> + try esockd:listener({ListenerId, ListenOn}) of + Pid when is_pid(Pid)-> + true + catch _:_ -> + false + end; -%% @doc Find listener by identifier. -%% Return 'false' if not found. --spec(find_by_id(string() | binary()) -> listener() | false). -find_by_id(Id) -> - find_by_id(iolist_to_binary(Id), emqx:get_env(listeners, [])). +is_running(ListenerId, #{type := ws})-> + try + Info = ranch:info(ListenerId), + proplists:get_value(status, Info) =:= running + catch _:_ -> + false + end; -%% @doc Return the ID of the given listener. --spec identifier(listener()) -> binary(). -identifier(#{proto := Proto, name := Name}) -> - identifier(Proto, Name). +is_running(_ListenerId, #{type := quic})-> +%% TODO: quic support + {error, no_found}. %% @doc Start all listeners. -spec(start() -> ok). start() -> - lists:foreach(fun start_listener/1, emqx:get_env(listeners, [])). + foreach_listeners(fun start_listener/3). -%% @doc Ensure all configured listeners are started. -%% Raise exception if any of them failed to start. --spec(ensure_all_started() -> ok). -ensure_all_started() -> - ensure_all_started(emqx:get_env(listeners, []), []). +-spec start_listener(atom()) -> ok | {error, term()}. +start_listener(ListenerId) -> + apply_on_listener(ListenerId, fun start_listener/3). -ensure_all_started([], []) -> ok; -ensure_all_started([], Failed) -> error(Failed); -ensure_all_started([L | Rest], Results) -> - #{proto := Proto, listen_on := ListenOn, opts := Options} = L, - NewResults = - case start_listener(Proto, ListenOn, Options) of - {ok, _Pid} -> - Results; - {error, {already_started, _Pid}} -> - Results; - {error, Reason} -> - [{identifier(L), Reason} | Results] - end, - ensure_all_started(Rest, NewResults). - -%% @doc Format address:port for logging. --spec(format_listen_on(esockd:listen_on()) -> [char()]). -format_listen_on(ListenOn) -> format(ListenOn). - --spec(start_listener(listener()) -> ok). -start_listener(#{proto := Proto, name := Name, listen_on := ListenOn, opts := Options}) -> - ID = identifier(Proto, Name), - case start_listener(Proto, ListenOn, Options) of +-spec start_listener(atom(), atom(), map()) -> ok | {error, term()}. +start_listener(ZoneName, ListenerName, #{type := Type, bind := Bind} = Conf) -> + case do_start_listener(ZoneName, ListenerName, Conf) of + {ok, {skipped, Reason}} when Reason =:= listener_disabled; + Reason =:= quic_app_missing -> + console_print("- Skip - starting ~s listener ~s on ~s ~n due to ~p", + [Type, listener_id(ZoneName, ListenerName), format(Bind), Reason]); {ok, _} -> - console_print("Start ~s listener on ~s successfully.~n", [ID, format(ListenOn)]); + console_print("Start ~s listener ~s on ~s successfully.~n", + [Type, listener_id(ZoneName, ListenerName), format(Bind)]); + {error, {already_started, Pid}} -> + {error, {already_started, Pid}}; {error, Reason} -> - io:format(standard_error, "Failed to start mqtt listener ~s on ~s: ~0p~n", - [ID, format(ListenOn), Reason]), + io:format(standard_error, "Failed to start ~s listener ~s on ~s: ~0p~n", + [Type, listener_id(ZoneName, ListenerName), format(Bind), Reason]), error(Reason) end. @@ -122,124 +121,136 @@ console_print(_Fmt, _Args) -> ok. -endif. %% Start MQTT/TCP listener --spec(start_listener(esockd:proto(), esockd:listen_on(), [esockd:option()]) - -> {ok, pid()} | {error, term()}). -start_listener(tcp, ListenOn, Options) -> - start_mqtt_listener('mqtt:tcp', ListenOn, Options); - -%% Start MQTT/TLS listener -start_listener(Proto, ListenOn, Options) when Proto == ssl; Proto == tls -> - start_mqtt_listener('mqtt:ssl', ListenOn, Options); +-spec(do_start_listener(atom(), atom(), map()) + -> {ok, pid() | {skipped, atom()}} | {error, term()}). +do_start_listener(_ZoneName, _ListenerName, #{enabled := false}) -> + {ok, {skipped, listener_disabled}}; +do_start_listener(ZoneName, ListenerName, #{type := tcp, bind := ListenOn} = Opts) -> + esockd:open(listener_id(ZoneName, ListenerName), ListenOn, merge_default(esockd_opts(Opts)), + {emqx_connection, start_link, + [#{zone => ZoneName, listener => ListenerName}]}); %% Start MQTT/WS listener -start_listener(Proto, ListenOn, Options) when Proto == http; Proto == ws -> - start_http_listener(fun cowboy:start_clear/3, 'mqtt:ws', ListenOn, - ranch_opts(Options), ws_opts(Options)); +do_start_listener(ZoneName, ListenerName, #{type := ws, bind := ListenOn} = Opts) -> + Id = listener_id(ZoneName, ListenerName), + RanchOpts = ranch_opts(ListenOn, Opts), + WsOpts = ws_opts(ZoneName, ListenerName, Opts), + case is_ssl(Opts) of + false -> + cowboy:start_clear(Id, RanchOpts, WsOpts); + true -> + cowboy:start_tls(Id, RanchOpts, WsOpts) + end; -%% Start MQTT/WSS listener -start_listener(Proto, ListenOn, Options) when Proto == https; Proto == wss -> - start_http_listener(fun cowboy:start_tls/3, 'mqtt:wss', ListenOn, - ranch_opts(Options), ws_opts(Options)). - -replace(Opts, Key, Value) -> [{Key, Value} | proplists:delete(Key, Opts)]. - -drop_tls13_for_old_otp(Options) -> - case proplists:get_value(ssl_options, Options) of - undefined -> Options; - SslOpts -> - SslOpts1 = emqx_tls_lib:drop_tls13_for_old_otp(SslOpts), - replace(Options, ssl_options, SslOpts1) +%% Start MQTT/QUIC listener +do_start_listener(ZoneName, ListenerName, #{type := quic, bind := ListenOn} = Opts) -> + case [ A || {quicer, _, _} = A<-application:which_applications() ] of + [_] -> + %% @fixme unsure why we need reopen lib and reopen config. + quicer_nif:open_lib(), + quicer_nif:reg_open(), + DefAcceptors = erlang:system_info(schedulers_online) * 8, + ListenOpts = [ {cert, maps:get(certfile, Opts)} + , {key, maps:get(keyfile, Opts)} + , {alpn, ["mqtt"]} + , {conn_acceptors, maps:get(acceptors, Opts, DefAcceptors)} + , {idle_timeout_ms, emqx_config:get_zone_conf(ZoneName, [mqtt, idle_timeout])} + ], + ConnectionOpts = #{conn_callback => emqx_quic_connection + , peer_unidi_stream_count => 1 + , peer_bidi_stream_count => 10 + , zone => ZoneName + , listener => ListenerName + }, + StreamOpts = [], + quicer:start_listener(listener_id(ZoneName, ListenerName), + port(ListenOn), {ListenOpts, ConnectionOpts, StreamOpts}); + [] -> + {ok, {skipped, quic_app_missing}} end. -start_mqtt_listener(Name, ListenOn, Options0) -> - Options = drop_tls13_for_old_otp(Options0), - SockOpts = esockd:parse_opt(Options), - esockd:open(Name, ListenOn, merge_default(SockOpts), - {emqx_connection, start_link, [Options -- SockOpts]}). +esockd_opts(Opts0) -> + Opts1 = maps:with([acceptors, max_connections, proxy_protocol, proxy_protocol_timeout], Opts0), + Opts2 = case emqx_map_lib:deep_get([rate_limit, max_conn_rate], Opts0) of + infinity -> Opts1; + Rate -> Opts1#{max_conn_rate => Rate} + end, + Opts3 = Opts2#{access_rules => esockd_access_rules(maps:get(access_rules, Opts0, []))}, + maps:to_list(case is_ssl(Opts0) of + false -> + Opts3#{tcp_options => tcp_opts(Opts0)}; + true -> + Opts3#{ssl_options => ssl_opts(Opts0), tcp_options => tcp_opts(Opts0)} + end). -start_http_listener(Start, Name, ListenOn, RanchOpts, ProtoOpts) -> - Start(ws_name(Name, ListenOn), with_port(ListenOn, RanchOpts), ProtoOpts). - -mqtt_path(Options) -> - proplists:get_value(mqtt_path, Options, "/mqtt"). - -ws_opts(Options) -> - WsPaths = [{mqtt_path(Options), emqx_ws_connection, Options}], +ws_opts(ZoneName, ListenerName, Opts) -> + WsPaths = [{maps:get(mqtt_path, Opts, "/mqtt"), emqx_ws_connection, + #{zone => ZoneName, listener => ListenerName}}], Dispatch = cowboy_router:compile([{'_', WsPaths}]), - ProxyProto = proplists:get_value(proxy_protocol, Options, false), + ProxyProto = maps:get(proxy_protocol, Opts, false), #{env => #{dispatch => Dispatch}, proxy_header => ProxyProto}. -ranch_opts(Options0) -> - Options = drop_tls13_for_old_otp(Options0), - NumAcceptors = proplists:get_value(acceptors, Options, 4), - MaxConnections = proplists:get_value(max_connections, Options, 1024), - TcpOptions = proplists:get_value(tcp_options, Options, []), - RanchOpts = #{num_acceptors => NumAcceptors, - max_connections => MaxConnections, - socket_opts => TcpOptions}, - case proplists:get_value(ssl_options, Options) of - undefined -> RanchOpts; - SslOptions -> RanchOpts#{socket_opts => TcpOptions ++ SslOptions} - end. +ranch_opts(ListenOn, Opts) -> + NumAcceptors = maps:get(acceptors, Opts, 4), + MaxConnections = maps:get(max_connections, Opts, 1024), + SocketOpts = case is_ssl(Opts) of + true -> tcp_opts(Opts) ++ proplists:delete(handshake_timeout, ssl_opts(Opts)); + false -> tcp_opts(Opts) + end, + #{num_acceptors => NumAcceptors, + max_connections => MaxConnections, + handshake_timeout => maps:get(handshake_timeout, Opts, 15000), + socket_opts => ip_port(ListenOn) ++ + %% cowboy don't allow us to set 'reuseaddr' + proplists:delete(reuseaddr, SocketOpts)}. -with_port(Port, Opts = #{socket_opts := SocketOption}) when is_integer(Port) -> - Opts#{socket_opts => [{port, Port}| SocketOption]}; -with_port({Addr, Port}, Opts = #{socket_opts := SocketOption}) -> - Opts#{socket_opts => [{ip, Addr}, {port, Port}| SocketOption]}. +ip_port(Port) when is_integer(Port) -> + [{port, Port}]; +ip_port({Addr, Port}) -> + [{ip, Addr}, {port, Port}]. + +port(Port) when is_integer(Port) -> Port; +port({_Addr, Port}) when is_integer(Port) -> Port. + +esockd_access_rules(StrRules) -> + Access = fun(S) -> + [A, CIDR] = string:tokens(S, " "), + {list_to_atom(A), case CIDR of "all" -> all; _ -> CIDR end} + end, + [Access(R) || R <- StrRules]. %% @doc Restart all listeners -spec(restart() -> ok). restart() -> - lists:foreach(fun restart_listener/1, emqx:get_env(listeners, [])). + foreach_listeners(fun restart_listener/3). --spec(restart_listener(listener() | string() | binary()) -> ok | {error, any()}). -restart_listener(#{proto := Proto, listen_on := ListenOn, opts := Options}) -> - restart_listener(Proto, ListenOn, Options); -restart_listener(Identifier) -> - case emqx_listeners:find_by_id(Identifier) of - false -> {error, {no_such_listener, Identifier}}; - Listener -> restart_listener(Listener) +-spec(restart_listener(atom()) -> ok | {error, term()}). +restart_listener(ListenerId) -> + apply_on_listener(ListenerId, fun restart_listener/3). + +-spec(restart_listener(atom(), atom(), map()) -> ok | {error, term()}). +restart_listener(ZoneName, ListenerName, Conf) -> + case stop_listener(ZoneName, ListenerName, Conf) of + ok -> start_listener(ZoneName, ListenerName, Conf); + Error -> Error end. --spec(restart_listener(esockd:proto(), esockd:listen_on(), [esockd:option()]) -> - ok | {error, any()}). -restart_listener(tcp, ListenOn, _Options) -> - esockd:reopen('mqtt:tcp', ListenOn); -restart_listener(Proto, ListenOn, _Options) when Proto == ssl; Proto == tls -> - esockd:reopen('mqtt:ssl', ListenOn); -restart_listener(Proto, ListenOn, Options) when Proto == http; Proto == ws -> - _ = cowboy:stop_listener(ws_name('mqtt:ws', ListenOn)), - ok(start_listener(Proto, ListenOn, Options)); -restart_listener(Proto, ListenOn, Options) when Proto == https; Proto == wss -> - _ = cowboy:stop_listener(ws_name('mqtt:wss', ListenOn)), - ok(start_listener(Proto, ListenOn, Options)); -restart_listener(Proto, ListenOn, _Opts) -> - esockd:reopen(Proto, ListenOn). - -ok({ok, _}) -> ok; -ok(Other) -> Other. - %% @doc Stop all listeners. -spec(stop() -> ok). stop() -> - lists:foreach(fun stop_listener/1, emqx:get_env(listeners, [])). + foreach_listeners(fun stop_listener/3). --spec(stop_listener(listener()) -> ok | {error, term()}). -stop_listener(#{proto := Proto, listen_on := ListenOn, opts := Opts}) -> - stop_listener(Proto, ListenOn, Opts). +-spec(stop_listener(atom()) -> ok | {error, term()}). +stop_listener(ListenerId) -> + apply_on_listener(ListenerId, fun stop_listener/3). --spec(stop_listener(esockd:proto(), esockd:listen_on(), [esockd:option()]) - -> ok | {error, term()}). -stop_listener(tcp, ListenOn, _Opts) -> - esockd:close('mqtt:tcp', ListenOn); -stop_listener(Proto, ListenOn, _Opts) when Proto == ssl; Proto == tls -> - esockd:close('mqtt:ssl', ListenOn); -stop_listener(Proto, ListenOn, _Opts) when Proto == http; Proto == ws -> - cowboy:stop_listener(ws_name('mqtt:ws', ListenOn)); -stop_listener(Proto, ListenOn, _Opts) when Proto == https; Proto == wss -> - cowboy:stop_listener(ws_name('mqtt:wss', ListenOn)); -stop_listener(Proto, ListenOn, _Opts) -> - esockd:close(Proto, ListenOn). +-spec(stop_listener(atom(), atom(), map()) -> ok | {error, term()}). +stop_listener(ZoneName, ListenerName, #{type := tcp, bind := ListenOn}) -> + esockd:close(listener_id(ZoneName, ListenerName), ListenOn); +stop_listener(ZoneName, ListenerName, #{type := ws}) -> + cowboy:stop_listener(listener_id(ZoneName, ListenerName)); +stop_listener(ZoneName, ListenerName, #{type := quic}) -> + quicer:stop_listener(listener_id(ZoneName, ListenerName)). merge_default(Options) -> case lists:keytake(tcp_options, 1, Options) of @@ -256,23 +267,46 @@ format({Addr, Port}) when is_list(Addr) -> format({Addr, Port}) when is_tuple(Addr) -> io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). -ws_name(Name, {_Addr, Port}) -> - ws_name(Name, Port); -ws_name(Name, Port) -> - list_to_atom(lists:concat([Name, ":", Port])). +listener_id(ZoneName, ListenerName) -> + list_to_atom(lists:append([atom_to_list(ZoneName), ":", atom_to_list(ListenerName)])). -identifier(Proto, Name) when is_atom(Proto) -> - identifier(atom_to_list(Proto), Name); -identifier(Proto, Name) -> - iolist_to_binary(["mqtt", ":", Proto, ":", Name]). - -find_by_listen_on(_ListenOn, []) -> false; -find_by_listen_on(ListenOn, [#{listen_on := ListenOn} = L | _]) -> L; -find_by_listen_on(ListenOn, [_ | Rest]) -> find_by_listen_on(ListenOn, Rest). - -find_by_id(_Id, []) -> false; -find_by_id(Id, [L | Rest]) -> - case identifier(L) =:= Id of - true -> L; - false -> find_by_id(Id, Rest) +decode_listener_id(Id) -> + case string:split(atom_to_list(Id), ":", leading) of + [Zone, Listen] -> {list_to_atom(Zone), list_to_atom(Listen)}; + _ -> error({invalid_listener_id, Id}) + end. + +ssl_opts(Opts) -> + maps:to_list( + emqx_tls_lib:drop_tls13_for_old_otp( + maps:without([enable], + maps:get(ssl, Opts, #{})))). + +tcp_opts(Opts) -> + maps:to_list( + maps:without([active_n], + maps:get(tcp, Opts, #{}))). + +is_ssl(Opts) -> + emqx_map_lib:deep_get([ssl, enable], Opts, false). + +foreach_listeners(Do) -> + lists:foreach(fun({ZoneName, ZoneConf}) -> + lists:foreach(fun({LName, LConf}) -> + Do(ZoneName, LName, merge_zone_and_listener_confs(ZoneConf, LConf)) + end, maps:to_list(maps:get(listeners, ZoneConf, #{}))) + end, maps:to_list(emqx_config:get([zones], #{}))). + +%% merge the configs in zone and listeners in a manner that +%% all config entries in the listener are prior to the ones in the zone. +merge_zone_and_listener_confs(ZoneConf, ListenerConf) -> + ConfsInZonesOnly = [listeners, overall_max_connections], + BaseConf = maps:without(ConfsInZonesOnly, ZoneConf), + emqx_map_lib:deep_merge(BaseConf, ListenerConf). + +apply_on_listener(ListenerId, Do) -> + {ZoneName, ListenerName} = decode_listener_id(ListenerId), + case emqx_config:find_listener_conf(ZoneName, ListenerName, []) of + {not_found, _, _} -> error({listener_config_not_found, ZoneName, ListenerName}); + {ok, Conf} -> Do(ZoneName, ListenerName, Conf) end. diff --git a/apps/emqx/src/emqx_map_lib.erl b/apps/emqx/src/emqx_map_lib.erl new file mode 100644 index 000000000..d720e771e --- /dev/null +++ b/apps/emqx/src/emqx_map_lib.erl @@ -0,0 +1,117 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_map_lib). + +-export([ deep_get/2 + , deep_get/3 + , deep_find/2 + , deep_put/3 + , deep_remove/2 + , deep_merge/2 + , safe_atom_key_map/1 + , unsafe_atom_key_map/1 + ]). + +-export_type([config_key/0, config_key_path/0]). +-type config_key() :: atom() | binary(). +-type config_key_path() :: [config_key()]. + +%%----------------------------------------------------------------- +-spec deep_get(config_key_path(), map()) -> term(). +deep_get(ConfKeyPath, Map) -> + Ref = make_ref(), + Res = deep_get(ConfKeyPath, Map, Ref), + case Res =:= Ref of + true -> error({config_not_found, ConfKeyPath}); + false -> Res + end. + +-spec deep_get(config_key_path(), map(), term()) -> term(). +deep_get(ConfKeyPath, Map, Default) -> + case deep_find(ConfKeyPath, Map) of + {not_found, _KeyPath, _Data} -> Default; + {ok, Data} -> Data + end. + +-spec deep_find(config_key_path(), map()) -> + {ok, term()} | {not_found, config_key_path(), term()}. +deep_find([], Map) -> + {ok, Map}; +deep_find([Key | KeyPath] = Path, Map) when is_map(Map) -> + case maps:find(Key, Map) of + {ok, SubMap} -> deep_find(KeyPath, SubMap); + error -> {not_found, Path, Map} + end; +deep_find(_KeyPath, Data) -> + {not_found, _KeyPath, Data}. + +-spec deep_put(config_key_path(), map(), term()) -> map(). +deep_put([], Map, Config) when is_map(Map) -> + Config; +deep_put([], _Map, Config) -> %% not map, replace it + Config; +deep_put([Key | KeyPath], Map, Config) -> + SubMap = deep_put(KeyPath, maps:get(Key, Map, #{}), Config), + Map#{Key => SubMap}. + +-spec deep_remove(config_key_path(), map()) -> map(). +deep_remove([], Map) -> + Map; +deep_remove([Key], Map) -> + maps:remove(Key, Map); +deep_remove([Key | KeyPath], Map) -> + case maps:find(Key, Map) of + {ok, SubMap} when is_map(SubMap) -> + Map#{Key => deep_remove(KeyPath, SubMap)}; + {ok, _Val} -> Map; + error -> Map + end. + +%% #{a => #{b => 3, c => 2}, d => 4} +%% = deep_merge(#{a => #{b => 1, c => 2}, d => 4}, #{a => #{b => 3}}). +-spec deep_merge(map(), map()) -> map(). +deep_merge(BaseMap, NewMap) -> + NewKeys = maps:keys(NewMap) -- maps:keys(BaseMap), + MergedBase = maps:fold(fun(K, V, Acc) -> + case maps:find(K, NewMap) of + error -> + Acc#{K => V}; + {ok, NewV} when is_map(V), is_map(NewV) -> + Acc#{K => deep_merge(V, NewV)}; + {ok, NewV} -> + Acc#{K => NewV} + end + end, #{}, BaseMap), + maps:merge(MergedBase, maps:with(NewKeys, NewMap)). + +unsafe_atom_key_map(Map) -> + covert_keys_to_atom(Map, fun(K) -> binary_to_atom(K, utf8) end). + +safe_atom_key_map(Map) -> + covert_keys_to_atom(Map, fun(K) -> binary_to_existing_atom(K, utf8) end). + +%%--------------------------------------------------------------------------- +covert_keys_to_atom(BinKeyMap, Conv) when is_map(BinKeyMap) -> + maps:fold( + fun(K, V, Acc) when is_binary(K) -> + Acc#{Conv(K) => covert_keys_to_atom(V, Conv)}; + (K, V, Acc) when is_atom(K) -> + %% richmap keys + Acc#{K => covert_keys_to_atom(V, Conv)} + end, #{}, BinKeyMap); +covert_keys_to_atom(ListV, Conv) when is_list(ListV) -> + [covert_keys_to_atom(V, Conv) || V <- ListV]; +covert_keys_to_atom(Val, _) -> Val. diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index c3ce14d83..cd0039791 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -172,7 +172,7 @@ {counter, 'client.connected'}, {counter, 'client.authenticate'}, {counter, 'client.auth.anonymous'}, - {counter, 'client.check_acl'}, + {counter, 'client.authorize'}, {counter, 'client.subscribe'}, {counter, 'client.unsubscribe'}, {counter, 'client.disconnected'} @@ -563,7 +563,7 @@ reserved_idx('client.connected') -> 202; reserved_idx('client.authenticate') -> 203; reserved_idx('client.enhanced_authenticate') -> 204; reserved_idx('client.auth.anonymous') -> 205; -reserved_idx('client.check_acl') -> 206; +reserved_idx('client.authorize') -> 206; reserved_idx('client.subscribe') -> 207; reserved_idx('client.unsubscribe') -> 208; reserved_idx('client.disconnected') -> 209; diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx/src/emqx_misc.erl index 04af5f72c..d45b6f7ce 100644 --- a/apps/emqx/src/emqx_misc.erl +++ b/apps/emqx/src/emqx_misc.erl @@ -197,7 +197,8 @@ check_oom(Policy) -> check_oom(self(), Policy). -spec(check_oom(pid(), emqx_types:oom_policy()) -> ok | {shutdown, term()}). -check_oom(Pid, #{message_queue_len := MaxQLen, +check_oom(_Pid, #{enable := false}) -> ok; +check_oom(Pid, #{max_message_queue_len := MaxQLen, max_heap_size := MaxHeapSize}) -> case process_info(Pid, [message_queue_len, total_heap_size]) of undefined -> ok; @@ -214,13 +215,26 @@ do_check_oom([{Val, Max, Reason}|Rest]) -> false -> do_check_oom(Rest) end. -tune_heap_size(#{max_heap_size := MaxHeapSize}) -> - %% If set to zero, the limit is disabled. - erlang:process_flag(max_heap_size, #{size => MaxHeapSize, - kill => false, - error_logger => true - }); -tune_heap_size(undefined) -> ok. +tune_heap_size(#{enable := false}) -> + ok; +%% If the max_heap_size is set to zero, the limit is disabled. +tune_heap_size(#{max_heap_size := MaxHeapSize}) when MaxHeapSize > 0 -> + MaxSize = case erlang:system_info(wordsize) of + 8 -> % arch_64 + (1 bsl 59) - 1; + 4 -> % arch_32 + (1 bsl 27) - 1 + end, + OverflowedSize = case erlang:trunc(MaxHeapSize * 1.5) of + SZ when SZ > MaxSize -> MaxSize; + SZ -> SZ + end, + erlang:process_flag(max_heap_size, #{ + size => OverflowedSize, + kill => true, + error_logger => true + }). + -spec(proc_name(atom(), pos_integer()) -> atom()). proc_name(Mod, Id) -> diff --git a/apps/emqx/src/emqx_mqtt_caps.erl b/apps/emqx/src/emqx_mqtt_caps.erl index b1be5d5a5..add86ef99 100644 --- a/apps/emqx/src/emqx_mqtt_caps.erl +++ b/apps/emqx/src/emqx_mqtt_caps.erl @@ -25,14 +25,8 @@ ]). -export([ get_caps/1 - , get_caps/2 - , get_caps/3 ]). --export([default_caps/0]). - --export([default/0]). - -export_type([caps/0]). -type(caps() :: #{max_packet_size => integer(), @@ -46,7 +40,7 @@ shared_subscription => boolean() }). --define(UNLIMITED, 0). +-define(MAX_TOPIC_LEVELS, 65535). -define(PUBCAP_KEYS, [max_topic_levels, max_qos_allowed, @@ -62,7 +56,7 @@ -define(DEFAULT_CAPS, #{max_packet_size => ?MAX_PACKET_SIZE, max_clientid_len => ?MAX_CLIENTID_LEN, max_topic_alias => ?MAX_TOPIC_AlIAS, - max_topic_levels => ?UNLIMITED, + max_topic_levels => ?MAX_TOPIC_LEVELS, max_qos_allowed => ?QOS_2, retain_available => true, wildcard_subscription => true, @@ -81,7 +75,7 @@ check_pub(Zone, Flags) when is_map(Flags) -> Flags1#{topic_levels => emqx_topic:levels(Topic)}; error -> Flags - end, get_caps(Zone, publish)). + end, maps:with(?PUBCAP_KEYS, get_caps(Zone))). do_check_pub(#{topic_levels := Levels}, #{max_topic_levels := Limit}) when Limit > 0, Levels > Limit -> @@ -98,7 +92,7 @@ do_check_pub(_Flags, _Caps) -> ok. emqx_types:subopts()) -> ok_or_error(emqx_types:reason_code())). check_sub(Zone, Topic, SubOpts) -> - Caps = get_caps(Zone, subscribe), + Caps = maps:with(?SUBCAP_KEYS, get_caps(Zone)), Flags = lists:foldl( fun(max_topic_levels, Map) -> Map#{topic_levels => emqx_topic:levels(Topic)}; @@ -119,42 +113,7 @@ do_check_sub(#{is_shared := true}, #{shared_subscription := false}) -> {error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED}; do_check_sub(_Flags, _Caps) -> ok. -default_caps() -> - ?DEFAULT_CAPS. - -get_caps(Zone, Cap, Def) -> - emqx_zone:get_env(Zone, Cap, Def). - -get_caps(Zone, publish) -> - with_env(Zone, '$mqtt_pub_caps', - fun() -> - filter_caps(?PUBCAP_KEYS, get_caps(Zone)) - end); - -get_caps(Zone, subscribe) -> - with_env(Zone, '$mqtt_sub_caps', - fun() -> - filter_caps(?SUBCAP_KEYS, get_caps(Zone)) - end). - get_caps(Zone) -> - with_env(Zone, '$mqtt_caps', - fun() -> - maps:map(fun(Cap, Def) -> - emqx_zone:get_env(Zone, Cap, Def) - end, ?DEFAULT_CAPS) - end). - -filter_caps(Keys, Caps) -> - maps:filter(fun(Key, _Val) -> lists:member(Key, Keys) end, Caps). - --spec(default() -> caps()). -default() -> ?DEFAULT_CAPS. - -with_env(Zone, Key, InitFun) -> - case emqx_zone:get_env(Zone, Key) of - undefined -> Caps = InitFun(), - ok = emqx_zone:set_env(Zone, Key, Caps), - Caps; - ZoneCaps -> ZoneCaps - end. + lists:foldl(fun({K, V}, Acc) -> + Acc#{K => emqx_config:get_zone_conf(Zone, [mqtt, K], V)} + end, #{}, maps:to_list(?DEFAULT_CAPS)). diff --git a/apps/emqx/src/emqx_mqueue.erl b/apps/emqx/src/emqx_mqueue.erl index d0c6365ff..d625209ca 100644 --- a/apps/emqx/src/emqx_mqueue.erl +++ b/apps/emqx/src/emqx_mqueue.erl @@ -67,6 +67,8 @@ , dropped/1 ]). +-define(NO_PRIORITY_TABLE, disabled). + -export_type([mqueue/0, options/0]). -type(topic() :: emqx_topic:topic()). diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index d6579cac9..fdcdf82b3 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -22,15 +22,9 @@ -logger_header("[OS_MON]"). --export([start_link/1]). +-export([start_link/0]). --export([ get_cpu_check_interval/0 - , set_cpu_check_interval/1 - , get_cpu_high_watermark/0 - , set_cpu_high_watermark/1 - , get_cpu_low_watermark/0 - , set_cpu_low_watermark/1 - , get_mem_check_interval/0 +-export([ get_mem_check_interval/0 , set_mem_check_interval/1 , get_sysmem_high_watermark/0 , set_sysmem_high_watermark/1 @@ -51,119 +45,76 @@ -define(OS_MON, ?MODULE). -start_link(Opts) -> - gen_server:start_link({local, ?OS_MON}, ?MODULE, [Opts], []). +start_link() -> + gen_server:start_link({local, ?OS_MON}, ?MODULE, [], []). %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- -get_cpu_check_interval() -> - call(get_cpu_check_interval). - -set_cpu_check_interval(Seconds) -> - call({set_cpu_check_interval, Seconds}). - -get_cpu_high_watermark() -> - call(get_cpu_high_watermark). - -set_cpu_high_watermark(Float) -> - call({set_cpu_high_watermark, Float}). - -get_cpu_low_watermark() -> - call(get_cpu_low_watermark). - -set_cpu_low_watermark(Float) -> - call({set_cpu_low_watermark, Float}). - get_mem_check_interval() -> - memsup:get_check_interval() div 1000. + memsup:get_check_interval(). -set_mem_check_interval(Seconds) when Seconds < 60 -> +set_mem_check_interval(Seconds) when Seconds < 60000 -> memsup:set_check_interval(1); set_mem_check_interval(Seconds) -> - memsup:set_check_interval(Seconds div 60). + memsup:set_check_interval(Seconds div 60000). get_sysmem_high_watermark() -> memsup:get_sysmem_high_watermark(). set_sysmem_high_watermark(Float) -> - memsup:set_sysmem_high_watermark(Float / 100). + memsup:set_sysmem_high_watermark(Float). get_procmem_high_watermark() -> memsup:get_procmem_high_watermark(). set_procmem_high_watermark(Float) -> - memsup:set_procmem_high_watermark(Float / 100). - -call(Req) -> - gen_server:call(?OS_MON, Req, infinity). + memsup:set_procmem_high_watermark(Float). %%-------------------------------------------------------------------- %% gen_server callbacks %%-------------------------------------------------------------------- -init([Opts]) -> - set_mem_check_interval(proplists:get_value(mem_check_interval, Opts)), - set_sysmem_high_watermark(proplists:get_value(sysmem_high_watermark, Opts)), - set_procmem_high_watermark(proplists:get_value(procmem_high_watermark, Opts)), - {ok, ensure_check_timer(#{cpu_high_watermark => proplists:get_value(cpu_high_watermark, Opts), - cpu_low_watermark => proplists:get_value(cpu_low_watermark, Opts), - cpu_check_interval => proplists:get_value(cpu_check_interval, Opts), - timer => undefined})}. - -handle_call(get_cpu_check_interval, _From, State) -> - {reply, maps:get(cpu_check_interval, State, undefined), State}; - -handle_call({set_cpu_check_interval, Seconds}, _From, State) -> - {reply, ok, State#{cpu_check_interval := Seconds}}; - -handle_call(get_cpu_high_watermark, _From, State) -> - {reply, maps:get(cpu_high_watermark, State, undefined), State}; - -handle_call({set_cpu_high_watermark, Float}, _From, State) -> - {reply, ok, State#{cpu_high_watermark := Float}}; - -handle_call(get_cpu_low_watermark, _From, State) -> - {reply, maps:get(cpu_low_watermark, State, undefined), State}; - -handle_call({set_cpu_low_watermark, Float}, _From, State) -> - {reply, ok, State#{cpu_low_watermark := Float}}; +init([]) -> + Opts = emqx_config:get([sysmon, os]), + set_mem_check_interval(maps:get(mem_check_interval, Opts)), + set_sysmem_high_watermark(maps:get(sysmem_high_watermark, Opts)), + set_procmem_high_watermark(maps:get(procmem_high_watermark, Opts)), + _ = start_check_timer(), + {ok, #{}}. handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), - {reply, ignored, State}. + {reply, {error, {unexpected_call, Req}}, State}. handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), + ?LOG(error, "unexpected_cast_discarded: ~p", [Msg]), {noreply, State}. -handle_info({timeout, Timer, check}, State = #{timer := Timer, - cpu_high_watermark := CPUHighWatermark, - cpu_low_watermark := CPULowWatermark}) -> - NState = - case emqx_vm:cpu_util() of %% TODO: should be improved? - 0 -> - State#{timer := undefined}; +handle_info({timeout, _Timer, check}, State) -> + CPUHighWatermark = emqx_config:get([sysmon, os, cpu_high_watermark]) * 100, + CPULowWatermark = emqx_config:get([sysmon, os, cpu_low_watermark]) * 100, + _ = case emqx_vm:cpu_util() of %% TODO: should be improved? + 0 -> ok; Busy when Busy >= CPUHighWatermark -> - emqx_alarm:activate(high_cpu_usage, #{usage => Busy, + emqx_alarm:activate(high_cpu_usage, #{usage => io_lib:format("~p%", [Busy]), high_watermark => CPUHighWatermark, low_watermark => CPULowWatermark}), - ensure_check_timer(State); + start_check_timer(); Busy when Busy =< CPULowWatermark -> emqx_alarm:deactivate(high_cpu_usage), - ensure_check_timer(State); + start_check_timer(); _Busy -> - ensure_check_timer(State) + start_check_timer() end, - {noreply, NState}; + {noreply, State}; handle_info(Info, State) -> - ?LOG(error, "unexpected info: ~p", [Info]), + ?LOG(info, "unexpected_info_discarded: ~p", [Info]), {noreply, State}. -terminate(_Reason, #{timer := Timer}) -> - emqx_misc:cancel_timer(Timer). +terminate(_Reason, _State) -> + ok. code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -172,8 +123,9 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- -ensure_check_timer(State = #{cpu_check_interval := Interval}) -> +start_check_timer() -> + Interval = emqx_config:get([sysmon, os, cpu_check_interval]), case erlang:system_info(system_architecture) of - "x86_64-pc-linux-musl" -> State; - _ -> State#{timer := emqx_misc:start_timer(timer:seconds(Interval), check)} + "x86_64-pc-linux-musl" -> ok; + _ -> emqx_misc:start_timer(Interval, check) end. diff --git a/apps/emqx/src/emqx_plugins.erl b/apps/emqx/src/emqx_plugins.erl index 8abc2b21f..3c91e612f 100644 --- a/apps/emqx/src/emqx_plugins.erl +++ b/apps/emqx/src/emqx_plugins.erl @@ -21,8 +21,6 @@ -logger_header("[Plugins]"). --export([init/0]). - -export([ load/0 , load/1 , unload/0 @@ -30,8 +28,6 @@ , reload/1 , list/0 , find_plugin/1 - , generate_configs/1 - , apply_configs/1 ]). -export([funlog/2]). @@ -41,35 +37,14 @@ -compile(nowarn_export_all). -endif. --dialyzer({no_match, [ plugin_loaded/2 - , plugin_unloaded/2 - ]}). - %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- -%% @doc Init plugins' config --spec(init() -> ok). -init() -> - case emqx:get_env(plugins_etc_dir) of - undefined -> ok; - PluginsEtc -> - CfgFiles = [filename:join(PluginsEtc, File) || - File <- filelib:wildcard("*.config", PluginsEtc)], - lists:foreach(fun init_config/1, CfgFiles) - end. - %% @doc Load all plugins when the broker started. -spec(load() -> ok | ignore | {error, term()}). load() -> - ok = load_ext_plugins(emqx:get_env(expand_plugins_dir)), - case emqx:get_env(plugins_loaded_file) of - undefined -> ignore; %% No plugins available - File -> - _ = ensure_file(File), - with_loaded_file(File, fun(Names) -> load_plugins(Names, false) end) - end. + ok = load_ext_plugins(emqx_config:get([plugins, expand_plugins_dir], undefined)). %% @doc Load a Plugin -spec(load(atom()) -> ok | {error, term()}). @@ -82,17 +57,13 @@ load(PluginName) when is_atom(PluginName) -> ?LOG(notice, "Plugin ~s is already started", [PluginName]), {error, already_started}; {_, false} -> - load_plugin(PluginName, true) + load_plugin(PluginName) end. %% @doc Unload all plugins before broker stopped. --spec(unload() -> list() | {error, term()}). +-spec(unload() -> ok). unload() -> - case emqx:get_env(plugins_loaded_file) of - undefined -> ignore; - File -> - with_loaded_file(File, fun stop_plugins/1) - end. + stop_plugins(list()). %% @doc UnLoad a Plugin -spec(unload(atom()) -> ok | {error, term()}). @@ -105,7 +76,7 @@ unload(PluginName) when is_atom(PluginName) -> ?LOG(error, "Plugin ~s is not started", [PluginName]), {error, not_started}; {_, _} -> - unload_plugin(PluginName, true) + unload_plugin(PluginName) end. reload(PluginName) when is_atom(PluginName)-> @@ -126,8 +97,8 @@ reload(PluginName) when is_atom(PluginName)-> -spec(list() -> [emqx_types:plugin()]). list() -> StartedApps = names(started_app), - lists:map(fun({Name, _, [Type| _]}) -> - Plugin = plugin(Name, Type), + lists:map(fun({Name, _, _}) -> + Plugin = plugin(Name), case lists:member(Name, StartedApps) of true -> Plugin#plugin{active = true}; false -> Plugin @@ -144,12 +115,6 @@ find_plugin(Name, Plugins) -> %% Internal functions %%-------------------------------------------------------------------- -init_config(CfgFile) -> - {ok, [AppsEnv]} = file:consult(CfgFile), - lists:foreach(fun({App, Envs}) -> - [application:set_env(App, Par, Val) || {Par, Val} <- Envs] - end, AppsEnv). - %% load external plugins which are placed in etc/plugins dir load_ext_plugins(undefined) -> ok; load_ext_plugins(Dir) -> @@ -173,15 +138,15 @@ load_ext_plugin(PluginDir) -> ?LOG(alert, "plugin_app_file_not_found: ~s", [AppFile]), error({plugin_app_file_not_found, AppFile}) end, - ok = load_plugin_app(AppName, Ebin), - try - ok = generate_configs(AppName, PluginDir) - catch - throw : {conf_file_not_found, ConfFile} -> - %% this is maybe a dependency of an external plugin - ?LOG(debug, "config_load_error_ignored for app=~p, path=~s", [AppName, ConfFile]), - ok - end. + ok = load_plugin_app(AppName, Ebin). + % try + % ok = generate_configs(AppName, PluginDir) + % catch + % throw : {conf_file_not_found, ConfFile} -> + % %% this is maybe a dependency of an external plugin + % ?LOG(debug, "config_load_error_ignored for app=~p, path=~s", [AppName, ConfFile]), + % ok + % end. load_plugin_app(AppName, Ebin) -> _ = code:add_patha(Ebin), @@ -199,57 +164,24 @@ load_plugin_app(AppName, Ebin) -> {error, {already_loaded, _}} -> ok end. -ensure_file(File) -> - case filelib:is_file(File) of false -> write_loaded([]); true -> ok end. - -with_loaded_file(File, SuccFun) -> - case read_loaded(File) of - {ok, Names0} -> - Names = filter_plugins(Names0), - SuccFun(Names); - {error, Error} -> - ?LOG(alert, "Failed to read: ~p, error: ~p", [File, Error]), - {error, Error} - end. - -filter_plugins(Names) -> - lists:filtermap(fun(Name1) when is_atom(Name1) -> {true, Name1}; - ({Name1, true}) -> {true, Name1}; - ({_Name1, false}) -> false - end, Names). - -load_plugins(Names, Persistent) -> - Plugins = list(), - NotFound = Names -- names(Plugins), - case NotFound of - [] -> ok; - NotFound -> ?LOG(alert, "cannot_find_plugins: ~p", [NotFound]) - end, - NeedToLoad = Names -- NotFound -- names(started_app), - lists:foreach(fun(Name) -> - Plugin = find_plugin(Name, Plugins), - load_plugin(Plugin#plugin.name, Persistent) - end, NeedToLoad). - %% Stop plugins -stop_plugins(Names) -> - _ = [stop_app(App) || App <- Names], +stop_plugins(Plugins) -> + _ = [stop_app(Plugin#plugin.name) || Plugin <- Plugins], ok. -plugin(AppName, Type) -> +plugin(AppName) -> case application:get_all_key(AppName) of {ok, Attrs} -> Descr = proplists:get_value(description, Attrs, ""), - #plugin{name = AppName, descr = Descr, type = plugin_type(Type)}; + #plugin{name = AppName, descr = Descr}; undefined -> error({plugin_not_found, AppName}) end. -load_plugin(Name, Persistent) -> +load_plugin(Name) -> try - ok = ?MODULE:generate_configs(Name), case load_app(Name) of ok -> - start_app(Name, fun(App) -> plugin_loaded(App, Persistent) end); + start_app(Name); {error, Error0} -> {error, Error0} end @@ -268,22 +200,21 @@ load_app(App) -> {error, Error} end. -start_app(App, SuccFun) -> +start_app(App) -> case application:ensure_all_started(App) of {ok, Started} -> ?LOG(info, "Started plugins: ~p", [Started]), ?LOG(info, "Load plugin ~s successfully", [App]), - _ = SuccFun(App), ok; {error, {ErrApp, Reason}} -> ?LOG(error, "Load plugin ~s failed, cannot start plugin ~s for ~0p", [App, ErrApp, Reason]), {error, {ErrApp, Reason}} end. -unload_plugin(App, Persistent) -> +unload_plugin(App) -> case stop_app(App) of ok -> - _ = plugin_unloaded(App, Persistent), ok; + ok; {error, Reason} -> {error, Reason} end. @@ -307,133 +238,5 @@ names(started_app) -> names(Plugins) -> [Name || #plugin{name = Name} <- Plugins]. -plugin_loaded(_Name, false) -> - ok; -plugin_loaded(Name, true) -> - case read_loaded() of - {ok, Names} -> - case lists:member(Name, Names) of - false -> - %% write file if plugin is loaded - write_loaded(lists:append(Names, [{Name, true}])); - true -> - ignore - end; - {error, Error} -> - ?LOG(error, "Cannot read loaded plugins: ~p", [Error]) - end. - -plugin_unloaded(_Name, false) -> - ok; -plugin_unloaded(Name, true) -> - case read_loaded() of - {ok, Names0} -> - Names = filter_plugins(Names0), - case lists:member(Name, Names) of - true -> - write_loaded(lists:delete(Name, Names)); - false -> - ?LOG(error, "Cannot find ~s in loaded_file", [Name]) - end; - {error, Error} -> - ?LOG(error, "Cannot read loaded_plugins: ~p", [Error]) - end. - -read_loaded() -> - case emqx:get_env(plugins_loaded_file) of - undefined -> {error, not_found}; - File -> read_loaded(File) - end. - -read_loaded(File) -> file:consult(File). - -write_loaded(AppNames) -> - FilePath = emqx:get_env(plugins_loaded_file), - case file:write_file(FilePath, [io_lib:format("~p.~n", [Name]) || Name <- AppNames]) of - ok -> ok; - {error, Error} -> - ?LOG(error, "Write File ~p Error: ~p", [FilePath, Error]), - {error, Error} - end. - -plugin_type(auth) -> auth; -plugin_type(protocol) -> protocol; -plugin_type(backend) -> backend; -plugin_type(bridge) -> bridge; -plugin_type(_) -> feature. - - funlog(Key, Value) -> ?LOG(info, "~s = ~p", [string:join(Key, "."), Value]). - -generate_configs(App) -> - PluginConfDir = emqx:get_env(plugins_etc_dir), - PluginSchemaDir = code:priv_dir(App), - generate_configs(App, PluginConfDir, PluginSchemaDir). - -generate_configs(App, PluginDir) -> - PluginConfDir = filename:join([PluginDir, "etc"]), - PluginSchemaDir = filename:join([PluginDir, "priv"]), - generate_configs(App, PluginConfDir, PluginSchemaDir). - -generate_configs(App, PluginConfDir, PluginSchemaDir) -> - ConfigFile = filename:join([PluginConfDir, App]) ++ ".config", - case filelib:is_file(ConfigFile) of - true -> - {ok, [Configs]} = file:consult(ConfigFile), - apply_configs(Configs); - false -> - SchemaFile = filename:join([PluginSchemaDir, App]) ++ ".schema", - case filelib:is_file(SchemaFile) of - true -> - AppsEnv = do_generate_configs(App), - apply_configs(AppsEnv); - false -> - SchemaMod = lists:concat([App, "_schema"]), - ConfName = filename:join([PluginConfDir, App]) ++ ".conf", - SchemaFile1 = filename:join([code:lib_dir(App), "ebin", SchemaMod]) ++ ".beam", - do_generate_hocon_configs(App, ConfName, SchemaFile1) - end - end. - -do_generate_configs(App) -> - Name1 = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".conf", - Name2 = filename:join([code:lib_dir(App), "etc", App]) ++ ".conf", - ConfFile = case {filelib:is_file(Name1), filelib:is_file(Name2)} of - {true, _} -> Name1; - {false, true} -> Name2; - {false, false} -> error({config_not_found, [Name1, Name2]}) - end, - SchemaFile = filename:join([code:priv_dir(App), App]) ++ ".schema", - case filelib:is_file(SchemaFile) of - true -> - Schema = cuttlefish_schema:files([SchemaFile]), - Conf = cuttlefish_conf:file(ConfFile), - cuttlefish_generator:map(Schema, Conf, undefined, fun ?MODULE:funlog/2); - false -> - error({schema_not_found, SchemaFile}) - end. - -do_generate_hocon_configs(App, ConfName, SchemaFile) -> - SchemaMod = lists:concat([App, "_schema"]), - case {filelib:is_file(ConfName), filelib:is_file(SchemaFile)} of - {true, true} -> - {ok, RawConfig} = hocon:load(ConfName, #{format => richmap}), - _ = hocon_schema:check(list_to_atom(SchemaMod), RawConfig, #{atom_key => true, - return_plain => true}), - ok; - % emqx_config:update_config([App], Config); - {true, false} -> - error({schema_not_found, [SchemaFile]}); - {false, true} -> - error({config_not_found, [ConfName]}); - {false, false} -> - error({conf_and_schema_not_found, [ConfName, SchemaFile]}) - end. - -apply_configs([]) -> - ok; -apply_configs([{App, Config} | More]) -> - lists:foreach(fun({Key, _}) -> application:unset_env(App, Key) end, application:get_all_env(App)), - lists:foreach(fun({Key, Val}) -> application:set_env(App, Key, Val) end, Config), - apply_configs(More). diff --git a/apps/emqx/src/emqx_gen_mod.erl b/apps/emqx/src/emqx_quic_connection.erl similarity index 72% rename from apps/emqx/src/emqx_gen_mod.erl rename to apps/emqx/src/emqx_quic_connection.erl index 0ebf6b59a..cd41e74a7 100644 --- a/apps/emqx/src/emqx_gen_mod.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -14,10 +14,11 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_gen_mod). +-module(emqx_quic_connection). --callback(load(Opts :: any()) -> ok | {error, term()}). +%% Callbacks +-export([ new_conn/2 + ]). --callback(unload(State :: term()) -> term()). - --callback(description() -> any()). +new_conn(Conn, {_L, COpts, _S}) when is_map(COpts) -> + emqx_connection:start_link(emqx_quic_stream, Conn, COpts). diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl new file mode 100644 index 000000000..236c11ad3 --- /dev/null +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -0,0 +1,92 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% MQTT/QUIC Stream +-module(emqx_quic_stream). + +%% emqx transport Callbacks +-export([ type/1 + , wait/1 + , getstat/2 + , fast_close/1 + , ensure_ok_or_exit/2 + , async_send/3 + , setopts/2 + , getopts/2 + , peername/1 + , sockname/1 + , peercert/1 + ]). + +wait(Conn) -> + quicer:accept_stream(Conn, []). + +type(_) -> + quic. + +peername(S) -> + quicer:peername(S). + +sockname(S) -> + quicer:sockname(S). + +peercert(_S) -> + nossl. + +getstat(Socket, Stats) -> + case quicer:getstat(Socket, Stats) of + {error, _} -> {error, closed}; + Res -> Res + end. + +setopts(Socket, Opts) -> + lists:foreach(fun({Opt, V}) when is_atom(Opt) -> + quicer:setopt(Socket, Opt, V); + (Opt) when is_atom(Opt) -> + quicer:setopt(Socket, Opt, true) + end, Opts), + ok. + +getopts(_Socket, _Opts) -> + %% @todo + {ok, [{high_watermark, 0}, + {high_msgq_watermark, 0}, + {sndbuf, 0}, + {recbuf, 0}, + {buffer,80000}]}. + +fast_close(Stream) -> + %% Stream might be closed already. + _ = quicer:async_close_stream(Stream), + ok. + +-spec(ensure_ok_or_exit(atom(), list(term())) -> term()). +ensure_ok_or_exit(Fun, Args = [Sock|_]) when is_atom(Fun), is_list(Args) -> + case erlang:apply(?MODULE, Fun, Args) of + {error, Reason} when Reason =:= enotconn; Reason =:= closed -> + fast_close(Sock), + exit(normal); + {error, Reason} -> + fast_close(Sock), + exit({shutdown, Reason}); + Result -> Result + end. + +async_send(Stream, Data, Options) when is_list(Data) -> + async_send(Stream, iolist_to_binary(Data), Options); +async_send(Stream, Data, _Options) when is_binary(Data) -> + {ok, _Len} = quicer:send(Stream, Data), + ok. diff --git a/apps/emqx/src/emqx_reason_codes.erl b/apps/emqx/src/emqx_reason_codes.erl index 893084b9d..b98fc1263 100644 --- a/apps/emqx/src/emqx_reason_codes.erl +++ b/apps/emqx/src/emqx_reason_codes.erl @@ -170,16 +170,11 @@ frame_error(frame_too_large) -> ?RC_PACKET_TOO_LARGE; frame_error(_) -> ?RC_MALFORMED_PACKET. connack_error(protocol_error) -> ?RC_PROTOCOL_ERROR; -connack_error(client_identifier_not_valid) -> ?RC_CLIENT_IDENTIFIER_NOT_VALID; connack_error(bad_username_or_password) -> ?RC_BAD_USER_NAME_OR_PASSWORD; -connack_error(bad_clientid_or_password) -> ?RC_BAD_USER_NAME_OR_PASSWORD; -connack_error(username_or_password_undefined) -> ?RC_BAD_USER_NAME_OR_PASSWORD; -connack_error(password_error) -> ?RC_BAD_USER_NAME_OR_PASSWORD; connack_error(not_authorized) -> ?RC_NOT_AUTHORIZED; connack_error(server_unavailable) -> ?RC_SERVER_UNAVAILABLE; connack_error(server_busy) -> ?RC_SERVER_BUSY; connack_error(banned) -> ?RC_BANNED; connack_error(bad_authentication_method) -> ?RC_BAD_AUTHENTICATION_METHOD; -%% TODO: ??? -connack_error(_) -> ?RC_NOT_AUTHORIZED. +connack_error(_) -> ?RC_UNSPECIFIED_ERROR. diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index 3641c49ff..02ac29cbc 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -251,7 +251,7 @@ delete_trie_route(Route = #route{topic = Topic}) -> %% @private -spec(maybe_trans(function(), list(any())) -> ok | {error, term()}). maybe_trans(Fun, Args) -> - case persistent_term:get(emqx_route_lock_type) of + case emqx_config:get([broker, perf, route_lock_type]) of key -> trans(Fun, Args); global -> diff --git a/apps/emqx/src/emqx_router_sup.erl b/apps/emqx/src/emqx_router_sup.erl index 1105a476b..2b75ce27f 100644 --- a/apps/emqx/src/emqx_router_sup.erl +++ b/apps/emqx/src/emqx_router_sup.erl @@ -33,11 +33,6 @@ init([]) -> shutdown => 5000, type => worker, modules => [emqx_router_helper]}, - - ok = persistent_term:put(emqx_route_lock_type, - application:get_env(emqx, route_lock_type, key) - ), - %% Router pool RouterPool = emqx_pool_sup:spec([router_pool, hash, {emqx_router, start_link, []}]), diff --git a/apps/emqx/src/emqx_rpc.erl b/apps/emqx/src/emqx_rpc.erl index a37d67a0a..e950e9e3d 100644 --- a/apps/emqx/src/emqx_rpc.erl +++ b/apps/emqx/src/emqx_rpc.erl @@ -53,11 +53,9 @@ cast(Key, Node, Mod, Fun, Args) -> filter_result(?RPC:cast(rpc_node({Key, Node}), Mod, Fun, Args)). rpc_node(Node) when is_atom(Node) -> - ClientNum = application:get_env(gen_rpc, tcp_client_num, ?DefaultClientNum), - {Node, rand:uniform(ClientNum)}; + {Node, rand:uniform(max_client_num())}; rpc_node({Key, Node}) when is_atom(Node) -> - ClientNum = application:get_env(gen_rpc, tcp_client_num, ?DefaultClientNum), - {Node, erlang:phash2(Key, ClientNum) + 1}. + {Node, erlang:phash2(Key, max_client_num()) + 1}. rpc_nodes(Nodes) -> rpc_nodes(Nodes, []). @@ -72,3 +70,6 @@ filter_result({Error, Reason}) {badrpc, Reason}; filter_result(Delivery) -> Delivery. + +max_client_num() -> + emqx_config:get([rpc, tcp_client_num], ?DefaultClientNum). diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 83e611ee0..3fd060d9f 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1,3 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -module(emqx_schema). -dialyzer(no_return). @@ -9,69 +25,73 @@ -include_lib("typerefl/include/types.hrl"). -type log_level() :: debug | info | notice | warning | error | critical | alert | emergency | all. --type flag() :: true | false. -type duration() :: integer(). -type duration_s() :: integer(). -type duration_ms() :: integer(). -type bytesize() :: integer(). +-type wordsize() :: bytesize(). -type percent() :: float(). -type file() :: string(). -type comma_separated_list() :: list(). -type comma_separated_atoms() :: [atom()]. -type bar_separated_list() :: list(). -type ip_port() :: tuple(). +-type cipher() :: map(). --typerefl_from_string({flag/0, emqx_schema, to_flag}). -typerefl_from_string({duration/0, emqx_schema, to_duration}). -typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}). -typerefl_from_string({duration_ms/0, emqx_schema, to_duration_ms}). -typerefl_from_string({bytesize/0, emqx_schema, to_bytesize}). +-typerefl_from_string({wordsize/0, emqx_schema, to_wordsize}). -typerefl_from_string({percent/0, emqx_schema, to_percent}). -typerefl_from_string({comma_separated_list/0, emqx_schema, to_comma_separated_list}). -typerefl_from_string({bar_separated_list/0, emqx_schema, to_bar_separated_list}). -typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}). +-typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}). -typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}). % workaround: prevent being recognized as unused functions --export([to_duration/1, to_duration_s/1, to_duration_ms/1, to_bytesize/1, - to_flag/1, to_percent/1, to_comma_separated_list/1, +-export([to_duration/1, to_duration_s/1, to_duration_ms/1, + to_bytesize/1, to_wordsize/1, + to_percent/1, to_comma_separated_list/1, to_bar_separated_list/1, to_ip_port/1, + to_erl_cipher_suite/1, to_comma_separated_atoms/1]). -behaviour(hocon_schema). --reflect_type([ log_level/0, flag/0, duration/0, duration_s/0, duration_ms/0, - bytesize/0, percent/0, file/0, +-reflect_type([ log_level/0, duration/0, duration_s/0, duration_ms/0, + bytesize/0, wordsize/0, percent/0, file/0, comma_separated_list/0, bar_separated_list/0, ip_port/0, + cipher/0, comma_separated_atoms/0]). -export([structs/0, fields/1, translations/0, translation/1]). -export([t/1, t/3, t/4, ref/1]). -export([conf_get/2, conf_get/3, keys/2, filter/1]). --export([ssl/2, tr_ssl/2, tr_password_hash/2]). +-export([ssl/1]). %% will be used by emqx_ct_helper to find the dependent apps --export([includes/0]). +-export([includes/0, extra_schema_fields/1]). -structs() -> ["cluster", "node", "rpc", "log", "lager", - "acl", "mqtt", "zone", "listener", "module", "broker", - "plugins", "sysmon", "os_mon", "vm_mon", "alarm"] - ++ includes(). +structs() -> ["cluster", "node", "rpc", "log", + "zones", "listeners", "broker", + "plugins", "sysmon", "alarm"] + ++ ?MODULE:includes(). --ifdef(TEST). -includes() ->[]. +-ifndef(EMQX_EXT_SCHEMAS). +includes() -> []. -else. includes() -> - [ "emqx_data_bridge" - , "emqx_telemetry" - ]. + [FieldName || {FieldName, _SchemaMod} <- ?EMQX_EXT_SCHEMAS]. -endif. fields("cluster") -> [ {"name", t(atom(), "ekka.cluster_name", emqxcl)} - , {"discovery", t(atom(), undefined, manual)} - , {"autoclean", t(duration(), "ekka.cluster_autoclean", undefined)} - , {"autoheal", t(flag(), "ekka.cluster_autoheal", false)} + , {"discovery_strategy", t(union([manual, static, mcast, dns, etcd, k8s]), + undefined, manual)} + , {"autoclean", t(duration(), "ekka.cluster_autoclean", "5m")} + , {"autoheal", t(boolean(), "ekka.cluster_autoheal", true)} , {"static", ref("static")} , {"mcast", ref("mcast")} , {"proto_dist", t(union([inet_tcp, inet6_tcp, inet_tls]), "ekka.proto_dist", inet_tcp)} @@ -83,39 +103,40 @@ fields("cluster") -> ]; fields("static") -> - [ {"seeds", t(comma_separated_list())}]; + [ {"seeds", t(hoconsc:array(string()), undefined, [])}]; fields("mcast") -> [ {"addr", t(string(), undefined, "239.192.0.1")} - , {"ports", t(comma_separated_list(), undefined, "4369")} + , {"ports", t(hoconsc:array(integer()), undefined, [4369, 4370])} , {"iface", t(string(), undefined, "0.0.0.0")} - , {"ttl", t(integer(), undefined, 255)} - , {"loop", t(flag(), undefined, true)} + , {"ttl", t(range(0, 255), undefined, 255)} + , {"loop", t(boolean(), undefined, true)} , {"sndbuf", t(bytesize(), undefined, "16KB")} , {"recbuf", t(bytesize(), undefined, "16KB")} , {"buffer", t(bytesize(), undefined, "32KB")} ]; fields("dns") -> - [ {"app", t(string())}]; + [ {"name", t(string(), undefined, "localhost")} + , {"app", t(string(), undefined, "emqx")}]; fields("etcd") -> [ {"server", t(comma_separated_list())} - , {"prefix", t(string())} + , {"prefix", t(string(), undefined, "emqxcl")} , {"node_ttl", t(duration(), undefined, "1m")} , {"ssl", ref("etcd_ssl")} ]; fields("etcd_ssl") -> - ssl(undefined, #{}); + ssl(#{}); fields("k8s") -> [ {"apiserver", t(string())} - , {"service_name", t(string())} + , {"service_name", t(string(), undefined, "emqx")} , {"address_type", t(union([ip, dns, hostname]))} - , {"app_name", t(string())} - , {"namespace", t(string())} - , {"suffix", t(string(), undefined, "")} + , {"app_name", t(string(), undefined, "emqx")} + , {"namespace", t(string(), undefined, "default")} + , {"suffix", t(string(), undefined, "pod.local")} ]; fields("rlog") -> @@ -124,37 +145,30 @@ fields("rlog") -> ]; fields("node") -> - [ {"name", t(string(), "vm_args.-name", "emqx@127.0.0.1", "EMQX_NODE_NAME")} - , {"ssl_dist_optfile", t(string(), "vm_args.-ssl_dist_optfile", undefined)} + [ {"name", hoconsc:t(string(), #{default => "emqx@127.0.0.1", + override_env => "EMQX_NODE_NAME" + })} , {"cookie", hoconsc:t(string(), #{mapping => "vm_args.-setcookie", default => "emqxsecretcookie", sensitive => true, override_env => "EMQX_NODE_COOKIE" })} - , {"data_dir", t(string(), "emqx.data_dir", undefined)} - , {"etc_dir", t(string(), "emqx.etc_dir", undefined)} - , {"heartbeat", t(flag(), undefined, false)} - , {"async_threads", t(range(1, 1024), "vm_args.+A", undefined)} - , {"process_limit", t(integer(), "vm_args.+P", undefined)} - , {"max_ports", t(range(1024, 134217727), "vm_args.+Q", undefined, "EMQX_MAX_PORTS")} - , {"dist_buffer_size", fun node__dist_buffer_size/1} - , {"global_gc_interval", t(duration_s(), "emqx.global_gc_interval", undefined)} - , {"fullsweep_after", t(non_neg_integer(), - "vm_args.-env ERL_FULLSWEEP_AFTER", 1000)} - , {"max_ets_tables", t(integer(), "vm_args.+e", 256000)} - , {"crash_dump", t(file(), "vm_args.-env ERL_CRASH_DUMP", undefined)} - , {"dist_net_ticktime", t(integer(), "vm_args.-kernel net_ticktime", undefined)} - , {"dist_listen_min", t(integer(), "kernel.inet_dist_listen_min", undefined)} - , {"dist_listen_max", t(integer(), "kernel.inet_dist_listen_max", undefined)} - , {"backtrace_depth", t(integer(), "emqx.backtrace_depth", 16)} + , {"data_dir", t(string())} + , {"config_files", t(comma_separated_list())} + , {"global_gc_interval", t(duration(), undefined, "15m")} + , {"crash_dump_dir", t(file(), "vm_args.-env ERL_CRASH_DUMP", undefined)} + , {"dist_net_ticktime", t(duration(), "vm_args.-kernel net_ticktime", "2m")} + , {"dist_listen_min", t(range(1024, 65535), "kernel.inet_dist_listen_min", 6369)} + , {"dist_listen_max", t(range(1024, 65535), "kernel.inet_dist_listen_max", 6369)} + , {"backtrace_depth", t(integer(), undefined, 23)} ]; fields("rpc") -> - [ {"mode", t(union(sync, async), "emqx.rpc_mode", async)} + [ {"mode", t(union(sync, async), undefined, async)} , {"async_batch_size", t(integer(), "gen_rpc.max_batch_size", 256)} , {"port_discovery",t(union(manual, stateless), "gen_rpc.port_discovery", stateless)} , {"tcp_server_port", t(integer(), "gen_rpc.tcp_server_port", 5369)} - , {"tcp_client_num", t(range(0, 255), undefined, 0)} + , {"tcp_client_num", t(range(1, 256), undefined, 1)} , {"connect_timeout", t(duration(), "gen_rpc.connect_timeout", "5s")} , {"send_timeout", t(duration(), "gen_rpc.send_timeout", "5s")} , {"authentication_timeout", t(duration(), "gen_rpc.authentication_timeout", "5s")} @@ -168,722 +182,423 @@ fields("rpc") -> ]; fields("log") -> - [ {"to", t(union([file, console, both]), undefined, file)} - , {"level", t(log_level(), undefined, warning)} + [ {"primary_level", t(log_level(), undefined, warning)} + , {"console_handler", ref("console_handler")} + , {"file_handlers", ref("file_handlers")} , {"time_offset", t(string(), undefined, "system")} - , {"primary_log_level", t(log_level(), undefined, warning)} - , {"dir", t(string(), undefined, "log")} - , {"file", t(file(), undefined, "emqx.log")} - , {"chars_limit", t(integer(), undefined, -1)} + , {"chars_limit", maybe_infinity(range(1, inf))} , {"supervisor_reports", t(union([error, progress]), undefined, error)} , {"max_depth", t(union([infinity, integer()]), "kernel.error_logger_format_depth", 80)} , {"formatter", t(union([text, json]), undefined, text)} , {"single_line", t(boolean(), undefined, true)} - , {"rotation", ref("rotation")} - , {"size", t(union(bytesize(), infinity), undefined, infinity)} , {"sync_mode_qlen", t(integer(), undefined, 100)} , {"drop_mode_qlen", t(integer(), undefined, 3000)} , {"flush_qlen", t(integer(), undefined, 8000)} - , {"overload_kill", t(flag(), undefined, true)} - , {"overload_kill_mem_size", t(bytesize(), undefined, "30MB")} - , {"overload_kill_qlen", t(integer(), undefined, 20000)} - , {"overload_kill_restart_after", t(union(duration(), infinity), undefined, "5s")} - , {"burst_limit", t(comma_separated_list(), undefined, "disabled")} + , {"overload_kill", ref("log_overload_kill")} + , {"burst_limit", ref("log_burst_limit")} , {"error_logger", t(atom(), "kernel.error_logger", silent)} - , {"debug", ref("additional_log_file")} - , {"info", ref("additional_log_file")} - , {"notice", ref("additional_log_file")} - , {"warning", ref("additional_log_file")} - , {"error", ref("additional_log_file")} - , {"critical", ref("additional_log_file")} - , {"alert", ref("additional_log_file")} - , {"emergency", ref("additional_log_file")} ]; -fields("additional_log_file") -> - [ {"file", t(string())}]; - -fields("rotation") -> - [ {"enable", t(flag(), undefined, true)} - , {"size", t(bytesize(), undefined, "10MB")} - , {"count", t(integer(), undefined, 5)} +fields("console_handler") -> + [ {"enable", t(boolean(), undefined, false)} + , {"level", t(log_level(), undefined, warning)} ]; -fields("lager") -> - [ {"handlers", t(string(), "lager.handlers", "")} - , {"crash_log", t(flag(), "lager.crash_log", false)} +fields("file_handlers") -> + [ {"$name", ref("log_file_handler")} ]; -fields("acl") -> - [ {"allow_anonymous", t(boolean(), "emqx.allow_anonymous", false)} - , {"acl_nomatch", t(union(allow, deny), "emqx.acl_nomatch", deny)} - , {"acl_file", t(string(), "emqx.acl_file", undefined)} - , {"enable_acl_cache", t(flag(), "emqx.enable_acl_cache", true)} - , {"acl_cache_ttl", t(duration(), "emqx.acl_cache_ttl", "1m")} - , {"acl_cache_max_size", t(range(1, inf), "emqx.acl_cache_max_size", 32)} - , {"acl_deny_action", t(union(ignore, disconnect), "emqx.acl_deny_action", ignore)} - , {"flapping_detect_policy", t(comma_separated_list(), undefined, "30,1m,5m")} +fields("log_file_handler") -> + [ {"level", t(log_level(), undefined, warning)} + , {"file", t(file(), undefined, undefined)} + , {"rotation", ref("log_rotation")} + , {"max_size", maybe_infinity(bytesize(), "10MB")} + ]; + +fields("log_rotation") -> + [ {"enable", t(boolean(), undefined, true)} + , {"count", t(range(1, 2048), undefined, 10)} + ]; + +fields("log_overload_kill") -> + [ {"enable", t(boolean(), undefined, true)} + , {"mem_size", t(bytesize(), undefined, "30MB")} + , {"qlen", t(integer(), undefined, 20000)} + , {"restart_after", t(union(duration(), infinity), undefined, "5s")} + ]; + +fields("log_burst_limit") -> + [ {"enable", t(boolean(), undefined, true)} + , {"max_count", t(integer(), undefined, 10000)} + , {"window_time", t(duration(), undefined, "1s")} + ]; + +fields("stats") -> + [ {"enable", t(boolean(), undefined, true)} + ]; + +fields("auth") -> + [ {"enable", t(boolean(), undefined, false)} + ]; + +fields("authorization_settings") -> + [ {"enable", t(boolean(), undefined, true)} + , {"cache", ref("authorization_cache")} + , {"deny_action", t(union(ignore, disconnect), undefined, ignore)} + ]; + +fields("authorization_cache") -> + [ {"enable", t(boolean(), undefined, true)} + , {"max_size", t(range(1, 1048576), undefined, 32)} + , {"ttl", t(duration(), undefined, "1m")} ]; fields("mqtt") -> - [ {"max_packet_size", t(bytesize(), "emqx.max_packet_size", "1MB", "EMQX_MAX_PACKET_SIZE")} - , {"max_clientid_len", t(integer(), "emqx.max_clientid_len", 65535)} - , {"max_topic_levels", t(integer(), "emqx.max_topic_levels", 0)} - , {"max_qos_allowed", t(range(0, 2), "emqx.max_qos_allowed", 2)} - , {"max_topic_alias", t(integer(), "emqx.max_topic_alias", 65535)} - , {"retain_available", t(boolean(), "emqx.retain_available", true)} - , {"wildcard_subscription", t(boolean(), "emqx.wildcard_subscription", true)} - , {"shared_subscription", t(boolean(), "emqx.shared_subscription", true)} - , {"ignore_loop_deliver", t(boolean(), "emqx.ignore_loop_deliver", true)} - , {"strict_mode", t(boolean(), "emqx.strict_mode", false)} - , {"response_information", t(string(), "emqx.response_information", undefined)} + [ {"mountpoint", t(binary(), undefined, <<>>)} + , {"idle_timeout", maybe_infinity(duration(), "15s")} + , {"max_packet_size", t(bytesize(), undefined, "1MB")} + , {"max_clientid_len", t(range(23, 65535), undefined, 65535)} + , {"max_topic_levels", t(range(1, 65535), undefined, 65535)} + , {"max_qos_allowed", t(range(0, 2), undefined, 2)} + , {"max_topic_alias", t(range(0, 65535), undefined, 65535)} + , {"retain_available", t(boolean(), undefined, true)} + , {"wildcard_subscription", t(boolean(), undefined, true)} + , {"shared_subscription", t(boolean(), undefined, true)} + , {"ignore_loop_deliver", t(boolean(), undefined, false)} + , {"strict_mode", t(boolean(), undefined, false)} + , {"response_information", t(string(), undefined, "")} + , {"server_keepalive", maybe_disabled(integer())} + , {"keepalive_backoff", t(float(), undefined, 0.75)} + , {"max_subscriptions", maybe_infinity(range(1, inf))} + , {"upgrade_qos", t(boolean(), undefined, false)} + , {"max_inflight", t(range(1, 65535), undefined, 32)} + , {"retry_interval", t(duration(), undefined, "30s")} + , {"max_awaiting_rel", maybe_infinity(integer(), 100)} + , {"await_rel_timeout", t(duration(), undefined, "300s")} + , {"session_expiry_interval", t(duration(), undefined, "2h")} + , {"max_mqueue_len", maybe_infinity(range(0, inf), 1000)} + , {"mqueue_priorities", maybe_disabled(map())} + , {"mqueue_default_priority", t(union(highest, lowest), undefined, lowest)} + , {"mqueue_store_qos0", t(boolean(), undefined, true)} + , {"use_username_as_clientid", t(boolean(), undefined, false)} + , {"peer_cert_as_username", maybe_disabled(union([cn, dn, crt, pem, md5]))} + , {"peer_cert_as_clientid", maybe_disabled(union([cn, dn, crt, pem, md5]))} ]; -fields("zone") -> +fields("zones") -> [ {"$name", ref("zone_settings")}]; fields("zone_settings") -> - [ {"idle_timeout", t(duration(), undefined, "15s")} - , {"allow_anonymous", t(boolean())} - , {"acl_nomatch", t(union(allow, deny))} - , {"enable_acl", t(flag(), undefined, false)} - , {"acl_deny_action", t(union(ignore, disconnect), undefined, ignore)} - , {"enable_ban", t(flag(), undefined, false)} - , {"enable_stats", t(flag(), undefined, false)} - , {"max_packet_size", t(bytesize())} - , {"max_clientid_len", t(integer())} - , {"max_topic_levels", t(integer())} - , {"max_qos_allowed", t(range(0, 2))} - , {"max_topic_alias", t(integer())} - , {"retain_available", t(boolean())} - , {"wildcard_subscription", t(boolean())} - , {"shared_subscription", t(boolean())} - , {"server_keepalive", t(integer())} - , {"keepalive_backoff", t(float(), undefined, 0.75)} - , {"max_subscriptions", t(integer(), undefined, 0)} - , {"upgrade_qos", t(flag(), undefined, false)} - , {"max_inflight", t(range(0, 65535))} - , {"retry_interval", t(duration_s(), undefined, "30s")} - , {"max_awaiting_rel", t(duration(), undefined, 0)} - , {"await_rel_timeout", t(duration_s(), undefined, "300s")} - , {"ignore_loop_deliver", t(boolean())} - , {"session_expiry_interval", t(duration_s(), undefined, "2h")} - , {"max_mqueue_len", t(integer(), undefined, 1000)} - , {"mqueue_priorities", t(comma_separated_list(), undefined, "none")} - , {"mqueue_default_priority", t(union(highest, lowest), undefined, lowest)} - , {"mqueue_store_qos0", t(boolean(), undefined, true)} - , {"enable_flapping_detect", t(flag(), undefined, false)} - , {"rate_limit", ref("rate_limit")} + [ {"mqtt", ref("mqtt")} + , {"authorization", ref("authorization_settings")} + , {"auth", ref("auth")} + , {"stats", ref("stats")} + , {"flapping_detect", ref("flapping_detect")} + , {"force_shutdown", ref("force_shutdown")} , {"conn_congestion", ref("conn_congestion")} - , {"quota", ref("quota")} - , {"force_gc_policy", t(bar_separated_list())} - , {"force_shutdown_policy", t(bar_separated_list(), undefined, "default")} - , {"mountpoint", t(string())} - , {"use_username_as_clientid", t(boolean(), undefined, false)} - , {"strict_mode", t(boolean(), undefined, false)} - , {"response_information", t(string())} - , {"bypass_auth_plugins", t(boolean(), undefined, false)} + , {"force_gc", ref("force_gc")} + , {"overall_max_connections", maybe_infinity(integer())} + , {"listeners", t("listeners")} ]; fields("rate_limit") -> - [ {"conn_messages_in", t(comma_separated_list())} - , {"conn_bytes_in", t(comma_separated_list())} + [ {"max_conn_rate", maybe_infinity(integer(), 1000)} + , {"conn_messages_in", maybe_infinity(comma_separated_list())} + , {"conn_bytes_in", maybe_infinity(comma_separated_list())} + , {"quota", ref("rate_limit_quota")} + ]; + +fields("rate_limit_quota") -> + [ {"conn_messages_routing", maybe_infinity(comma_separated_list())} + , {"overall_messages_routing", maybe_infinity(comma_separated_list())} + ]; + +fields("flapping_detect") -> + [ {"enable", t(boolean(), undefined, false)} + , {"max_count", t(integer(), undefined, 15)} + , {"window_time", t(duration(), undefined, "1m")} + , {"ban_time", t(duration(), undefined, "5m")} + ]; + +fields("force_shutdown") -> + [ {"enable", t(boolean(), undefined, true)} + , {"max_message_queue_len", t(range(0, inf), undefined, 1000)} + , {"max_heap_size", t(wordsize(), undefined, "32MB", undefined, + fun(Siz) -> + MaxSiz = case erlang:system_info(wordsize) of + 8 -> % arch_64 + (1 bsl 59) - 1; + 4 -> % arch_32 + (1 bsl 27) - 1 + end, + case Siz > MaxSiz of + true -> + error(io_lib:format("force_shutdown_policy: heap-size ~s is too large", [Siz])); + false -> + ok + end + end)} ]; fields("conn_congestion") -> - [ {"alarm", t(flag(), undefined, false)} + [ {"enable_alarm", t(boolean(), undefined, false)} , {"min_alarm_sustain_duration", t(duration(), undefined, "1m")} ]; -fields("quota") -> - [ {"conn_messages_routing", t(comma_separated_list())} - , {"overall_messages_routing", t(comma_separated_list())} +fields("force_gc") -> + [ {"enable", t(boolean(), undefined, true)} + , {"count", t(range(0, inf), undefined, 16000)} + , {"bytes", t(bytesize(), undefined, "16MB")} ]; -fields("listener") -> - [ {"tcp", ref("tcp_listener")} - , {"ssl", ref("ssl_listener")} - , {"ws", ref("ws_listener")} - , {"wss", ref("wss_listener")} +fields("listeners") -> + [ {"$name", hoconsc:union( + [ hoconsc:ref("mqtt_tcp_listener") + , hoconsc:ref("mqtt_ws_listener") + , hoconsc:ref("mqtt_quic_listener") + ])} ]; -fields("tcp_listener") -> - [ {"$name", ref("tcp_listener_settings")}]; +fields("mqtt_tcp_listener") -> + [ {"type", t(tcp)} + , {"tcp", ref("tcp_opts")} + , {"ssl", ref("ssl_opts")} + ] ++ mqtt_listener(); -fields("ssl_listener") -> - [ {"$name", ref("ssl_listener_settings")}]; +fields("mqtt_ws_listener") -> + [ {"type", t(ws)} + , {"tcp", ref("tcp_opts")} + , {"ssl", ref("ssl_opts")} + , {"websocket", ref("ws_opts")} + ] ++ mqtt_listener(); -fields("ws_listener") -> - [ {"$name", ref("ws_listener_settings")}]; +fields("mqtt_quic_listener") -> + [ {"enabled", t(boolean(), undefined, true)} + , {"type", t(quic)} + , {"certfile", t(string(), undefined, undefined)} + , {"keyfile", t(string(), undefined, undefined)} + , {"ciphers", t(comma_separated_list(), undefined, "TLS_AES_256_GCM_SHA384," + "TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256")} + , {"idle_timeout", t(duration(), undefined, "15s")} + ] ++ base_listener(); -fields("wss_listener") -> - [ {"$name", ref("wss_listener_settings")}]; +fields("ws_opts") -> + [ {"mqtt_path", t(string(), undefined, "/mqtt")} + , {"mqtt_piggyback", t(union(single, multiple), undefined, multiple)} + , {"compress", t(boolean(), undefined, false)} + , {"idle_timeout", t(duration(), undefined, "15s")} + , {"max_frame_size", maybe_infinity(integer())} + , {"fail_if_no_subprotocol", t(boolean(), undefined, true)} + , {"supported_subprotocols", t(comma_separated_list(), undefined, + "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5")} + , {"check_origin_enable", t(boolean(), undefined, false)} + , {"allow_origin_absence", t(boolean(), undefined, true)} + , {"check_origins", t(hoconsc:array(binary()), undefined, [])} + , {"proxy_address_header", t(string(), undefined, "x-forwarded-for")} + , {"proxy_port_header", t(string(), undefined, "x-forwarded-port")} + , {"deflate_opts", ref("deflate_opts")} + ]; -fields("listener_settings") -> - [ {"endpoint", t(union(ip_port(), integer()))} - , {"acceptors", t(integer(), undefined, 8)} - , {"max_connections", t(integer(), undefined, 1024)} - , {"max_conn_rate", t(integer())} - , {"active_n", t(integer(), undefined, 100)} - , {"zone", t(string())} - , {"rate_limit", t(comma_separated_list())} - , {"access", ref("access")} - , {"proxy_protocol", t(flag())} - , {"proxy_protocol_timeout", t(duration())} +fields("tcp_opts") -> + [ {"active_n", t(integer(), undefined, 100)} , {"backlog", t(integer(), undefined, 1024)} , {"send_timeout", t(duration(), undefined, "15s")} - , {"send_timeout_close", t(flag(), undefined, true)} + , {"send_timeout_close", t(boolean(), undefined, true)} , {"recbuf", t(bytesize())} , {"sndbuf", t(bytesize())} , {"buffer", t(bytesize())} , {"high_watermark", t(bytesize(), undefined, "1MB")} - , {"tune_buffer", t(flag())} - , {"nodelay", t(boolean())} - , {"reuseaddr", t(boolean())} + , {"nodelay", t(boolean(), undefined, false)} + , {"reuseaddr", t(boolean(), undefined, true)} ]; -fields("tcp_listener_settings") -> - [ {"peer_cert_as_username", t(cn)} - , {"peer_cert_as_clientid", t(cn)} - ] ++ fields("listener_settings"); - -fields("ssl_listener_settings") -> - [ {"peer_cert_as_username", t(union([cn, dn, crt, pem, md5]))} - , {"peer_cert_as_clientid", t(union([cn, dn, crt, pem, md5]))} - ] ++ - ssl(undefined, #{handshake_timeout => "15s" - , depth => 10 - , reuse_sessions => true}) ++ fields("listener_settings"); - -fields("ws_listener_settings") -> - [ {"mqtt_path", t(string(), undefined, "/mqtt")} - , {"fail_if_no_subprotocol", t(boolean(), undefined, true)} - , {"supported_subprotocols", t(string(), undefined, "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5")} - , {"proxy_address_header", t(string(), undefined, "x-forwarded-for")} - , {"proxy_port_header", t(string(), undefined, "x-forwarded-port")} - , {"compress", t(boolean())} - , {"deflate_opts", ref("deflate_opts")} - , {"idle_timeout", t(duration())} - , {"max_frame_size", t(integer())} - , {"mqtt_piggyback", t(union(single, multiple), undefined, multiple)} - , {"check_origin_enable", t(boolean(), undefined, false)} - , {"allow_origin_absence", t(boolean(), undefined, true)} - , {"check_origins", t(comma_separated_list())} - % @fixme - ] ++ lists:keydelete("high_watermark", 1, fields("tcp_listener_settings")); - -fields("wss_listener_settings") -> - % @fixme - Ssl = ssl(undefined, #{depth => 10 - , reuse_sessions => true}) ++ fields("listener_settings"), - Settings = lists:ukeymerge(1, Ssl, fields("ws_listener_settings")), - lists:keydelete("high_watermark", 1, Settings); - -fields("access") -> - [ {"$id", t(string(), undefined, undefined)}]; +fields("ssl_opts") -> + ssl(#{handshake_timeout => "15s" + , depth => 10 + , reuse_sessions => true + , versions => default_tls_vsns() + , ciphers => default_ciphers() + }); fields("deflate_opts") -> [ {"level", t(union([none, default, best_compression, best_speed]))} - , {"mem_level", t(range(1, 9))} + , {"mem_level", t(range(1, 9), undefined, 8)} , {"strategy", t(union([default, filtered, huffman_only, rle]))} , {"server_context_takeover", t(union(takeover, no_takeover))} , {"client_context_takeover", t(union(takeover, no_takeover))} - , {"server_max_window_bits", t(integer())} - , {"client_max_window_bits", t(integer())} + , {"server_max_window_bits", t(range(8, 15), undefined, 15)} + , {"client_max_window_bits", t(range(8, 15), undefined, 15)} ]; -fields("module") -> - [ {"loaded_file", t(string(), "emqx.modules_loaded_file", undefined)} - , {"presence", ref("presence")} - , {"subscription", ref("subscription")} - , {"rewrite", ref("rewrite")} - ]; - -fields("presence") -> - [ {"qos", t(range(0, 2), undefined, 1)}]; - -fields("subscription") -> - [ {"$id", ref("subscription_settings")}]; - -fields("subscription_settings") -> - [ {"topic", t(string())} - , {"qos", t(range(0, 2), undefined, 1)} - , {"nl", t(range(0, 1), undefined, 0)} - , {"rap", t(range(0, 1), undefined, 0)} - , {"rh", t(range(0, 2), undefined, 0)} - ]; - - -fields("rewrite") -> - [ {"rule", ref("rule")} - , {"pub_rule", ref("rule")} - , {"sub_rule", ref("rule")} - ]; - -fields("rule") -> - [ {"$id", t(string())}]; - fields("plugins") -> - [ {"etc_dir", t(string(), "emqx.plugins_etc_dir", undefined)} - , {"loaded_file", t(string(), "emqx.plugins_loaded_file", undefined)} - , {"expand_plugins_dir", t(string(), "emqx.expand_plugins_dir", undefined)} + [ {"expand_plugins_dir", t(string())} ]; fields("broker") -> - [ {"sys_interval", t(duration(), "emqx.broker_sys_interval", "1m")} - , {"sys_heartbeat", t(duration(), "emqx.broker_sys_heartbeat", "30s")} - , {"enable_session_registry", t(flag(), "emqx.enable_session_registry", true)} - , {"session_locking_strategy", t(union([local, leader, quorum, all]), - "emqx.session_locking_strategy", quorum)} - , {"shared_subscription_strategy", t(union(random, round_robin), - "emqx.shared_subscription_strategy", round_robin)} - , {"shared_dispatch_ack_enabled", t(boolean(), "emqx.shared_dispatch_ack_enabled", false)} - , {"route_batch_clean", t(flag(), "emqx.route_batch_clean", true)} + [ {"sys_msg_interval", maybe_disabled(duration(), "1m")} + , {"sys_heartbeat_interval", maybe_disabled(duration(), "30s")} + , {"enable_session_registry", t(boolean(), undefined, true)} + , {"session_locking_strategy", t(union([local, leader, quorum, all]), undefined, quorum)} + , {"shared_subscription_strategy", t(union(random, round_robin), undefined, round_robin)} + , {"shared_dispatch_ack_enabled", t(boolean(), undefined, false)} + , {"route_batch_clean", t(boolean(), undefined, true)} , {"perf", ref("perf")} ]; fields("perf") -> - [ {"route_lock_type", t(union([key, tab, global]), "emqx.route_lock_type", key)} - , {"trie_compaction", t(boolean(), "emqx.trie_compaction", true)} + [ {"route_lock_type", t(union([key, tab, global]), undefined, key)} + , {"trie_compaction", t(boolean(), undefined, true)} ]; fields("sysmon") -> - [ {"long_gc", t(duration(), undefined, 0)} - , {"long_schedule", t(duration(), undefined, 240)} - , {"large_heap", t(bytesize(), undefined, "8MB")} - , {"busy_dist_port", t(boolean(), undefined, true)} - , {"busy_port", t(boolean(), undefined, false)} + [ {"vm", ref("sysmon_vm")} + , {"os", ref("sysmon_os")} ]; -fields("os_mon") -> - [ {"cpu_check_interval", t(duration_s(), undefined, 60)} +fields("sysmon_vm") -> + [ {"process_check_interval", t(duration(), undefined, "30s")} + , {"process_high_watermark", t(percent(), undefined, "80%")} + , {"process_low_watermark", t(percent(), undefined, "60%")} + , {"long_gc", maybe_disabled(duration())} + , {"long_schedule", maybe_disabled(duration(), "240ms")} + , {"large_heap", maybe_disabled(bytesize(), "32MB")} + , {"busy_dist_port", t(boolean(), undefined, true)} + , {"busy_port", t(boolean(), undefined, true)} + ]; + +fields("sysmon_os") -> + [ {"cpu_check_interval", t(duration(), undefined, "60s")} , {"cpu_high_watermark", t(percent(), undefined, "80%")} , {"cpu_low_watermark", t(percent(), undefined, "60%")} - , {"mem_check_interval", t(duration_s(), undefined, 60)} + , {"mem_check_interval", maybe_disabled(duration(), "60s")} , {"sysmem_high_watermark", t(percent(), undefined, "70%")} , {"procmem_high_watermark", t(percent(), undefined, "5%")} ]; -fields("vm_mon") -> - [ {"check_interval", t(duration_s(), undefined, 30)} - , {"process_high_watermark", t(percent(), undefined, "80%")} - , {"process_low_watermark", t(percent(), undefined, "60%")} - ]; - fields("alarm") -> - [ {"actions", t(comma_separated_list(), undefined, "log,publish")} + [ {"actions", t(hoconsc:array(atom()), undefined, [log, publish])} , {"size_limit", t(integer(), undefined, 1000)} - , {"validity_period", t(duration_s(), undefined, "24h")} + , {"validity_period", t(duration(), undefined, "24h")} ]; -fields(ExtraField) -> - Mod = list_to_atom(ExtraField++"_schema"), - Mod:fields(ExtraField). +fields(FieldName) -> + ?MODULE:extra_schema_fields(FieldName). -translations() -> ["ekka", "vm_args", "gen_rpc", "kernel", "emqx"]. +-ifndef(EMQX_EXT_SCHEMAS). +%% Function extra_schema_fields/1 only terminates with explicit exception +-dialyzer([{nowarn_function, [extra_schema_fields/1]}]). +extra_schema_fields(FieldName) -> error({unknown_field, FieldName}). +-else. +extra_schema_fields(FieldName) -> + {_, Mod} = lists:keyfind(FieldName, 1, ?EMQX_EXT_SCHEMAS), + Mod:fields(FieldName). +-endif. + +mqtt_listener() -> + base_listener() ++ + [ {"access_rules", t(hoconsc:array(string()))} + , {"proxy_protocol", t(boolean(), undefined, false)} + , {"proxy_protocol_timeout", t(duration())} + ]. + +base_listener() -> + [ {"bind", t(union(ip_port(), integer()))} + , {"acceptors", t(integer(), undefined, 16)} + , {"max_connections", maybe_infinity(integer(), infinity)} + , {"rate_limit", ref("rate_limit")} + ]. + +translations() -> ["ekka", "kernel", "emqx"]. translation("ekka") -> [ {"cluster_discovery", fun tr_cluster__discovery/1}]; -translation("vm_args") -> - [ {"+zdbbl", fun tr_zdbbl/1} - , {"-heart", fun tr_heart/1}]; - -translation("gen_rpc") -> - [ {"tcp_client_num", fun tr_tcp_client_num/1} - , {"tcp_client_port", fun tr_tcp_client_port/1}]; - translation("kernel") -> [ {"logger_level", fun tr_logger_level/1} , {"logger", fun tr_logger/1}]; translation("emqx") -> - [ {"flapping_detect_policy", fun tr_flapping_detect_policy/1} - , {"zones", fun tr_zones/1} - , {"listeners", fun tr_listeners/1} - , {"modules", fun tr_modules/1} - , {"sysmon", fun tr_sysmon/1} - , {"os_mon", fun tr_os_mon/1} - , {"vm_mon", fun tr_vm_mon/1} - , {"alarm", fun tr_alarm/1} + [ {"config_files", fun tr_config_files/1} ]. +tr_config_files(Conf) -> + case conf_get("emqx.config_files", Conf) of + [_ | _] = Files -> + Files; + _ -> + case os:getenv("RUNNER_ETC_DIR") of + false -> + [filename:join([code:lib_dir(emqx), "etc", "emqx.conf"])]; + Dir -> + [filename:join([Dir, "emqx.conf"])] + end + end. + tr_cluster__discovery(Conf) -> - Strategy = conf_get("cluster.discovery", Conf), + Strategy = conf_get("cluster.discovery_strategy", Conf), {Strategy, filter(options(Strategy, Conf))}. -tr_heart(Conf) -> - case conf_get("node.heartbeat", Conf) of - true -> ""; - "on" -> ""; - _ -> undefined - end. - -%% @doc http://www.erlang.org/doc/man/erl.html#%2bzdbbl -node__dist_buffer_size(type) -> bytesize(); -node__dist_buffer_size(validator) -> - fun(ZDBBL) -> - case ZDBBL >= 1024 andalso ZDBBL =< 2147482624 of - true -> - ok; - false -> - {error, "must be between 1KB and 2097151KB"} - end - end; -node__dist_buffer_size(_) -> undefined. - -tr_zdbbl(Conf) -> - case conf_get("node.dist_buffer_size", Conf) of - undefined -> undefined; - X when is_integer(X) -> ceiling(X / 1024); %% Bytes to Kilobytes; - _ -> undefined - end. - -%% Force client to use server listening port, because we do no provide -%% per-node listening port manual mapping from configs. -%% i.e. all nodes in the cluster should agree to the same -%% listening port number. -tr_tcp_client_num(Conf) -> - case conf_get("rpc.tcp_client_num", Conf) of - 0 -> max(1, erlang:system_info(schedulers) div 2); - V -> V - end. - -tr_tcp_client_port(Conf) -> - conf_get("rpc.tcp_server_port", Conf). - -tr_logger_level(Conf) -> conf_get("log.level", Conf). +tr_logger_level(Conf) -> conf_get("log.primary_level", Conf). tr_logger(Conf) -> - LogTo = conf_get("log.to", Conf), - LogLevel = conf_get("log.level", Conf), - LogType = case conf_get("log.rotation.enable", Conf) of - true -> wrap; - _ -> halt - end, CharsLimit = case conf_get("log.chars_limit", Conf) of - -1 -> unlimited; + infinity -> unlimited; V -> V end, SingleLine = conf_get("log.single_line", Conf), FmtName = conf_get("log.formatter", Conf), Formatter = formatter(FmtName, CharsLimit, SingleLine), - BurstLimit = conf_get("log.burst_limit", Conf), - {BustLimitOn, {MaxBurstCount, TimeWindow}} = burst_limit(BurstLimit), - FileConf = fun (Filename) -> - BasicConf = - #{type => LogType, - file => filename:join(conf_get("log.dir", Conf), Filename), - max_no_files => conf_get("log.rotation.count", Conf), - sync_mode_qlen => conf_get("log.sync_mode_qlen", Conf), - drop_mode_qlen => conf_get("log.drop_mode_qlen", Conf), - flush_qlen => conf_get("log.flush_qlen", Conf), - overload_kill_enable => conf_get("log.overload_kill", Conf), - overload_kill_qlen => conf_get("log.overload_kill_qlen", Conf), - overload_kill_mem_size => conf_get("log.overload_kill_mem_size", Conf), - overload_kill_restart_after => conf_get("log.overload_kill_restart_after", Conf), - burst_limit_enable => BustLimitOn, - burst_limit_max_count => MaxBurstCount, - burst_limit_window_time => TimeWindow - }, - MaxNoBytes = case LogType of - wrap -> conf_get("log.rotation.size", Conf); - halt -> conf_get("log.size", Conf) - end, - BasicConf#{max_no_bytes => MaxNoBytes} end, - + BasicConf = #{ + sync_mode_qlen => conf_get("log.sync_mode_qlen", Conf), + drop_mode_qlen => conf_get("log.drop_mode_qlen", Conf), + flush_qlen => conf_get("log.flush_qlen", Conf), + overload_kill_enable => conf_get("log.overload_kill.enable", Conf), + overload_kill_qlen => conf_get("log.overload_kill.qlen", Conf), + overload_kill_mem_size => conf_get("log.overload_kill.mem_size", Conf), + overload_kill_restart_after => conf_get("log.overload_kill.restart_after", Conf), + burst_limit_enable => conf_get("log.burst_limit.enable", Conf), + burst_limit_max_count => conf_get("log.burst_limit.max_count", Conf), + burst_limit_window_time => conf_get("log.burst_limit.window_time", Conf) + }, Filters = case conf_get("log.supervisor_reports", Conf) of error -> [{drop_progress_reports, {fun logger_filters:progress/2, stop}}]; progress -> [] end, - %% For the default logger that outputs to console - DefaultHandler = - if LogTo =:= console orelse LogTo =:= both -> - [{handler, console, logger_std_h, - #{level => LogLevel, - config => #{type => standard_io}, - formatter => Formatter, - filters => Filters - } - }]; + ConsoleHandler = + case conf_get("log.console_handler.enable", Conf) of true -> - [{handler, default, undefined}] - end, - - %% For the file logger - FileHandler = - if LogTo =:= file orelse LogTo =:= both -> - [{handler, file, logger_disk_log_h, - #{level => LogLevel, - config => FileConf(conf_get("log.file", Conf)), + [{handler, console, logger_std_h, #{ + level => conf_get("log.console_handler.level", Conf), + config => BasicConf#{type => standard_io}, formatter => Formatter, - filesync_repeat_interval => no_repeat, filters => Filters }}]; - true -> [] + false -> [] end, - - AdditionalLogFiles = additional_log_files(Conf), - AdditionalHandlers = - [{handler, list_to_atom("file_for_"++Level), logger_disk_log_h, - #{level => list_to_atom(Level), - config => FileConf(Filename), + %% For the file logger + FileHandlers = + [{handler, binary_to_atom(HandlerName, latin1), logger_disk_log_h, #{ + level => conf_get("level", SubConf), + config => BasicConf#{ + type => case conf_get("rotation.enable", SubConf) of + true -> wrap; + _ -> halt + end, + file => conf_get("file", SubConf), + max_no_files => conf_get("rotation.count", SubConf), + max_no_bytes => conf_get("max_size", SubConf) + }, formatter => Formatter, - filesync_repeat_interval => no_repeat}} - || {Level, Filename} <- AdditionalLogFiles], + filters => Filters, + filesync_repeat_interval => no_repeat + }} + || {HandlerName, SubConf} <- maps:to_list(conf_get("log.file_handlers", Conf, #{}))], - DefaultHandler ++ FileHandler ++ AdditionalHandlers. - -tr_flapping_detect_policy(Conf) -> - [Threshold, Duration, Interval] = conf_get("acl.flapping_detect_policy", Conf), - ParseDuration = fun(S, F) -> - case F(S) of - {ok, I} -> I; - {error, Reason} -> error({duration, Reason}) - end end, - #{threshold => list_to_integer(Threshold), - duration => ParseDuration(Duration, fun to_duration/1), - banned_interval => ParseDuration(Interval, fun to_duration_s/1) - }. - -tr_zones(Conf) -> - Names = lists:usort(keys("zone", Conf)), - lists:foldl( - fun(Name, Zones) -> - Zone = keys("zone." ++ Name, Conf), - Mapped = lists:flatten([map_zones(K, conf_get(["zone", Name, K], Conf)) || K <- Zone]), - [{list_to_atom(Name), lists:filter(fun ({K, []}) when K =:= ratelimit; K =:= quota -> false; - ({_, undefined}) -> false; - (_) -> true end, Mapped)} | Zones] - end, [], Names). - -tr_listeners(Conf) -> - Atom = fun(undefined) -> undefined; - (B) when is_binary(B)-> binary_to_atom(B, utf8); - (S) when is_list(S) -> list_to_atom(S) end, - - Access = fun(S) -> - [A, CIDR] = string:tokens(S, " "), - {list_to_atom(A), case CIDR of "all" -> all; _ -> CIDR end} - end, - - AccOpts = fun(Prefix) -> - case keys(Prefix ++ ".access", Conf) of - [] -> []; - Ids -> - [{access_rules, [Access(conf_get(Prefix ++ ".access." ++ Id, Conf)) || Id <- Ids]}] - end end, - - RateLimit = fun(undefined) -> - undefined; - ([L, D]) -> - Limit = case to_bytesize(L) of - {ok, I0} -> I0; - {error, R0} -> error({bytesize, R0}) - end, - Duration = case to_duration_s(D) of - {ok, I1} -> I1; - {error, R1} -> error({duration, R1}) - end, - {Limit, Duration} - end, - - CheckOrigin = fun(S) -> [ list_to_binary(string:trim(O)) || O <- S] end, - - WsOpts = fun(Prefix) -> - case conf_get(Prefix ++ ".check_origins", Conf) of - undefined -> undefined; - Rules -> lists:flatten(CheckOrigin(Rules)) - end - end, - - LisOpts = fun(Prefix) -> - filter([{acceptors, conf_get(Prefix ++ ".acceptors", Conf)}, - {mqtt_path, conf_get(Prefix ++ ".mqtt_path", Conf)}, - {max_connections, conf_get(Prefix ++ ".max_connections", Conf)}, - {max_conn_rate, conf_get(Prefix ++ ".max_conn_rate", Conf)}, - {active_n, conf_get(Prefix ++ ".active_n", Conf)}, - {tune_buffer, conf_get(Prefix ++ ".tune_buffer", Conf)}, - {zone, Atom(conf_get(Prefix ++ ".zone", Conf))}, - {rate_limit, RateLimit(conf_get(Prefix ++ ".rate_limit", Conf))}, - {proxy_protocol, conf_get(Prefix ++ ".proxy_protocol", Conf)}, - {proxy_address_header, list_to_binary(string:lowercase(conf_get(Prefix ++ ".proxy_address_header", Conf, <<"">>)))}, - {proxy_port_header, list_to_binary(string:lowercase(conf_get(Prefix ++ ".proxy_port_header", Conf, <<"">>)))}, - {proxy_protocol_timeout, conf_get(Prefix ++ ".proxy_protocol_timeout", Conf)}, - {fail_if_no_subprotocol, conf_get(Prefix ++ ".fail_if_no_subprotocol", Conf)}, - {supported_subprotocols, string:tokens(conf_get(Prefix ++ ".supported_subprotocols", Conf, ""), ", ")}, - {peer_cert_as_username, conf_get(Prefix ++ ".peer_cert_as_username", Conf)}, - {peer_cert_as_clientid, conf_get(Prefix ++ ".peer_cert_as_clientid", Conf)}, - {compress, conf_get(Prefix ++ ".compress", Conf)}, - {idle_timeout, conf_get(Prefix ++ ".idle_timeout", Conf)}, - {max_frame_size, conf_get(Prefix ++ ".max_frame_size", Conf)}, - {mqtt_piggyback, conf_get(Prefix ++ ".mqtt_piggyback", Conf)}, - {check_origin_enable, conf_get(Prefix ++ ".check_origin_enable", Conf)}, - {allow_origin_absence, conf_get(Prefix ++ ".allow_origin_absence", Conf)}, - {check_origins, WsOpts(Prefix)} | AccOpts(Prefix)]) - end, - DeflateOpts = fun(Prefix) -> - filter([{level, conf_get(Prefix ++ ".deflate_opts.level", Conf)}, - {mem_level, conf_get(Prefix ++ ".deflate_opts.mem_level", Conf)}, - {strategy, conf_get(Prefix ++ ".deflate_opts.strategy", Conf)}, - {server_context_takeover, conf_get(Prefix ++ ".deflate_opts.server_context_takeover", Conf)}, - {client_context_takeover, conf_get(Prefix ++ ".deflate_opts.client_context_takeover", Conf)}, - {server_max_windows_bits, conf_get(Prefix ++ ".deflate_opts.server_max_window_bits", Conf)}, - {client_max_windows_bits, conf_get(Prefix ++ ".deflate_opts.client_max_window_bits", Conf)}]) - end, - TcpOpts = fun(Prefix) -> - filter([{backlog, conf_get(Prefix ++ ".backlog", Conf)}, - {send_timeout, conf_get(Prefix ++ ".send_timeout", Conf)}, - {send_timeout_close, conf_get(Prefix ++ ".send_timeout_close", Conf)}, - {recbuf, conf_get(Prefix ++ ".recbuf", Conf)}, - {sndbuf, conf_get(Prefix ++ ".sndbuf", Conf)}, - {buffer, conf_get(Prefix ++ ".buffer", Conf)}, - {high_watermark, conf_get(Prefix ++ ".high_watermark", Conf)}, - {nodelay, conf_get(Prefix ++ ".nodelay", Conf, true)}, - {reuseaddr, conf_get(Prefix ++ ".reuseaddr", Conf)}]) - end, - - SslOpts = fun(Prefix) -> - Opts = tr_ssl(Prefix, Conf), - case lists:keyfind(ciphers, 1, Opts) of - false -> - error(Prefix ++ ".ciphers or " ++ Prefix ++ ".psk_ciphers is absent"); - _ -> - Opts - end end, - - TcpListeners = fun(Type, Name) -> - Prefix = string:join(["listener", Type, Name], "."), - ListenOnN = case conf_get(Prefix ++ ".endpoint", Conf) of - undefined -> []; - ListenOn -> ListenOn - end, - [#{ proto => Atom(Type) - , name => Name - , listen_on => ListenOnN - , opts => [ {deflate_options, DeflateOpts(Prefix)} - , {tcp_options, TcpOpts(Prefix)} - | LisOpts(Prefix) - ] - } - ] - end, - SslListeners = fun(Type, Name) -> - Prefix = string:join(["listener", Type, Name], "."), - case conf_get(Prefix ++ ".endpoint", Conf) of - undefined -> - []; - ListenOn -> - [#{ proto => Atom(Type) - , name => Name - , listen_on => ListenOn - , opts => [ {deflate_options, DeflateOpts(Prefix)} - , {tcp_options, TcpOpts(Prefix)} - , {ssl_options, SslOpts(Prefix)} - | LisOpts(Prefix) - ] - } - ] - end end, - - - lists:flatten([TcpListeners("tcp", Name) || Name <- keys("listener.tcp", Conf)] - ++ [TcpListeners("ws", Name) || Name <- keys("listener.ws", Conf)] - ++ [SslListeners("ssl", Name) || Name <- keys("listener.ssl", Conf)] - ++ [SslListeners("wss", Name) || Name <- keys("listener.wss", Conf)]). - -tr_modules(Conf) -> - Subscriptions = fun() -> - List = keys("module.subscription", Conf), - TopicList = [{N, conf_get(["module", "subscription", N, "topic"], Conf)}|| N <- List], - [{list_to_binary(T), #{ qos => conf_get("module.subscription." ++ N ++ ".qos", Conf, 0), - nl => conf_get("module.subscription." ++ N ++ ".nl", Conf, 0), - rap => conf_get("module.subscription." ++ N ++ ".rap", Conf, 0), - rh => conf_get("module.subscription." ++ N ++ ".rh", Conf, 0) - }} || {N, T} <- TopicList] - end, - Rewrites = fun() -> - Rules = keys("module.rewrite.rule", Conf), - PubRules = keys("module.rewrite.pub_rule", Conf), - SubRules = keys("module.rewrite.sub_rule", Conf), - TotalRules = - [ {["module", "rewrite", "pub", "rule", R], conf_get(["module.rewrite.rule", R], Conf)} || R <- Rules] ++ - [ {["module", "rewrite", "pub", "rule", R], conf_get(["module.rewrite.pub_rule", R], Conf)} || R <- PubRules] ++ - [ {["module", "rewrite", "sub", "rule", R], conf_get(["module.rewrite.rule", R], Conf)} || R <- Rules] ++ - [ {["module", "rewrite", "sub", "rule", R], conf_get(["module.rewrite.sub_rule", R], Conf)} || R <- SubRules], - lists:map(fun({[_, "rewrite", PubOrSub, "rule", _], Rule}) -> - [Topic, Re, Dest] = string:tokens(Rule, " "), - {rewrite, list_to_atom(PubOrSub), list_to_binary(Topic), list_to_binary(Re), list_to_binary(Dest)} - end, TotalRules) - end, - lists:append([ - [{emqx_mod_presence, [{qos, conf_get("module.presence.qos", Conf, 1)}]}], - [{emqx_mod_subscription, Subscriptions()}], - [{emqx_mod_rewrite, Rewrites()}], - [{emqx_mod_topic_metrics, []}], - [{emqx_mod_delayed, []}] - ]). - -tr_sysmon(Conf) -> - Keys = maps:to_list(conf_get("sysmon", Conf, #{})), - [{binary_to_atom(K, utf8), maps:get(value, V)} || {K, V} <- Keys]. - -tr_os_mon(Conf) -> - [{cpu_check_interval, conf_get("os_mon.cpu_check_interval", Conf)} - , {cpu_high_watermark, conf_get("os_mon.cpu_high_watermark", Conf) * 100} - , {cpu_low_watermark, conf_get("os_mon.cpu_low_watermark", Conf) * 100} - , {mem_check_interval, conf_get("os_mon.mem_check_interval", Conf)} - , {sysmem_high_watermark, conf_get("os_mon.sysmem_high_watermark", Conf) * 100} - , {procmem_high_watermark, conf_get("os_mon.procmem_high_watermark", Conf) * 100} - ]. - -tr_vm_mon(Conf) -> - [ {check_interval, conf_get("vm_mon.check_interval", Conf)} - , {process_high_watermark, conf_get("vm_mon.process_high_watermark", Conf) * 100} - , {process_low_watermark, conf_get("vm_mon.process_low_watermark", Conf) * 100} - ]. - -tr_alarm(Conf) -> - [ {actions, [list_to_atom(Action) || Action <- conf_get("alarm.actions", Conf)]} - , {size_limit, conf_get("alarm.size_limit", Conf)} - , {validity_period, conf_get("alarm.validity_period", Conf)} - ]. + [{handler, default, undefined}] ++ ConsoleHandler ++ FileHandlers. %% helpers - -options(static, Conf) -> - [{seeds, [list_to_atom(S) || S <- conf_get("cluster.static.seeds", Conf, "")]}]; -options(mcast, Conf) -> - {ok, Addr} = inet:parse_address(conf_get("cluster.mcast.addr", Conf)), - {ok, Iface} = inet:parse_address(conf_get("cluster.mcast.iface", Conf)), - Ports = [list_to_integer(S) || S <- conf_get("cluster.mcast.ports", Conf)], - [{addr, Addr}, {ports, Ports}, {iface, Iface}, - {ttl, conf_get("cluster.mcast.ttl", Conf, 1)}, - {loop, conf_get("cluster.mcast.loop", Conf, true)}]; -options(dns, Conf) -> - [{name, conf_get("cluster.dns.name", Conf)}, - {app, conf_get("cluster.dns.app", Conf)}]; -options(etcd, Conf) -> - Namespace = "cluster.etcd.ssl", - SslOpts = fun(C) -> - Options = keys(Namespace, C), - lists:map(fun(Key) -> {list_to_atom(Key), conf_get([Namespace, Key], Conf)} end, Options) end, - [{server, conf_get("cluster.etcd.server", Conf)}, - {prefix, conf_get("cluster.etcd.prefix", Conf, "emqxcl")}, - {node_ttl, conf_get("cluster.etcd.node_ttl", Conf, 60)}, - {ssl_options, filter(SslOpts(Conf))}]; -options(k8s, Conf) -> - [{apiserver, conf_get("cluster.k8s.apiserver", Conf)}, - {service_name, conf_get("cluster.k8s.service_name", Conf)}, - {address_type, conf_get("cluster.k8s.address_type", Conf, ip)}, - {app_name, conf_get("cluster.k8s.app_name", Conf)}, - {namespace, conf_get("cluster.k8s.namespace", Conf)}, - {suffix, conf_get("cluster.k8s.suffix", Conf, "")}]; -options(manual, _Conf) -> - []. - formatter(json, CharsLimit, SingleLine) -> {emqx_logger_jsonfmt, #{chars_limit => CharsLimit, @@ -905,161 +620,10 @@ formatter(text, CharsLimit, SingleLine) -> single_line => SingleLine }}. -burst_limit(["disabled"]) -> - {false, {20000, 1000}}; -burst_limit([Count, Window]) -> - {true, {list_to_integer(Count), - case to_duration(Window) of - {ok, I} -> I; - {error, R} -> error({duration, R}) - end}}. - -%% For creating additional log files for specific log levels. -additional_log_files(Conf) -> - LogLevel = ["debug", "info", "notice", "warning", - "error", "critical", "alert", "emergency"], - additional_log_files(Conf, LogLevel, []). - -additional_log_files(_Conf, [], Acc) -> - Acc; -additional_log_files(Conf, [L | More], Acc) -> - case conf_get(["log", L, "file"], Conf) of - undefined -> additional_log_files(Conf, More, Acc); - F -> additional_log_files(Conf, More, [{L, F} | Acc]) - end. - -rate_limit_byte_dur([L, D]) -> - Limit = case to_bytesize(L) of - {ok, I0} -> I0; - {error, R0} -> error({bytesize, R0}) - end, - Duration = case to_duration_s(D) of - {ok, I1} -> I1; - {error, R1} -> error({duration, R1}) - end, - {Limit, Duration}. - -rate_limit_num_dur([L, D]) -> - Limit = case string:to_integer(L) of - {Int, []} when is_integer(Int) -> Int; - _ -> error("failed to parse bytesize string") - end, - Duration = case to_duration_s(D) of - {ok, I} -> I; - {error, Reason} -> error(Reason) - end, - {Limit, Duration}. - -map_zones(_, undefined) -> - {undefined, undefined}; -map_zones("force_gc_policy", [Count, Bytes]) -> - GcPolicy = case to_bytesize(Bytes) of - {error, Reason} -> - error({bytesize, Reason}); - {ok, Bytes1} -> - #{bytes => Bytes1, - count => list_to_integer(Count)} - end, - {force_gc_policy, GcPolicy}; -map_zones("force_shutdown_policy", ["default"]) -> - WordSize = erlang:system_info(wordsize), - {DefaultLen, DefaultSize} = - case WordSize of - 8 -> % arch_64 - {10000, hocon_postprocess:bytesize("64MB")}; - 4 -> % arch_32 - {1000, hocon_postprocess:bytesize("32MB")} - end, - {force_shutdown_policy, #{message_queue_len => DefaultLen, - max_heap_size => DefaultSize div WordSize - }}; -map_zones("force_shutdown_policy", [Len, Siz]) -> - WordSize = erlang:system_info(wordsize), - MaxSiz = case WordSize of - 8 -> % arch_64 - (1 bsl 59) - 1; - 4 -> % arch_32 - (1 bsl 27) - 1 - end, - ShutdownPolicy = - case to_bytesize(Siz) of - {error, Reason} -> - error(Reason); - {ok, Siz1} when Siz1 > MaxSiz -> - error(io_lib:format("force_shutdown_policy: heap-size ~s is too large", [Siz])); - {ok, Siz1} -> - #{message_queue_len => list_to_integer(Len), - max_heap_size => Siz1 div WordSize} - end, - {force_shutdown_policy, ShutdownPolicy}; -map_zones("mqueue_priorities", Val) -> - case Val of - ["none"] -> {mqueue_priorities, none}; % NO_PRIORITY_TABLE - _ -> - MqueuePriorities = lists:foldl(fun(T, Acc) -> - %% NOTE: space in "= " is intended - [Topic, Prio] = string:tokens(T, "= "), - P = list_to_integer(Prio), - (P < 0 orelse P > 255) andalso error({bad_priority, Topic, Prio}), - maps:put(iolist_to_binary(Topic), P, Acc) - end, #{}, Val), - {mqueue_priorities, MqueuePriorities} - end; -map_zones("mountpoint", Val) -> - {mountpoint, iolist_to_binary(Val)}; -map_zones("response_information", Val) -> - {response_information, iolist_to_binary(Val)}; -map_zones("rate_limit", Conf) -> - Messages = case conf_get("conn_messages_in", #{value => Conf}) of - undefined -> - []; - M -> - [{conn_messages_in, rate_limit_num_dur(M)}] - end, - Bytes = case conf_get("conn_bytes_in", #{value => Conf}) of - undefined -> - []; - B -> - [{conn_bytes_in, rate_limit_byte_dur(B)}] - end, - {ratelimit, Messages ++ Bytes}; -map_zones("conn_congestion", Conf) -> - Alarm = case conf_get("alarm", #{value => Conf}) of - undefined -> - []; - A -> - [{conn_congestion_alarm_enabled, A}] - end, - MinAlarm = case conf_get("min_alarm_sustain_duration", #{value => Conf}) of - undefined -> - []; - M -> - [{conn_congestion_min_alarm_sustain_duration, M}] - end, - Alarm ++ MinAlarm; -map_zones("quota", Conf) -> - Conn = case conf_get("conn_messages_routing", #{value => Conf}) of - undefined -> - []; - C -> - [{conn_messages_routing, rate_limit_num_dur(C)}] - end, - Overall = case conf_get("overall_messages_routing", #{value => Conf}) of - undefined -> - []; - O -> - [{overall_messages_routing, rate_limit_num_dur(O)}] - end, - {quota, Conn ++ Overall}; -map_zones(Opt, Val) -> - {list_to_atom(Opt), Val}. - - %% utils - -spec(conf_get(string() | [string()], hocon:config()) -> term()). conf_get(Key, Conf) -> - V = hocon_schema:deep_get(Key, Conf, value), + V = hocon_schema:get_value(Key, Conf), case is_binary(V) of true -> binary_to_list(V); @@ -1068,7 +632,7 @@ conf_get(Key, Conf) -> end. conf_get(Key, Conf, Default) -> - V = hocon_schema:deep_get(Key, Conf, value, Default), + V = hocon_schema:get_value(Key, Conf, Default), case is_binary(V) of true -> binary_to_list(V); @@ -1080,104 +644,67 @@ filter(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined]. %% generate a ssl field. -%% ssl("emqx", #{"verify" => verify_peer}) will return -%% [ {"cacertfile", t(string(), "emqx.cacertfile", undefined)} -%% , {"certfile", t(string(), "emqx.certfile", undefined)} -%% , {"keyfile", t(string(), "emqx.keyfile", undefined)} -%% , {"verify", t(union(verify_peer, verify_none), "emqx.verify", verify_peer)} -%% , {"server_name_indication", "emqx.server_name_indication", undefined)} -%% ... -ssl(Mapping, Defaults) -> - M = fun (Field) -> - case (Mapping) of - undefined -> undefined; - _ -> Mapping ++ "." ++ Field - end end, - D = fun (Field) -> maps:get(list_to_atom(Field), Defaults, undefined) end, - [ {"enable", t(flag(), M("enable"), D("enable"))} - , {"cacertfile", t(string(), M("cacertfile"), D("cacertfile"))} - , {"certfile", t(string(), M("certfile"), D("certfile"))} - , {"keyfile", t(string(), M("keyfile"), D("keyfile"))} - , {"verify", t(union(verify_peer, verify_none), M("verify"), D("verify"))} - , {"fail_if_no_peer_cert", t(boolean(), M("fail_if_no_peer_cert"), D("fail_if_no_peer_cert"))} - , {"secure_renegotiate", t(flag(), M("secure_renegotiate"), D("secure_renegotiate"))} - , {"reuse_sessions", t(flag(), M("reuse_sessions"), D("reuse_sessions"))} - , {"honor_cipher_order", t(flag(), M("honor_cipher_order"), D("honor_cipher_order"))} - , {"handshake_timeout", t(duration(), M("handshake_timeout"), D("handshake_timeout"))} - , {"depth", t(integer(), M("depth"), D("depth"))} - , {"password", hoconsc:t(string(), #{mapping => M("key_password"), - default => D("key_password"), +%% ssl(#{"verify" => verify_peer}) will return: +%% [ {"cacertfile", t(string(), undefined, undefined)} +%% , {"certfile", t(string(), undefined, undefined)} +%% , {"keyfile", t(string(), undefined, undefined)} +%% , {"verify", t(union(verify_peer, verify_none), undefined, verify_peer)} +%% , {"server_name_indication", undefined, undefined)} +%% ...] +ssl(Defaults) -> + D = fun (Field) -> maps:get(to_atom(Field), Defaults, undefined) end, + [ {"enable", t(boolean(), undefined, D("enable"))} + , {"cacertfile", t(string(), undefined, D("cacertfile"))} + , {"certfile", t(string(), undefined, D("certfile"))} + , {"keyfile", t(string(), undefined, D("keyfile"))} + , {"verify", t(union(verify_peer, verify_none), undefined, D("verify"))} + , {"fail_if_no_peer_cert", t(boolean(), undefined, D("fail_if_no_peer_cert"))} + , {"secure_renegotiate", t(boolean(), undefined, D("secure_renegotiate"))} + , {"reuse_sessions", t(boolean(), undefined, D("reuse_sessions"))} + , {"honor_cipher_order", t(boolean(), undefined, D("honor_cipher_order"))} + , {"handshake_timeout", t(duration(), undefined, D("handshake_timeout"))} + , {"depth", t(integer(), undefined, D("depth"))} + , {"password", hoconsc:t(string(), #{default => D("key_password"), sensitive => true })} - , {"dhfile", t(string(), M("dhfile"), D("dhfile"))} - , {"server_name_indication", t(union(disable, string()), M("server_name_indication"), + , {"dhfile", t(string(), undefined, D("dhfile"))} + , {"server_name_indication", t(union(disable, string()), undefined, D("server_name_indication"))} - , {"tls_versions", t(comma_separated_list(), M("tls_versions"), D("tls_versions"))} - , {"ciphers", t(comma_separated_list(), M("ciphers"), D("ciphers"))} - , {"psk_ciphers", t(comma_separated_list(), M("ciphers"), D("ciphers"))}]. + , {"versions", #{ type => list(atom()) + , default => maps:get(versions, Defaults, default_tls_vsns()) + , converter => fun (Vsns) -> [tls_vsn(V) || V <- Vsns] end + }} + , {"ciphers", t(hoconsc:array(string()), undefined, D("ciphers"))} + , {"user_lookup_fun", t(any(), undefined, {fun emqx_psk:lookup/3, <<>>})} + ]. -tr_ssl(Field, Conf) -> - Versions = case conf_get([Field, "tls_versions"], Conf) of - undefined -> undefined; - Vs -> [list_to_existing_atom(V) || V <- Vs] - end, - TLSCiphers = conf_get([Field, "ciphers"], Conf), - PSKCiphers = conf_get([Field, "psk_ciphers"], Conf), - Ciphers = ciphers(TLSCiphers, PSKCiphers, Field), - case emqx_schema:conf_get([Field, "enable"], Conf) of - X when X =:= true orelse X =:= undefined -> - filter([{versions, Versions}, - {ciphers, Ciphers}, - {user_lookup_fun, user_lookup_fun(PSKCiphers)}, - {handshake_timeout, conf_get([Field, "handshake_timeout"], Conf)}, - {depth, conf_get([Field, "depth"], Conf)}, - {password, conf_get([Field, "key_password"], Conf)}, - {dhfile, conf_get([Field, "dhfile"], Conf)}, - {keyfile, emqx_schema:conf_get([Field, "keyfile"], Conf)}, - {certfile, emqx_schema:conf_get([Field, "certfile"], Conf)}, - {cacertfile, emqx_schema:conf_get([Field, "cacertfile"], Conf)}, - {verify, emqx_schema:conf_get([Field, "verify"], Conf)}, - {fail_if_no_peer_cert, conf_get([Field, "fail_if_no_peer_cert"], Conf)}, - {secure_renegotiate, conf_get([Field, "secure_renegotiate"], Conf)}, - {reuse_sessions, conf_get([Field, "reuse_sessions"], Conf)}, - {honor_cipher_order, conf_get([Field, "honor_cipher_order"], Conf)}, - {server_name_indication, emqx_schema:conf_get([Field, "server_name_indication"], Conf)} - ]); - _ -> - [] - end. +%% on erl23.2.7.2-emqx-2, sufficient_crypto_support('tlsv1.3') -> false +default_tls_vsns() -> [<<"tlsv1.2">>, <<"tlsv1.1">>, <<"tlsv1">>]. +tls_vsn(<<"tlsv1.3">>) -> 'tlsv1.3'; +tls_vsn(<<"tlsv1.2">>) -> 'tlsv1.2'; +tls_vsn(<<"tlsv1.1">>) -> 'tlsv1.1'; +tls_vsn(<<"tlsv1">>) -> 'tlsv1'. -map_psk_ciphers(PSKCiphers) -> - lists:map( - fun("PSK-AES128-CBC-SHA") -> {psk, aes_128_cbc, sha}; - ("PSK-AES256-CBC-SHA") -> {psk, aes_256_cbc, sha}; - ("PSK-3DES-EDE-CBC-SHA") -> {psk, '3des_ede_cbc', sha}; - ("PSK-RC4-SHA") -> {psk, rc4_128, sha} - end, PSKCiphers). - -ciphers(undefined, undefined, _) -> - undefined; -ciphers(TLSCiphers, undefined, _) -> - TLSCiphers; -ciphers(undefined, PSKCiphers, _) -> - map_psk_ciphers(PSKCiphers); -ciphers(_, _, Field) -> - error(Field ++ ".ciphers and " ++ Field ++ ".psk_ciphers cannot be configured at the same time"). - -user_lookup_fun(undefined) -> - undefined; -user_lookup_fun(_PSKCiphers) -> - {fun emqx_psk:lookup/3, <<>>}. - -tr_password_hash(Field, Conf) -> - case emqx_schema:conf_get([Field, "password_hash"], Conf) of - [Hash] -> list_to_atom(Hash); - [Prefix, Suffix] -> {list_to_atom(Prefix), list_to_atom(Suffix)}; - [Hash, MacFun, Iterations, Dklen] -> {list_to_atom(Hash), list_to_atom(MacFun), - list_to_integer(Iterations), list_to_integer(Dklen)}; - _ -> plain - end. +default_ciphers() -> [ + "TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256", "TLS_CHACHA20_POLY1305_SHA256", + "TLS_AES_128_CCM_SHA256", "TLS_AES_128_CCM_8_SHA256", "ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-SHA384", "ECDHE-RSA-AES256-SHA384", + "ECDHE-ECDSA-DES-CBC3-SHA", "ECDH-ECDSA-AES256-GCM-SHA384", "ECDH-RSA-AES256-GCM-SHA384", + "ECDH-ECDSA-AES256-SHA384", "ECDH-RSA-AES256-SHA384", "DHE-DSS-AES256-GCM-SHA384", + "DHE-DSS-AES256-SHA256", "AES256-GCM-SHA384", "AES256-SHA256", + "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-RSA-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES128-SHA256", "ECDHE-RSA-AES128-SHA256", "ECDH-ECDSA-AES128-GCM-SHA256", + "ECDH-RSA-AES128-GCM-SHA256", "ECDH-ECDSA-AES128-SHA256", "ECDH-RSA-AES128-SHA256", + "DHE-DSS-AES128-GCM-SHA256", "DHE-DSS-AES128-SHA256", "AES128-GCM-SHA256", "AES128-SHA256", + "ECDHE-ECDSA-AES256-SHA", "ECDHE-RSA-AES256-SHA", "DHE-DSS-AES256-SHA", + "ECDH-ECDSA-AES256-SHA", "ECDH-RSA-AES256-SHA", "AES256-SHA", "ECDHE-ECDSA-AES128-SHA", + "ECDHE-RSA-AES128-SHA", "DHE-DSS-AES128-SHA", "ECDH-ECDSA-AES128-SHA", + "ECDH-RSA-AES128-SHA", "AES128-SHA" + ] ++ psk_ciphers(). +psk_ciphers() -> [ + "PSK-AES128-CBC-SHA", "PSK-AES256-CBC-SHA", "PSK-3DES-EDE-CBC-SHA", "PSK-RC4-SHA" + ]. %% @private return a list of keys in a parent field -spec(keys(string(), hocon:config()) -> [string()]). @@ -1206,11 +733,30 @@ t(Type, Mapping, Default, OverrideEnv) -> , override_env => OverrideEnv }). +t(Type, Mapping, Default, OverrideEnv, Validator) -> + hoconsc:t(Type, #{ mapping => Mapping + , default => Default + , override_env => OverrideEnv + , validator => Validator + }). + ref(Field) -> fun (type) -> Field; (_) -> undefined end. -to_flag(Str) -> - {ok, hocon_postprocess:onoff(Str)}. +maybe_disabled(T) -> + maybe_sth(disabled, T, disabled). + +maybe_disabled(T, Default) -> + maybe_sth(disabled, T, Default). + +maybe_infinity(T) -> + maybe_sth(infinity, T, infinity). + +maybe_infinity(T, Default) -> + maybe_sth(infinity, T, Default). + +maybe_sth(What, Type, Default) -> + t(union([What, Type]), undefined, Default). to_duration(Str) -> case hocon_postprocess:duration(Str) of @@ -1236,6 +782,13 @@ to_bytesize(Str) -> _ -> {error, Str} end. +to_wordsize(Str) -> + WordSize = erlang:system_info(wordsize), + case to_bytesize(Str) of + {ok, Bytes} -> {ok, Bytes div WordSize}; + Error -> Error + end. + to_percent(Str) -> {ok, hocon_postprocess:percent(Str)}. @@ -1243,7 +796,7 @@ to_comma_separated_list(Str) -> {ok, string:tokens(Str, ", ")}. to_comma_separated_atoms(Str) -> - {ok, lists:map(fun list_to_atom/1, string:tokens(Str, ", "))}. + {ok, lists:map(fun to_atom/1, string:tokens(Str, ", "))}. to_bar_separated_list(Str) -> {ok, string:tokens(Str, "| ")}. @@ -1257,3 +810,47 @@ to_ip_port(Str) -> end; _ -> {error, Str} end. + +to_erl_cipher_suite(Str) -> + case ssl:str_to_suite(Str) of + {error, Reason} -> error({invalid_cipher, Reason}); + Cipher -> Cipher + end. + +options(static, Conf) -> + [{seeds, [to_atom(S) || S <- conf_get("cluster.static.seeds", Conf, [])]}]; +options(mcast, Conf) -> + {ok, Addr} = inet:parse_address(conf_get("cluster.mcast.addr", Conf)), + {ok, Iface} = inet:parse_address(conf_get("cluster.mcast.iface", Conf)), + Ports = conf_get("cluster.mcast.ports", Conf), + [{addr, Addr}, {ports, Ports}, {iface, Iface}, + {ttl, conf_get("cluster.mcast.ttl", Conf, 1)}, + {loop, conf_get("cluster.mcast.loop", Conf, true)}]; +options(dns, Conf) -> + [{name, conf_get("cluster.dns.name", Conf)}, + {app, conf_get("cluster.dns.app", Conf)}]; +options(etcd, Conf) -> + Namespace = "cluster.etcd.ssl", + SslOpts = fun(C) -> + Options = keys(Namespace, C), + lists:map(fun(Key) -> {to_atom(Key), conf_get([Namespace, Key], Conf)} end, Options) end, + [{server, conf_get("cluster.etcd.server", Conf)}, + {prefix, conf_get("cluster.etcd.prefix", Conf, "emqxcl")}, + {node_ttl, conf_get("cluster.etcd.node_ttl", Conf, 60)}, + {ssl_options, filter(SslOpts(Conf))}]; +options(k8s, Conf) -> + [{apiserver, conf_get("cluster.k8s.apiserver", Conf)}, + {service_name, conf_get("cluster.k8s.service_name", Conf)}, + {address_type, conf_get("cluster.k8s.address_type", Conf, ip)}, + {app_name, conf_get("cluster.k8s.app_name", Conf)}, + {namespace, conf_get("cluster.k8s.namespace", Conf)}, + {suffix, conf_get("cluster.k8s.suffix", Conf, "")}]; +options(manual, _Conf) -> + []. + +to_atom(Atom) when is_atom(Atom) -> + Atom; +to_atom(Str) when is_list(Str) -> + list_to_atom(Str); +to_atom(Bin) when is_binary(Bin) -> + binary_to_atom(Bin, utf8). diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index 9463345d4..3e0f56610 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -55,7 +55,7 @@ -compile(nowarn_export_all). -endif. --export([init/2]). +-export([init/1]). -export([ info/1 , info/2 @@ -92,13 +92,11 @@ -export_type([session/0]). --import(emqx_zone, [get_env/3]). - -record(session, { %% Client’s Subscriptions. subscriptions :: map(), %% Max subscriptions allowed - max_subscriptions :: non_neg_integer(), + max_subscriptions :: non_neg_integer() | infinity, %% Upgrade QoS? upgrade_qos :: boolean(), %% Client <- Broker: QoS1/2 messages sent to the client but @@ -117,7 +115,7 @@ %% have not been completely acknowledged awaiting_rel :: map(), %% Maximum number of awaiting QoS2 messages allowed - max_awaiting_rel :: non_neg_integer(), + max_awaiting_rel :: non_neg_integer() | infinity, %% Awaiting PUBREL Timeout (Unit: millsecond) await_rel_timeout :: timeout(), %% Created at @@ -153,33 +151,39 @@ -define(DEFAULT_BATCH_N, 1000). +-type options() :: #{ max_subscriptions => non_neg_integer() + , upgrade_qos => boolean() + , retry_interval => timeout() + , max_awaiting_rel => non_neg_integer() | infinity + , await_rel_timeout => timeout() + , max_inflight => integer() + , mqueue => emqx_mqueue:options() + }. %%-------------------------------------------------------------------- %% Init a Session %%-------------------------------------------------------------------- --spec(init(emqx_types:clientinfo(), emqx_types:conninfo()) -> session()). -init(#{zone := Zone}, #{receive_maximum := MaxInflight}) -> - #session{max_subscriptions = get_env(Zone, max_subscriptions, 0), - subscriptions = #{}, - upgrade_qos = get_env(Zone, upgrade_qos, false), - inflight = emqx_inflight:new(MaxInflight), - mqueue = init_mqueue(Zone), - next_pkt_id = 1, - retry_interval = timer:seconds(get_env(Zone, retry_interval, 0)), - awaiting_rel = #{}, - max_awaiting_rel = get_env(Zone, max_awaiting_rel, 100), - await_rel_timeout = timer:seconds(get_env(Zone, await_rel_timeout, 300)), - created_at = erlang:system_time(millisecond) - }. - -%% @private init mq -init_mqueue(Zone) -> - emqx_mqueue:init(#{max_len => get_env(Zone, max_mqueue_len, 1000), - store_qos0 => get_env(Zone, mqueue_store_qos0, true), - priorities => get_env(Zone, mqueue_priorities, none), - default_priority => get_env(Zone, mqueue_default_priority, lowest) - }). +-spec(init(options()) -> session()). +init(Opts) -> + MaxInflight = maps:get(max_inflight, Opts, 1), + QueueOpts = maps:merge( + #{max_len => 1000, + store_qos0 => true + }, maps:get(mqueue, Opts, #{})), + #session{ + max_subscriptions = maps:get(max_subscriptions, Opts, infinity), + subscriptions = #{}, + upgrade_qos = maps:get(upgrade_qos, Opts, false), + inflight = emqx_inflight:new(MaxInflight), + mqueue = emqx_mqueue:init(QueueOpts), + next_pkt_id = 1, + retry_interval = maps:get(retry_interval, Opts, 30000), + awaiting_rel = #{}, + max_awaiting_rel = maps:get(max_awaiting_rel, Opts, 100), + await_rel_timeout = maps:get(await_rel_timeout, Opts, 300000), + created_at = erlang:system_time(millisecond) + }. %%-------------------------------------------------------------------- %% Info, Stats @@ -207,7 +211,7 @@ info(inflight_cnt, #session{inflight = Inflight}) -> info(inflight_max, #session{inflight = Inflight}) -> emqx_inflight:max_size(Inflight); info(retry_interval, #session{retry_interval = Interval}) -> - Interval div 1000; + Interval; info(mqueue, #session{mqueue = MQueue}) -> MQueue; info(mqueue_len, #session{mqueue = MQueue}) -> @@ -225,7 +229,7 @@ info(awaiting_rel_cnt, #session{awaiting_rel = AwaitingRel}) -> info(awaiting_rel_max, #session{max_awaiting_rel = Max}) -> Max; info(await_rel_timeout, #session{await_rel_timeout = Timeout}) -> - Timeout div 1000; + Timeout; info(created_at, #session{created_at = CreatedAt}) -> CreatedAt. @@ -253,7 +257,7 @@ subscribe(ClientInfo = #{clientid := ClientId}, TopicFilter, SubOpts, end. -compile({inline, [is_subscriptions_full/1]}). -is_subscriptions_full(#session{max_subscriptions = 0}) -> +is_subscriptions_full(#session{max_subscriptions = infinity}) -> false; is_subscriptions_full(#session{subscriptions = Subs, max_subscriptions = MaxLimit}) -> @@ -302,7 +306,7 @@ publish(_PacketId, Msg, Session) -> {ok, emqx_broker:publish(Msg), Session}. -compile({inline, [is_awaiting_full/1]}). -is_awaiting_full(#session{max_awaiting_rel = 0}) -> +is_awaiting_full(#session{max_awaiting_rel = infinity}) -> false; is_awaiting_full(#session{awaiting_rel = AwaitingRel, max_awaiting_rel = MaxLimit}) -> @@ -696,4 +700,3 @@ age(Now, Ts) -> Now - Ts. set_field(Name, Value, Session) -> Pos = emqx_misc:index_of(Name, record_info(fields, session)), setelement(Pos+1, Session, Value). - diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 97aa778f3..65c0e4d8d 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -77,6 +77,8 @@ -define(NACK(Reason), {shared_sub_nack, Reason}). -define(NO_ACK, no_ack). +-rlog_shard({?SHARED_SUB_SHARD, ?TAB}). + -record(state, {pmon}). -record(emqx_shared_subscription, {group, topic, subpid}). @@ -135,11 +137,11 @@ dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) -> -spec(strategy() -> strategy()). strategy() -> - emqx:get_env(shared_subscription_strategy, random). + emqx_config:get([broker, shared_subscription_strategy]). -spec(ack_enabled() -> boolean()). ack_enabled() -> - emqx:get_env(shared_dispatch_ack_enabled, false). + emqx_config:get([broker, shared_dispatch_ack_enabled]). do_dispatch(SubPid, Topic, Msg, _Type) when SubPid =:= self() -> %% Deadlock otherwise @@ -297,7 +299,7 @@ subscribers(Group, Topic) -> init([]) -> {ok, _} = mnesia:subscribe({table, ?TAB, simple}), - {atomic, PMon} = mnesia:transaction(fun init_monitors/0), + {atomic, PMon} = ekka_mnesia:transaction(?SHARED_SUB_SHARD, fun init_monitors/0), ok = emqx_tables:new(?SHARED_SUBS, [protected, bag]), ok = emqx_tables:new(?ALIVE_SUBS, [protected, set, {read_concurrency, true}]), {ok, update_stats(#state{pmon = PMon})}. @@ -309,7 +311,7 @@ init_monitors() -> end, emqx_pmon:new(), ?TAB). handle_call({subscribe, Group, Topic, SubPid}, _From, State = #state{pmon = PMon}) -> - mnesia:dirty_write(?TAB, record(Group, Topic, SubPid)), + ekka_mnesia:dirty_write(?TAB, record(Group, Topic, SubPid)), case ets:member(?SHARED_SUBS, {Group, Topic}) of true -> ok; false -> ok = emqx_router:do_add_route(Topic, {Group, node()}) @@ -319,7 +321,7 @@ handle_call({subscribe, Group, Topic, SubPid}, _From, State = #state{pmon = PMon {reply, ok, update_stats(State#state{pmon = emqx_pmon:monitor(SubPid, PMon)})}; handle_call({unsubscribe, Group, Topic, SubPid}, _From, State) -> - mnesia:dirty_delete_object(?TAB, record(Group, Topic, SubPid)), + ekka_mnesia:dirty_delete_object(?TAB, record(Group, Topic, SubPid)), true = ets:delete_object(?SHARED_SUBS, {{Group, Topic}, SubPid}), delete_route_if_needed({Group, Topic}), {reply, ok, State}; @@ -373,7 +375,7 @@ cleanup_down(SubPid) -> ?IS_LOCAL_PID(SubPid) orelse ets:delete(?ALIVE_SUBS, SubPid), lists:foreach( fun(Record = #emqx_shared_subscription{topic = Topic, group = Group}) -> - ok = mnesia:dirty_delete_object(?TAB, Record), + ok = ekka_mnesia:dirty_delete_object(?TAB, Record), true = ets:delete_object(?SHARED_SUBS, {{Group, Topic}, SubPid}), delete_route_if_needed({Group, Topic}) end, mnesia:dirty_match_object(#emqx_shared_subscription{_ = '_', subpid = SubPid})). diff --git a/apps/emqx/src/emqx_sys.erl b/apps/emqx/src/emqx_sys.erl index 2d816569d..2f3f782e6 100644 --- a/apps/emqx/src/emqx_sys.erl +++ b/apps/emqx/src/emqx_sys.erl @@ -32,8 +32,6 @@ , uptime/0 , datetime/0 , sysdescr/0 - , sys_interval/0 - , sys_heatbeat_interval/0 ]). -export([info/0]). @@ -104,15 +102,11 @@ datetime() -> io_lib:format( "~4..0w-~2..0w-~2..0w ~2..0w:~2..0w:~2..0w", [Y, M, D, H, MM, S])). -%% @doc Get sys interval --spec(sys_interval() -> pos_integer()). sys_interval() -> - emqx:get_env(broker_sys_interval, 60000). + emqx_config:get([broker, sys_msg_interval]). -%% @doc Get sys heatbeat interval --spec(sys_heatbeat_interval() -> pos_integer()). sys_heatbeat_interval() -> - emqx:get_env(broker_sys_heartbeat, 30000). + emqx_config:get([broker, sys_heartbeat_interval]). %% @doc Get sys info -spec(info() -> list(tuple())). diff --git a/apps/emqx/src/emqx_sys_mon.erl b/apps/emqx/src/emqx_sys_mon.erl index 152f975eb..54a5c533a 100644 --- a/apps/emqx/src/emqx_sys_mon.erl +++ b/apps/emqx/src/emqx_sys_mon.erl @@ -23,7 +23,7 @@ -logger_header("[SYSMON]"). --export([start_link/1]). +-export([start_link/0]). %% compress unused warning -export([procinfo/1]). @@ -37,25 +37,19 @@ , code_change/3 ]). --type(option() :: {long_gc, non_neg_integer()} - | {long_schedule, non_neg_integer()} - | {large_heap, non_neg_integer()} - | {busy_port, boolean()} - | {busy_dist_port, boolean()}). - -define(SYSMON, ?MODULE). %% @doc Start the system monitor. --spec(start_link(list(option())) -> startlink_ret()). -start_link(Opts) -> - gen_server:start_link({local, ?SYSMON}, ?MODULE, [Opts], []). +-spec(start_link() -> startlink_ret()). +start_link() -> + gen_server:start_link({local, ?SYSMON}, ?MODULE, [], []). %%-------------------------------------------------------------------- %% gen_server callbacks %%-------------------------------------------------------------------- -init([Opts]) -> - _ = erlang:system_monitor(self(), parse_opt(Opts)), +init([]) -> + _ = erlang:system_monitor(self(), sysm_opts()), emqx_logger:set_proc_metadata(#{sysmon => true}), %% Monitor cluster partition event @@ -66,30 +60,28 @@ init([Opts]) -> start_timer(State) -> State#{timer := emqx_misc:start_timer(timer:seconds(2), reset)}. -parse_opt(Opts) -> - parse_opt(Opts, []). -parse_opt([], Acc) -> +sysm_opts() -> + sysm_opts(maps:to_list(emqx_config:get([sysmon, vm])), []). +sysm_opts([], Acc) -> Acc; -parse_opt([{long_gc, 0}|Opts], Acc) -> - parse_opt(Opts, Acc); -parse_opt([{long_gc, Ms}|Opts], Acc) when is_integer(Ms) -> - parse_opt(Opts, [{long_gc, Ms}|Acc]); -parse_opt([{long_schedule, 0}|Opts], Acc) -> - parse_opt(Opts, Acc); -parse_opt([{long_schedule, Ms}|Opts], Acc) when is_integer(Ms) -> - parse_opt(Opts, [{long_schedule, Ms}|Acc]); -parse_opt([{large_heap, Size}|Opts], Acc) when is_integer(Size) -> - parse_opt(Opts, [{large_heap, Size}|Acc]); -parse_opt([{busy_port, true}|Opts], Acc) -> - parse_opt(Opts, [busy_port|Acc]); -parse_opt([{busy_port, false}|Opts], Acc) -> - parse_opt(Opts, Acc); -parse_opt([{busy_dist_port, true}|Opts], Acc) -> - parse_opt(Opts, [busy_dist_port|Acc]); -parse_opt([{busy_dist_port, false}|Opts], Acc) -> - parse_opt(Opts, Acc); -parse_opt([_Opt|Opts], Acc) -> - parse_opt(Opts, Acc). +sysm_opts([{_, disabled}|Opts], Acc) -> + sysm_opts(Opts, Acc); +sysm_opts([{long_gc, Ms}|Opts], Acc) when is_integer(Ms) -> + sysm_opts(Opts, [{long_gc, Ms}|Acc]); +sysm_opts([{long_schedule, Ms}|Opts], Acc) when is_integer(Ms) -> + sysm_opts(Opts, [{long_schedule, Ms}|Acc]); +sysm_opts([{large_heap, Size}|Opts], Acc) when is_integer(Size) -> + sysm_opts(Opts, [{large_heap, Size}|Acc]); +sysm_opts([{busy_port, true}|Opts], Acc) -> + sysm_opts(Opts, [busy_port|Acc]); +sysm_opts([{busy_port, false}|Opts], Acc) -> + sysm_opts(Opts, Acc); +sysm_opts([{busy_dist_port, true}|Opts], Acc) -> + sysm_opts(Opts, [busy_dist_port|Acc]); +sysm_opts([{busy_dist_port, false}|Opts], Acc) -> + sysm_opts(Opts, Acc); +sysm_opts([_Opt|Opts], Acc) -> + sysm_opts(Opts, Acc). handle_call(Req, _From, State) -> ?LOG(error, "Unexpected call: ~p", [Req]), diff --git a/apps/emqx/src/emqx_sys_sup.erl b/apps/emqx/src/emqx_sys_sup.erl index 50d086156..61342fd0e 100644 --- a/apps/emqx/src/emqx_sys_sup.erl +++ b/apps/emqx/src/emqx_sys_sup.erl @@ -27,10 +27,10 @@ start_link() -> init([]) -> Childs = [child_spec(emqx_sys), - child_spec(emqx_alarm, [config(alarm)]), - child_spec(emqx_sys_mon, [config(sysmon)]), - child_spec(emqx_os_mon, [config(os_mon)]), - child_spec(emqx_vm_mon, [config(vm_mon)])], + child_spec(emqx_alarm), + child_spec(emqx_sys_mon), + child_spec(emqx_os_mon), + child_spec(emqx_vm_mon)], {ok, {{one_for_one, 10, 100}, Childs}}. %%-------------------------------------------------------------------- @@ -48,6 +48,3 @@ child_spec(Mod, Args) -> type => worker, modules => [Mod] }. - -config(Name) -> emqx:get_env(Name, []). - diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 72a955962..24a9a15cf 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -161,17 +161,16 @@ drop_tls13_for_old_otp(SslOpts) -> , "TLS_AES_128_CCM_8_SHA256" ]). drop_tls13(SslOpts0) -> - SslOpts1 = case proplists:get_value(versions, SslOpts0) of - undefined -> SslOpts0; - Vsns -> replace(SslOpts0, versions, Vsns -- ['tlsv1.3']) + SslOpts1 = case maps:find(versions, SslOpts0) of + error -> SslOpts0; + {ok, Vsns} -> SslOpts0#{versions => (Vsns -- ['tlsv1.3'])} end, - case proplists:get_value(ciphers, SslOpts1) of - undefined -> SslOpts1; - Ciphers -> replace(SslOpts1, ciphers, Ciphers -- ?TLSV13_EXCLUSIVE_CIPHERS) + case maps:find(ciphers, SslOpts1) of + error -> SslOpts1; + {ok, Ciphers} -> + SslOpts1#{ciphers => Ciphers -- ?TLSV13_EXCLUSIVE_CIPHERS} end. -replace(Opts, Key, Value) -> [{Key, Value} | proplists:delete(Key, Opts)]. - -if(?OTP_RELEASE > 22). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -181,13 +180,13 @@ drop_tls13_test() -> ?assert(lists:member('tlsv1.3', Versions)), Ciphers = default_ciphers(), ?assert(has_tlsv13_cipher(Ciphers)), - Opts0 = [{versions, Versions}, {ciphers, Ciphers}, other, {bool, true}], + Opts0 = #{versions => Versions, ciphers => Ciphers, other => true}, Opts = drop_tls13(Opts0), - ?assertNot(lists:member('tlsv1.3', proplists:get_value(versions, Opts))), - ?assertNot(has_tlsv13_cipher(proplists:get_value(ciphers, Opts))). + ?assertNot(lists:member('tlsv1.3', maps:get(versions, Opts, undefined))), + ?assertNot(has_tlsv13_cipher(maps:get(ciphers, Opts, undefined))). drop_tls13_no_versions_cipers_test() -> - Opts0 = [other, {bool, true}], + Opts0 = #{other => 0, bool => true}, Opts = drop_tls13(Opts0), ?_assertEqual(Opts0, Opts). diff --git a/apps/emqx/src/emqx_trie.erl b/apps/emqx/src/emqx_trie.erl index bb5e171b1..32c176b65 100644 --- a/apps/emqx/src/emqx_trie.erl +++ b/apps/emqx/src/emqx_trie.erl @@ -28,14 +28,14 @@ -export([ insert/1 , match/1 , delete/1 - , put_compaction_flag/1 - , put_default_compaction_flag/0 ]). -export([ empty/0 , lock_tables/0 ]). +-export([is_compact/0, set_compact/1]). + -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). @@ -50,21 +50,12 @@ , count = 0 :: non_neg_integer() }). --define(IS_COMPACT, true). - -rlog_shard({?ROUTE_SHARD, ?TRIE}). %%-------------------------------------------------------------------- %% Mnesia bootstrap %%-------------------------------------------------------------------- -put_compaction_flag(Bool) when is_boolean(Bool) -> - _ = persistent_term:put({?MODULE, compaction}, Bool), - ok. - -put_default_compaction_flag() -> - ok = put_compaction_flag(?IS_COMPACT). - %% @doc Create or replicate topics table. -spec(mnesia(boot | copy) -> ok). mnesia(boot) -> @@ -279,16 +270,10 @@ match_compact([Word | Words], Prefix, IsWildcard, Acc0) -> lookup_topic(MlTopic). is_compact() -> - case persistent_term:get({?MODULE, compaction}, undefined) of - undefined -> - Default = ?IS_COMPACT, - FromEnv = emqx:get_env(trie_compaction, Default), - _ = put_compaction_flag(FromEnv), - true = is_boolean(FromEnv), - FromEnv; - Value when is_boolean(Value) -> - Value - end. + emqx_config:get([broker, perf, trie_compaction], true). + +set_compact(Bool) -> + emqx_config:put([broker, perf, trie_compaction], Bool). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -315,10 +300,11 @@ words(T) -> emqx_topic:words(T). make_prefixes_t(Topic) -> make_prefixes(words(Topic)). -with_compact_flag(IsCmopact, F) -> - put_compaction_flag(IsCmopact), +with_compact_flag(IsCompact, F) -> + OldV = is_compact(), + set_compact(IsCompact), try F() - after put_default_compaction_flag() + after set_compact(OldV) end. make_prefixes_test_() -> diff --git a/apps/emqx/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl index fbe62e4b2..84868f473 100644 --- a/apps/emqx/src/emqx_types.erl +++ b/apps/emqx/src/emqx_types.erl @@ -94,14 +94,16 @@ -type(ver() :: ?MQTT_PROTO_V3 | ?MQTT_PROTO_V4 | ?MQTT_PROTO_V5 - | non_neg_integer()). + | non_neg_integer() + | binary() % For lwm2m, mqtt-sn... + ). -type(qos() :: ?QOS_0 | ?QOS_1 | ?QOS_2). -type(qos_name() :: qos0 | at_most_once | qos1 | at_least_once | qos2 | exactly_once). --type(zone() :: emqx_zone:zone()). +-type(zone() :: atom()). -type(pubsub() :: publish | subscribe). -type(topic() :: emqx_topic:topic()). -type(subid() :: binary() | atom()). @@ -209,7 +211,8 @@ -type(infos() :: #{atom() => term()}). -type(stats() :: [{atom(), term()}]). --type(oom_policy() :: #{message_queue_len => non_neg_integer(), - max_heap_size => non_neg_integer() +-type(oom_policy() :: #{max_message_queue_len => non_neg_integer(), + max_heap_size => non_neg_integer(), + enable => boolean() }). diff --git a/apps/emqx/src/emqx_vm_mon.erl b/apps/emqx/src/emqx_vm_mon.erl index ce34fff43..13a470959 100644 --- a/apps/emqx/src/emqx_vm_mon.erl +++ b/apps/emqx/src/emqx_vm_mon.erl @@ -21,15 +21,7 @@ -include("logger.hrl"). %% APIs --export([start_link/1]). - --export([ get_check_interval/0 - , set_check_interval/1 - , get_process_high_watermark/0 - , set_process_high_watermark/1 - , get_process_low_watermark/0 - , set_process_low_watermark/1 - ]). +-export([start_link/0]). %% gen_server callbacks -export([ init/1 @@ -42,61 +34,19 @@ -define(VM_MON, ?MODULE). -start_link(Opts) -> - gen_server:start_link({local, ?VM_MON}, ?MODULE, [Opts], []). - %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- - -get_check_interval() -> - call(get_check_interval). - -set_check_interval(Seconds) -> - call({set_check_interval, Seconds}). - -get_process_high_watermark() -> - call(get_process_high_watermark). - -set_process_high_watermark(Float) -> - call({set_process_high_watermark, Float}). - -get_process_low_watermark() -> - call(get_process_low_watermark). - -set_process_low_watermark(Float) -> - call({set_process_low_watermark, Float}). - -call(Req) -> - gen_server:call(?VM_MON, Req, infinity). +start_link() -> + gen_server:start_link({local, ?VM_MON}, ?MODULE, [], []). %%-------------------------------------------------------------------- %% gen_server callbacks %%-------------------------------------------------------------------- -init([Opts]) -> - {ok, ensure_check_timer(#{check_interval => proplists:get_value(check_interval, Opts), - process_high_watermark => proplists:get_value(process_high_watermark, Opts), - process_low_watermark => proplists:get_value(process_low_watermark, Opts), - timer => undefined})}. - -handle_call(get_check_interval, _From, State) -> - {reply, maps:get(check_interval, State, undefined), State}; - -handle_call({set_check_interval, Seconds}, _From, State) -> - {reply, ok, State#{check_interval := Seconds}}; - -handle_call(get_process_high_watermark, _From, State) -> - {reply, maps:get(process_high_watermark, State, undefined), State}; - -handle_call({set_process_high_watermark, Float}, _From, State) -> - {reply, ok, State#{process_high_watermark := Float}}; - -handle_call(get_process_low_watermark, _From, State) -> - {reply, maps:get(process_low_watermark, State, undefined), State}; - -handle_call({set_process_low_watermark, Float}, _From, State) -> - {reply, ok, State#{process_low_watermark := Float}}; +init([]) -> + start_check_timer(), + {ok, #{}}. handle_call(Req, _From, State) -> ?LOG(error, "[VM_MON] Unexpected call: ~p", [Req]), @@ -106,29 +56,30 @@ handle_cast(Msg, State) -> ?LOG(error, "[VM_MON] Unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({timeout, Timer, check}, - State = #{timer := Timer, - process_high_watermark := ProcHighWatermark, - process_low_watermark := ProcLowWatermark}) -> +handle_info({timeout, _Timer, check}, State) -> + ProcHighWatermark = emqx_config:get([sysmon, vm, process_high_watermark]), + ProcLowWatermark = emqx_config:get([sysmon, vm, process_low_watermark]), ProcessCount = erlang:system_info(process_count), - case ProcessCount / erlang:system_info(process_limit) * 100 of + case ProcessCount / erlang:system_info(process_limit) of Percent when Percent >= ProcHighWatermark -> - emqx_alarm:activate(too_many_processes, #{usage => Percent, - high_watermark => ProcHighWatermark, - low_watermark => ProcLowWatermark}); + emqx_alarm:activate(too_many_processes, #{ + usage => io_lib:format("~p%", [Percent*100]), + high_watermark => ProcHighWatermark, + low_watermark => ProcLowWatermark}); Percent when Percent < ProcLowWatermark -> emqx_alarm:deactivate(too_many_processes); _Precent -> ok end, - {noreply, ensure_check_timer(State)}; + start_check_timer(), + {noreply, State}; handle_info(Info, State) -> ?LOG(error, "[VM_MON] Unexpected info: ~p", [Info]), {noreply, State}. -terminate(_Reason, #{timer := Timer}) -> - emqx_misc:cancel_timer(Timer). +terminate(_Reason, _State) -> + ok. code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -137,5 +88,6 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- -ensure_check_timer(State = #{check_interval := Interval}) -> - State#{timer := emqx_misc:start_timer(timer:seconds(Interval), check)}. +start_check_timer() -> + Interval = emqx_config:get([sysmon, vm, process_check_interval]), + emqx_misc:start_timer(Interval, check). diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index d686e1611..540076eaf 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -62,8 +62,6 @@ sockname :: emqx_types:peername(), %% Sock state sockstate :: emqx_types:sockstate(), - %% Simulate the active_n opt - active_n :: pos_integer(), %% MQTT Piggyback mqtt_piggyback :: single | multiple, %% Limiter @@ -85,7 +83,11 @@ %% Idle Timeout idle_timeout :: timeout(), %% Idle Timer - idle_timer :: maybe(reference()) + idle_timer :: maybe(reference()), + %% Zone name + zone :: atom(), + %% Listener Name + listener :: atom() }). -type(state() :: #state{}). @@ -93,7 +95,7 @@ -type(ws_cmd() :: {active, boolean()}|close). -define(ACTIVE_N, 100). --define(INFO_KEYS, [socktype, peername, sockname, sockstate, active_n]). +-define(INFO_KEYS, [socktype, peername, sockname, sockstate]). -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt]). -define(CONN_STATS, [recv_pkt, recv_msg, send_pkt, send_msg]). @@ -124,8 +126,6 @@ info(sockname, #state{sockname = Sockname}) -> Sockname; info(sockstate, #state{sockstate = SockSt}) -> SockSt; -info(active_n, #state{active_n = ActiveN}) -> - ActiveN; info(limiter, #state{limiter = Limiter}) -> maybe_apply(fun emqx_limiter:info/1, Limiter); info(channel, #state{channel = Channel}) -> @@ -174,21 +174,13 @@ call(WsPid, Req, Timeout) when is_pid(WsPid) -> %% WebSocket callbacks %%-------------------------------------------------------------------- -init(Req, Opts) -> +init(Req, #{zone := Zone, listener := Listener} = Opts) -> %% WS Transport Idle Timeout - IdleTimeout = proplists:get_value(idle_timeout, Opts, 7200000), - DeflateOptions = maps:from_list(proplists:get_value(deflate_options, Opts, [])), - MaxFrameSize = case proplists:get_value(max_frame_size, Opts, 0) of - 0 -> infinity; - I -> I - end, - Compress = proplists:get_bool(compress, Opts), - WsOpts = #{compress => Compress, - deflate_opts => DeflateOptions, - max_frame_size => MaxFrameSize, - idle_timeout => IdleTimeout + WsOpts = #{compress => get_ws_opts(Zone, Listener, compress), + deflate_opts => get_ws_opts(Zone, Listener, deflate_opts), + max_frame_size => get_ws_opts(Zone, Listener, max_frame_size), + idle_timeout => get_ws_opts(Zone, Listener, idle_timeout) }, - case check_origin_header(Req, Opts) of {error, Message} -> ?LOG(error, "Invalid Origin Header ~p~n", [Message]), @@ -196,18 +188,17 @@ init(Req, Opts) -> ok -> parse_sec_websocket_protocol(Req, Opts, WsOpts) end. -parse_sec_websocket_protocol(Req, Opts, WsOpts) -> - FailIfNoSubprotocol = proplists:get_value(fail_if_no_subprotocol, Opts), +parse_sec_websocket_protocol(Req, #{zone := Zone, listener := Listener} = Opts, WsOpts) -> case cowboy_req:parse_header(<<"sec-websocket-protocol">>, Req) of undefined -> - case FailIfNoSubprotocol of + case get_ws_opts(Zone, Listener, fail_if_no_subprotocol) of true -> {ok, cowboy_req:reply(400, Req), WsOpts}; false -> {cowboy_websocket, Req, [Req, Opts], WsOpts} end; Subprotocols -> - SupportedSubprotocols = proplists:get_value(supported_subprotocols, Opts), + SupportedSubprotocols = get_ws_opts(Zone, Listener, supported_subprotocols), NSupportedSubprotocols = [list_to_binary(Subprotocol) || Subprotocol <- SupportedSubprotocols], case pick_subprotocol(Subprotocols, NSupportedSubprotocols) of @@ -231,31 +222,30 @@ pick_subprotocol([Subprotocol | Rest], SupportedSubprotocols) -> pick_subprotocol(Rest, SupportedSubprotocols) end. -parse_header_fun_origin(Req, Opts) -> +parse_header_fun_origin(Req, #{zone := Zone, listener := Listener}) -> case cowboy_req:header(<<"origin">>, Req) of undefined -> - case proplists:get_bool(allow_origin_absence, Opts) of + case get_ws_opts(Zone, Listener, allow_origin_absence) of true -> ok; false -> {error, origin_header_cannot_be_absent} end; Value -> - Origins = proplists:get_value(check_origins, Opts, []), - case lists:member(Value, Origins) of + case lists:member(Value, get_ws_opts(Zone, Listener, check_origins)) of true -> ok; false -> {origin_not_allowed, Value} end end. -check_origin_header(Req, Opts) -> - case proplists:get_bool(check_origin_enable, Opts) of +check_origin_header(Req, #{zone := Zone, listener := Listener} = Opts) -> + case get_ws_opts(Zone, Listener, check_origin_enable) of true -> parse_header_fun_origin(Req, Opts); false -> ok end. -websocket_init([Req, Opts]) -> +websocket_init([Req, #{zone := Zone, listener := Listener} = Opts]) -> {Peername, Peercert} = - case proplists:get_bool(proxy_protocol, Opts) - andalso maps:get(proxy_header, Req) of + case emqx_config:get_listener_conf(Zone, Listener, [proxy_protocol]) andalso + maps:get(proxy_header, Req) of #{src_address := SrcAddr, src_port := SrcPort, ssl := SSL} -> SourceName = {SrcAddr, SrcPort}, %% Notice: Only CN is available in Proxy Protocol V2 additional info @@ -266,7 +256,7 @@ websocket_init([Req, Opts]) -> {SourceName, SourceSSL}; #{src_address := SrcAddr, src_port := SrcPort} -> SourceName = {SrcAddr, SrcPort}, - {SourceName , nossl}; + {SourceName, nossl}; _ -> {get_peer(Req, Opts), cowboy_req:cert(Req)} end, @@ -288,28 +278,35 @@ websocket_init([Req, Opts]) -> ws_cookie => WsCookie, conn_mod => ?MODULE }, - Zone = proplists:get_value(zone, Opts), - PubLimit = emqx_zone:publish_limit(Zone), - BytesIn = proplists:get_value(rate_limit, Opts), - RateLimit = emqx_zone:ratelimit(Zone), - Limiter = emqx_limiter:init(Zone, PubLimit, BytesIn, RateLimit), - ActiveN = proplists:get_value(active_n, Opts, ?ACTIVE_N), - MQTTPiggyback = proplists:get_value(mqtt_piggyback, Opts, multiple), - FrameOpts = emqx_zone:mqtt_frame_options(Zone), + Limiter = emqx_limiter:init(Zone, undefined, undefined, []), + MQTTPiggyback = get_ws_opts(Zone, Listener, mqtt_piggyback), + FrameOpts = #{ + strict_mode => emqx_config:get_zone_conf(Zone, [mqtt, strict_mode]), + max_size => emqx_config:get_zone_conf(Zone, [mqtt, max_packet_size]) + }, ParseState = emqx_frame:initial_parse_state(FrameOpts), Serialize = emqx_frame:serialize_opts(), Channel = emqx_channel:init(ConnInfo, Opts), - GcState = emqx_zone:init_gc_state(Zone), - StatsTimer = emqx_zone:stats_timer(Zone), + GcState = case emqx_config:get_zone_conf(Zone, [force_gc]) of + #{enable := false} -> undefined; + GcPolicy -> emqx_gc:init(GcPolicy) + end, + StatsTimer = case emqx_config:get_zone_conf(Zone, [stats, enable]) of + true -> undefined; + false -> disabled + end, %% MQTT Idle Timeout - IdleTimeout = emqx_zone:idle_timeout(Zone), + IdleTimeout = emqx_channel:get_mqtt_conf(Zone, idle_timeout), IdleTimer = start_timer(IdleTimeout, idle_timeout), - emqx_misc:tune_heap_size(emqx_zone:oom_policy(Zone)), + case emqx_config:get_zone_conf(emqx_channel:info(zone, Channel), + [force_shutdown]) of + #{enable := false} -> ok; + ShutdownPolicy -> emqx_misc:tune_heap_size(ShutdownPolicy) + end, emqx_logger:set_metadata_peername(esockd:format(Peername)), {ok, #state{peername = Peername, sockname = Sockname, sockstate = running, - active_n = ActiveN, mqtt_piggyback = MQTTPiggyback, limiter = Limiter, parse_state = ParseState, @@ -319,7 +316,9 @@ websocket_init([Req, Opts]) -> postponed = [], stats_timer = StatsTimer, idle_timeout = IdleTimeout, - idle_timer = IdleTimer + idle_timer = IdleTimer, + zone = Zone, + listener = Listener }, hibernate}. websocket_handle({binary, Data}, State) when is_list(Data) -> @@ -372,7 +371,8 @@ websocket_info({check_gc, Stats}, State) -> return(check_oom(run_gc(Stats, State))); websocket_info(Deliver = {deliver, _Topic, _Msg}, - State = #state{active_n = ActiveN}) -> + State = #state{zone = Zone, listener = Listener}) -> + ActiveN = emqx_config:get_listener_conf(Zone, Listener, [tcp, active_n]), Delivers = [Deliver|emqx_misc:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); @@ -521,11 +521,16 @@ run_gc(Stats, State = #state{gc_state = GcSt}) -> end. check_oom(State = #state{channel = Channel}) -> - OomPolicy = emqx_zone:oom_policy(emqx_channel:info(zone, Channel)), - case ?ENABLED(OomPolicy) andalso emqx_misc:check_oom(OomPolicy) of - Shutdown = {shutdown, _Reason} -> - postpone(Shutdown, State); - _Other -> State + ShutdownPolicy = emqx_config:get_zone_conf( + emqx_channel:info(zone, Channel), [force_shutdown]), + case ShutdownPolicy of + #{enable := false} -> State; + #{enable := true} -> + case emqx_misc:check_oom(ShutdownPolicy) of + Shutdown = {shutdown, _Reason} -> + postpone(Shutdown, State); + _Other -> State + end end. %%-------------------------------------------------------------------- @@ -554,11 +559,12 @@ parse_incoming(Data, State = #state{parse_state = ParseState}) -> %% Handle incoming packet %%-------------------------------------------------------------------- -handle_incoming(Packet, State = #state{active_n = ActiveN}) +handle_incoming(Packet, State = #state{zone = Zone, listener = Listener}) when is_record(Packet, mqtt_packet) -> ?LOG(debug, "RECV ~s", [emqx_packet:format(Packet)]), ok = inc_incoming_stats(Packet), - NState = case emqx_pd:get_counter(incoming_pubs) > ActiveN of + NState = case emqx_pd:get_counter(incoming_pubs) > + emqx_config:get_listener_conf(Zone, Listener, [tcp, active_n]) of true -> postpone({cast, rate_limit}, State); false -> State end, @@ -589,11 +595,13 @@ with_channel(Fun, Args, State = #state{channel = Channel}) -> %% Handle outgoing packets %%-------------------------------------------------------------------- -handle_outgoing(Packets, State = #state{active_n = ActiveN, mqtt_piggyback = MQTTPiggyback}) -> +handle_outgoing(Packets, State = #state{mqtt_piggyback = MQTTPiggyback, + zone = Zone, listener = Listener}) -> IoData = lists:map(serialize_and_inc_stats_fun(State), Packets), Oct = iolist_size(IoData), ok = inc_sent_stats(length(Packets), Oct), - NState = case emqx_pd:get_counter(outgoing_pubs) > ActiveN of + NState = case emqx_pd:get_counter(outgoing_pubs) > + emqx_config:get_listener_conf(Zone, Listener, [tcp, active_n]) of true -> Stats = #{cnt => emqx_pd:reset_counter(outgoing_pubs), oct => emqx_pd:reset_counter(outgoing_bytes) @@ -742,9 +750,10 @@ classify([Event|More], Packets, Cmds, Events) -> trigger(Event) -> erlang:send(self(), Event). -get_peer(Req, Opts) -> +get_peer(Req, #{zone := Zone, listener := Listener}) -> {PeerAddr, PeerPort} = cowboy_req:peer(Req), - AddrHeader = cowboy_req:header(proplists:get_value(proxy_address_header, Opts), Req, <<>>), + AddrHeader = cowboy_req:header( + get_ws_opts(Zone, Listener, proxy_address_header), Req, <<>>), ClientAddr = case string:tokens(binary_to_list(AddrHeader), ", ") of [] -> undefined; @@ -757,7 +766,8 @@ get_peer(Req, Opts) -> _ -> PeerAddr end, - PortHeader = cowboy_req:header(proplists:get_value(proxy_port_header, Opts), Req, <<>>), + PortHeader = cowboy_req:header( + get_ws_opts(Zone, Listener, proxy_port_header), Req, <<>>), ClientPort = case string:tokens(binary_to_list(PortHeader), ", ") of [] -> undefined; @@ -778,3 +788,5 @@ set_field(Name, Value, State) -> Pos = emqx_misc:index_of(Name, record_info(fields, state)), setelement(Pos+1, State, Value). +get_ws_opts(Zone, Listener, Key) -> + emqx_config:get_listener_conf(Zone, Listener, [websocket, Key]). diff --git a/apps/emqx/src/emqx_zone.erl b/apps/emqx/src/emqx_zone.erl deleted file mode 100644 index 459c36764..000000000 --- a/apps/emqx/src/emqx_zone.erl +++ /dev/null @@ -1,298 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_zone). - --behaviour(gen_server). - --include("emqx.hrl"). --include("emqx_mqtt.hrl"). --include("logger.hrl"). --include("types.hrl"). - --logger_header("[Zone]"). - --compile({inline, - [ idle_timeout/1 - , publish_limit/1 - , ratelimit/1 - , mqtt_frame_options/1 - , mqtt_strict_mode/1 - , max_packet_size/1 - , mountpoint/1 - , use_username_as_clientid/1 - , stats_timer/1 - , enable_stats/1 - , enable_acl/1 - , enable_ban/1 - , enable_flapping_detect/1 - , ignore_loop_deliver/1 - , server_keepalive/1 - , keepalive_backoff/1 - , max_inflight/1 - , session_expiry_interval/1 - , force_gc_policy/1 - , force_shutdown_policy/1 - , response_information/1 - , quota_policy/1 - , get_env/2 - , get_env/3 - ]}). - -%% APIs --export([start_link/0, stop/0]). - -%% Zone Option API --export([ idle_timeout/1 - %% XXX: Dedeprecated at v4.2 - , publish_limit/1 - , ratelimit/1 - , mqtt_frame_options/1 - , mqtt_strict_mode/1 - , max_packet_size/1 - , mountpoint/1 - , use_username_as_clientid/1 - , stats_timer/1 - , enable_stats/1 - , enable_acl/1 - , enable_ban/1 - , enable_flapping_detect/1 - , ignore_loop_deliver/1 - , server_keepalive/1 - , keepalive_backoff/1 - , max_inflight/1 - , session_expiry_interval/1 - , force_gc_policy/1 - , force_shutdown_policy/1 - , response_information/1 - , quota_policy/1 - ]). - --export([ init_gc_state/1 - , oom_policy/1 - ]). - -%% Zone API --export([ get_env/2 - , get_env/3 - , set_env/3 - , unset_env/2 - , unset_all_env/0 - ]). - --export([force_reload/0]). - -%% gen_server callbacks --export([ init/1 - , handle_call/3 - , handle_cast/2 - , handle_info/2 - , terminate/2 - , code_change/3 - ]). - --import(emqx_misc, [maybe_apply/2]). - --export_type([zone/0]). - --type(zone() :: atom()). - --define(TAB, ?MODULE). --define(SERVER, ?MODULE). --define(DEFAULT_IDLE_TIMEOUT, 30000). --define(KEY(Zone, Key), {?MODULE, Zone, Key}). - --spec(start_link() -> startlink_ret()). -start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). - --spec(stop() -> ok). -stop() -> - gen_server:stop(?SERVER). - --spec(init_gc_state(zone()) -> maybe(emqx_gc:gc_state())). -init_gc_state(Zone) -> - maybe_apply(fun emqx_gc:init/1, force_gc_policy(Zone)). - --spec(oom_policy(zone()) -> emqx_types:oom_policy()). -oom_policy(Zone) -> force_shutdown_policy(Zone). - -%%-------------------------------------------------------------------- -%% Zone Options API -%%-------------------------------------------------------------------- - --spec(idle_timeout(zone()) -> pos_integer()). -idle_timeout(Zone) -> - get_env(Zone, idle_timeout, ?DEFAULT_IDLE_TIMEOUT). - --spec(publish_limit(zone()) -> maybe(esockd_rate_limit:config())). -publish_limit(Zone) -> - get_env(Zone, publish_limit). - --spec(ratelimit(zone()) -> [emqx_limiter:specs()]). -ratelimit(Zone) -> - get_env(Zone, ratelimit, []). - --spec(mqtt_frame_options(zone()) -> emqx_frame:options()). -mqtt_frame_options(Zone) -> - #{strict_mode => mqtt_strict_mode(Zone), - max_size => max_packet_size(Zone) - }. - --spec(mqtt_strict_mode(zone()) -> boolean()). -mqtt_strict_mode(Zone) -> - get_env(Zone, strict_mode, false). - --spec(max_packet_size(zone()) -> integer()). -max_packet_size(Zone) -> - get_env(Zone, max_packet_size, ?MAX_PACKET_SIZE). - --spec(mountpoint(zone()) -> maybe(emqx_mountpoint:mountpoint())). -mountpoint(Zone) -> get_env(Zone, mountpoint). - --spec(use_username_as_clientid(zone()) -> boolean()). -use_username_as_clientid(Zone) -> - get_env(Zone, use_username_as_clientid, false). - --spec(stats_timer(zone()) -> undefined | disabled). -stats_timer(Zone) -> - case enable_stats(Zone) of true -> undefined; false -> disabled end. - --spec(enable_stats(zone()) -> boolean()). -enable_stats(Zone) -> - get_env(Zone, enable_stats, true). - --spec(enable_acl(zone()) -> boolean()). -enable_acl(Zone) -> - get_env(Zone, enable_acl, true). - --spec(enable_ban(zone()) -> boolean()). -enable_ban(Zone) -> - get_env(Zone, enable_ban, false). - --spec(enable_flapping_detect(zone()) -> boolean()). -enable_flapping_detect(Zone) -> - get_env(Zone, enable_flapping_detect, false). - --spec(ignore_loop_deliver(zone()) -> boolean()). -ignore_loop_deliver(Zone) -> - get_env(Zone, ignore_loop_deliver, false). - --spec(server_keepalive(zone()) -> maybe(pos_integer())). -server_keepalive(Zone) -> - get_env(Zone, server_keepalive). - --spec(keepalive_backoff(zone()) -> float()). -keepalive_backoff(Zone) -> - get_env(Zone, keepalive_backoff, 0.75). - --spec(max_inflight(zone()) -> 0..65535). -max_inflight(Zone) -> - get_env(Zone, max_inflight, 65535). - --spec(session_expiry_interval(zone()) -> non_neg_integer()). -session_expiry_interval(Zone) -> - get_env(Zone, session_expiry_interval, 0). - --spec(force_gc_policy(zone()) -> maybe(emqx_gc:opts())). -force_gc_policy(Zone) -> - get_env(Zone, force_gc_policy). - --spec(force_shutdown_policy(zone()) -> maybe(emqx_oom:opts())). -force_shutdown_policy(Zone) -> - get_env(Zone, force_shutdown_policy). - --spec(response_information(zone()) -> string()). -response_information(Zone) -> - get_env(Zone, response_information). - --spec(quota_policy(zone()) -> emqx_quota:policy()). -quota_policy(Zone) -> - get_env(Zone, quota, []). - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - --spec(get_env(maybe(zone()), atom()) -> maybe(term())). -get_env(undefined, Key) -> emqx:get_env(Key); -get_env(Zone, Key) -> - get_env(Zone, Key, undefined). - --spec(get_env(maybe(zone()), atom(), term()) -> maybe(term())). -get_env(undefined, Key, Def) -> - emqx:get_env(Key, Def); -get_env(Zone, Key, Def) -> - try persistent_term:get(?KEY(Zone, Key)) - catch error:badarg -> - emqx:get_env(Key, Def) - end. - --spec(set_env(zone(), atom(), term()) -> ok). -set_env(Zone, Key, Val) -> - persistent_term:put(?KEY(Zone, Key), Val). - --spec(unset_env(zone(), atom()) -> boolean()). -unset_env(Zone, Key) -> - persistent_term:erase(?KEY(Zone, Key)). - --spec(unset_all_env() -> ok). -unset_all_env() -> - [unset_env(Zone, Key) || {?KEY(Zone, Key), _Val} <- persistent_term:get()], - ok. - --spec(force_reload() -> ok). -force_reload() -> - gen_server:call(?SERVER, force_reload). - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init([]) -> - _ = do_reload(), - {ok, #{}}. - -handle_call(force_reload, _From, State) -> - _ = do_reload(), - {reply, ok, State}; - -handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), - {reply, ignored, State}. - -handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), - {noreply, State}. - -handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -do_reload() -> - [persistent_term:put(?KEY(Zone, Key), Val) - || {Zone, Opts} <- emqx:get_env(zones, []), {Key, Val} <- Opts]. - diff --git a/apps/emqx/test/emqx_SUITE.erl b/apps/emqx/test/emqx_SUITE.erl index 4213a5aac..9614822ba 100644 --- a/apps/emqx/test/emqx_SUITE.erl +++ b/apps/emqx/test/emqx_SUITE.erl @@ -51,14 +51,6 @@ t_stop_start(_) -> ok = emqx:shutdown(for_test), false = emqx:is_running(node()). -t_get_env(_) -> - ?assertEqual(undefined, emqx:get_env(undefined_key)), - ?assertEqual(default_value, emqx:get_env(undefined_key, default_value)), - application:set_env(emqx, undefined_key, hello), - ?assertEqual(hello, emqx:get_env(undefined_key)), - ?assertEqual(hello, emqx:get_env(undefined_key, default_value)), - application:unset_env(emqx, undefined_key). - t_emqx_pubsub_api(_) -> true = emqx:is_running(node()), {ok, C} = emqtt:start_link([{host, "localhost"}, {clientid, "myclient"}]), diff --git a/apps/emqx/test/emqx_SUITE_data/acl.conf b/apps/emqx/test/emqx_SUITE_data/acl.conf index 3cb3b8c52..f466cf771 100644 --- a/apps/emqx/test/emqx_SUITE_data/acl.conf +++ b/apps/emqx/test/emqx_SUITE_data/acl.conf @@ -1,6 +1,6 @@ %%-------------------------------------------------------------------- %% -%% [ACL](https://github.com/emqtt/emqttd/wiki/ACL) +%% [Authorization](https://github.com/emqtt/emqttd/wiki/Authorization) %% %% -type who() :: all | binary() | %% {ipaddr, esockd_access:cidr()} | diff --git a/apps/emqx/test/emqx_access_control_SUITE.erl b/apps/emqx/test/emqx_access_control_SUITE.erl index e4a888d14..d459d28b2 100644 --- a/apps/emqx/test/emqx_access_control_SUITE.erl +++ b/apps/emqx/test/emqx_access_control_SUITE.erl @@ -33,43 +33,20 @@ end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([]). t_authenticate(_) -> - emqx_zone:set_env(zone, allow_anonymous, false), - ?assertMatch({error, _}, emqx_access_control:authenticate(clientinfo())), - emqx_zone:set_env(zone, allow_anonymous, true), - ?assertMatch({ok, _}, emqx_access_control:authenticate(clientinfo())). + ?assertMatch(ok, emqx_access_control:authenticate(clientinfo())). -t_check_acl(_) -> - emqx_zone:set_env(zone, acl_nomatch, deny), - application:set_env(emqx, enable_acl_cache, false), +t_authorize(_) -> Publish = ?PUBLISH_PACKET(?QOS_0, <<"t">>, 1, <<"payload">>), - ?assertEqual(deny, emqx_access_control:check_acl(clientinfo(), Publish, <<"t">>)), - - emqx_zone:set_env(zone, acl_nomatch, allow), - application:set_env(emqx, enable_acl_cache, true), - Publish = ?PUBLISH_PACKET(?QOS_0, <<"t">>, 1, <<"payload">>), - ?assertEqual(allow, emqx_access_control:check_acl(clientinfo(), Publish, <<"t">>)). - -t_bypass_auth_plugins(_) -> - ClientInfo = clientinfo(), - emqx_zone:set_env(bypass_zone, allow_anonymous, true), - emqx_zone:set_env(zone, allow_anonymous, false), - emqx_zone:set_env(bypass_zone, bypass_auth_plugins, true), - emqx:hook('client.authenticate',{?MODULE, auth_fun, []}), - ?assertMatch({ok, _}, emqx_access_control:authenticate(ClientInfo#{zone => bypass_zone})), - ?assertMatch({ok, _}, emqx_access_control:authenticate(ClientInfo)). + ?assertEqual(allow, emqx_access_control:authorize(clientinfo(), Publish, <<"t">>)). %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- -auth_fun(#{zone := bypass_zone}, AuthRes) -> - {stop, AuthRes#{auth_result => password_error}}; -auth_fun(#{zone := _}, AuthRes) -> - {stop, AuthRes#{auth_result => success}}. - clientinfo() -> clientinfo(#{}). clientinfo(InitProps) -> - maps:merge(#{zone => zone, + maps:merge(#{zone => default, + listener => mqtt_tcp, protocol => mqtt, peerhost => {127,0,0,1}, clientid => <<"clientid">>, @@ -79,3 +56,6 @@ clientinfo(InitProps) -> peercert => undefined, mountpoint => undefined }, InitProps). + +toggle_auth(Bool) when is_boolean(Bool) -> + emqx_config:put_zone_conf(default, [auth, enable], Bool). diff --git a/apps/emqx/test/emqx_alarm_SUITE.erl b/apps/emqx/test/emqx_alarm_SUITE.erl index db6cdfe7f..1157f94bc 100644 --- a/apps/emqx/test/emqx_alarm_SUITE.erl +++ b/apps/emqx/test/emqx_alarm_SUITE.erl @@ -27,27 +27,17 @@ all() -> emqx_ct:all(?MODULE). init_per_testcase(t_size_limit, Config) -> emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([], - fun(emqx) -> - application:set_env(emqx, alarm, [{actions, [log,publish]}, - {size_limit, 2}, - {validity_period, 3600}]), - ok; - (_) -> - ok - end), + emqx_ct_helpers:start_apps([]), + emqx_config:update([alarm], #{ + <<"size_limit">> => 2 + }), Config; init_per_testcase(t_validity_period, Config) -> emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([], - fun(emqx) -> - application:set_env(emqx, alarm, [{actions, [log,publish]}, - {size_limit, 1000}, - {validity_period, 1}]), - ok; - (_) -> - ok - end), + emqx_ct_helpers:start_apps([]), + emqx_config:update([alarm], #{ + <<"validity_period">> => <<"1s">> + }), Config; init_per_testcase(_, Config) -> emqx_ct_helpers:boot_modules(all), @@ -89,7 +79,7 @@ t_size_limit(_) -> ok = emqx_alarm:activate(b), ok = emqx_alarm:deactivate(b), ?assertNotEqual({error, not_found}, get_alarm(a, emqx_alarm:get_alarms(deactivated))), - ?assertNotEqual({error, not_found}, get_alarm(a, emqx_alarm:get_alarms(deactivated))), + ?assertNotEqual({error, not_found}, get_alarm(b, emqx_alarm:get_alarms(deactivated))), ok = emqx_alarm:activate(c), ok = emqx_alarm:deactivate(c), ?assertNotEqual({error, not_found}, get_alarm(c, emqx_alarm:get_alarms(deactivated))), diff --git a/apps/emqx/test/emqx_acl_cache_SUITE.erl b/apps/emqx/test/emqx_authz_cache_SUITE.erl similarity index 52% rename from apps/emqx/test/emqx_acl_cache_SUITE.erl rename to apps/emqx/test/emqx_authz_cache_SUITE.erl index be7c29055..849997298 100644 --- a/apps/emqx/test/emqx_acl_cache_SUITE.erl +++ b/apps/emqx/test/emqx_authz_cache_SUITE.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_acl_cache_SUITE). +-module(emqx_authz_cache_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -26,6 +26,7 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> emqx_ct_helpers:boot_modules(all), emqx_ct_helpers:start_apps([]), + toggle_authz(true), Config. end_per_suite(_Config) -> @@ -35,7 +36,7 @@ end_per_suite(_Config) -> %% Test cases %%-------------------------------------------------------------------- -t_clean_acl_cache(_) -> +t_clean_authz_cache(_) -> {ok, Client} = emqtt:start_link([{clientid, <<"emqx_c">>}]), {ok, _} = emqtt:connect(Client), {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), @@ -48,15 +49,14 @@ t_clean_acl_cache(_) -> lists:last(Pids); _ -> {error, not_found} end, - Caches = gen_server:call(ClientPid, list_acl_cache), - ct:log("acl caches: ~p", [Caches]), + Caches = gen_server:call(ClientPid, list_authz_cache), + ct:log("authz caches: ~p", [Caches]), ?assert(length(Caches) > 0), - erlang:send(ClientPid, clean_acl_cache), - ?assertEqual(0, length(gen_server:call(ClientPid, list_acl_cache))), + erlang:send(ClientPid, clean_authz_cache), + ?assertEqual(0, length(gen_server:call(ClientPid, list_authz_cache))), emqtt:stop(Client). - -t_drain_acl_cache(_) -> +t_drain_authz_cache(_) -> {ok, Client} = emqtt:start_link([{clientid, <<"emqx_c">>}]), {ok, _} = emqtt:connect(Client), {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), @@ -69,80 +69,15 @@ t_drain_acl_cache(_) -> lists:last(Pids); _ -> {error, not_found} end, - Caches = gen_server:call(ClientPid, list_acl_cache), - ct:log("acl caches: ~p", [Caches]), + Caches = gen_server:call(ClientPid, list_authz_cache), + ct:log("authz caches: ~p", [Caches]), ?assert(length(Caches) > 0), - emqx_acl_cache:drain_cache(), - ?assertEqual(0, length(gen_server:call(ClientPid, list_acl_cache))), + emqx_authz_cache:drain_cache(), + ?assertEqual(0, length(gen_server:call(ClientPid, list_authz_cache))), ct:sleep(100), {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - ?assert(length(gen_server:call(ClientPid, list_acl_cache)) > 0), + ?assert(length(gen_server:call(ClientPid, list_authz_cache)) > 0), emqtt:stop(Client). -% optimize?? -t_reload_aclfile_and_cleanall(_Config) -> - - RasieMsg = fun() -> Self = self(), #{puback => fun(Msg) -> Self ! {puback, Msg} end, - disconnected => fun(_) -> ok end, - publish => fun(_) -> ok end } end, - - {ok, Client} = emqtt:start_link([{clientid, <<"emqx_c">>}, {proto_ver, v5}, - {msg_handler, RasieMsg()}]), - {ok, _} = emqtt:connect(Client), - - {ok, PktId} = emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, qos1), - - %% Success publish to broker - receive - {puback, #{packet_id := PktId, reason_code := Rc}} -> - ?assertEqual(16#10, Rc); - _ -> - ?assert(false) - end, - - %% Check acl cache list - [ClientPid] = emqx_cm:lookup_channels(<<"emqx_c">>), - ?assert(length(gen_server:call(ClientPid, list_acl_cache)) > 0), - emqtt:stop(Client). - -%% @private -testdir(DataPath) -> - Ls = filename:split(DataPath), - filename:join(lists:sublist(Ls, 1, length(Ls) - 1)). - -% t_cache_k(_) -> -% error('TODO'). - -% t_cache_v(_) -> -% error('TODO'). - -% t_cleanup_acl_cache(_) -> -% error('TODO'). - -% t_get_oldest_key(_) -> -% error('TODO'). - -% t_get_newest_key(_) -> -% error('TODO'). - -% t_get_cache_max_size(_) -> -% error('TODO'). - -% t_get_cache_size(_) -> -% error('TODO'). - -% t_dump_acl_cache(_) -> -% error('TODO'). - -% t_empty_acl_cache(_) -> -% error('TODO'). - -% t_put_acl_cache(_) -> -% error('TODO'). - -% t_get_acl_cache(_) -> -% error('TODO'). - -% t_is_enabled(_) -> -% error('TODO'). - +toggle_authz(Bool) when is_boolean(Bool) -> + emqx_config:put_zone_conf(default, [authorization, enable], Bool). diff --git a/apps/emqx/test/emqx_acl_test_mod.erl b/apps/emqx/test/emqx_authz_test_mod.erl similarity index 81% rename from apps/emqx/test/emqx_acl_test_mod.erl rename to apps/emqx/test/emqx_authz_test_mod.erl index da400f076..3786f6686 100644 --- a/apps/emqx/test/emqx_acl_test_mod.erl +++ b/apps/emqx/test/emqx_authz_test_mod.erl @@ -14,20 +14,20 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_acl_test_mod). +-module(emqx_authz_test_mod). -%% ACL callbacks +%% Authorization callbacks -export([ init/1 - , check_acl/2 + , authorize/2 , description/0 ]). -init(AclOpts) -> - {ok, AclOpts}. +init(AuthzOpts) -> + {ok, AuthzOpts}. -check_acl({_User, _PubSub, _Topic}, _State) -> +authorize({_User, _PubSub, _Topic}, _State) -> allow. description() -> - "Test ACL Mod". + "Test Authorization Mod". diff --git a/apps/emqx/test/emqx_broker_SUITE.erl b/apps/emqx/test/emqx_broker_SUITE.erl index d6fa36c18..fe754e9df 100644 --- a/apps/emqx/test/emqx_broker_SUITE.erl +++ b/apps/emqx/test/emqx_broker_SUITE.erl @@ -42,19 +42,19 @@ end_per_suite(_Config) -> %%-------------------------------------------------------------------- t_stats_fun(_) -> - ?assertEqual(0, emqx_stats:getstat('subscribers.count')), - ?assertEqual(0, emqx_stats:getstat('subscriptions.count')), - ?assertEqual(0, emqx_stats:getstat('suboptions.count')), + Subscribers = emqx_stats:getstat('subscribers.count'), + Subscriptions = emqx_stats:getstat('subscriptions.count'), + Subopts = emqx_stats:getstat('suboptions.count'), ok = emqx_broker:subscribe(<<"topic">>, <<"clientid">>), ok = emqx_broker:subscribe(<<"topic2">>, <<"clientid">>), emqx_broker:stats_fun(), ct:sleep(10), - ?assertEqual(2, emqx_stats:getstat('subscribers.count')), - ?assertEqual(2, emqx_stats:getstat('subscribers.max')), - ?assertEqual(2, emqx_stats:getstat('subscriptions.count')), - ?assertEqual(2, emqx_stats:getstat('subscriptions.max')), - ?assertEqual(2, emqx_stats:getstat('suboptions.count')), - ?assertEqual(2, emqx_stats:getstat('suboptions.max')). + ?assertEqual(Subscribers + 2, emqx_stats:getstat('subscribers.count')), + ?assertEqual(Subscribers + 2, emqx_stats:getstat('subscribers.max')), + ?assertEqual(Subscriptions + 2, emqx_stats:getstat('subscriptions.count')), + ?assertEqual(Subscriptions + 2, emqx_stats:getstat('subscriptions.max')), + ?assertEqual(Subopts + 2, emqx_stats:getstat('suboptions.count')), + ?assertEqual(Subopts + 2, emqx_stats:getstat('suboptions.max')). t_subscribed(_) -> emqx_broker:subscribe(<<"topic">>), diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 9558dfd28..be7c94ede 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -24,7 +24,152 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> + emqx_ct:all(?MODULE). + +mqtt_conf() -> + #{await_rel_timeout => 300000, + idle_timeout => 15000, + ignore_loop_deliver => false, + keepalive_backoff => 0.75, + max_awaiting_rel => 100, + max_clientid_len => 65535, + max_inflight => 32, + max_mqueue_len => 1000, + max_packet_size => 1048576, + max_qos_allowed => 2, + max_subscriptions => infinity, + max_topic_alias => 65535, + max_topic_levels => 65535, + mountpoint => <<>>, + mqueue_default_priority => lowest, + mqueue_priorities => #{}, + mqueue_store_qos0 => true, + peer_cert_as_clientid => disabled, + peer_cert_as_username => disabled, + response_information => [], + retain_available => true, + retry_interval => 30000, + server_keepalive => disabled, + session_expiry_interval => 7200000, + shared_subscription => true, + strict_mode => false, + upgrade_qos => false, + use_username_as_clientid => false, + wildcard_subscription => true}. + +listener_mqtt_tcp_conf() -> + #{acceptors => 16, + access_rules => ["allow all"], + bind => {{0,0,0,0},1883}, + max_connections => 1024000, + proxy_protocol => false, + proxy_protocol_timeout => 3000, + rate_limit => + #{conn_bytes_in => + ["100KB","10s"], + conn_messages_in => + ["100","10s"], + max_conn_rate => 1000, + quota => + #{conn_messages_routing => infinity, + overall_messages_routing => infinity}}, + tcp => + #{active_n => 100, + backlog => 1024, + buffer => 4096, + high_watermark => 1048576, + send_timeout => 15000, + send_timeout_close => + true}, + type => tcp}. + +listener_mqtt_ws_conf() -> + #{acceptors => 16, + access_rules => ["allow all"], + bind => {{0,0,0,0},8083}, + max_connections => 1024000, + proxy_protocol => false, + proxy_protocol_timeout => 3000, + rate_limit => + #{conn_bytes_in => + ["100KB","10s"], + conn_messages_in => + ["100","10s"], + max_conn_rate => 1000, + quota => + #{conn_messages_routing => infinity, + overall_messages_routing => infinity}}, + tcp => + #{active_n => 100, + backlog => 1024, + buffer => 4096, + high_watermark => 1048576, + send_timeout => 15000, + send_timeout_close => + true}, + type => ws, + websocket => + #{allow_origin_absence => + true, + check_origin_enable => + false, + check_origins => [], + compress => false, + deflate_opts => + #{client_max_window_bits => + 15, + mem_level => 8, + server_max_window_bits => + 15}, + fail_if_no_subprotocol => + true, + idle_timeout => 86400000, + max_frame_size => infinity, + mqtt_path => "/mqtt", + mqtt_piggyback => multiple, + proxy_address_header => + "x-forwarded-for", + proxy_port_header => + "x-forwarded-port", + supported_subprotocols => + ["mqtt","mqtt-v3", + "mqtt-v3.1.1", + "mqtt-v5"]}}. + +default_zone_conf() -> + #{zones => + #{default => + #{ authorization => #{ + cache => #{enable => true,max_size => 32, ttl => 60000}, + deny_action => ignore, + enable => false + }, + auth => #{enable => false}, + overall_max_connections => infinity, + stats => #{enable => true}, + conn_congestion => + #{enable_alarm => true, min_alarm_sustain_duration => 60000}, + flapping_detect => + #{ban_time => 300000,enable => false, + max_count => 15,window_time => 60000}, + force_gc => + #{bytes => 16777216,count => 16000, + enable => true}, + force_shutdown => + #{enable => true, + max_heap_size => 4194304, + max_message_queue_len => 1000}, + mqtt => mqtt_conf(), + listeners => + #{mqtt_tcp => listener_mqtt_tcp_conf(), + mqtt_ws => listener_mqtt_ws_conf()} + } + } + }. + +set_default_zone_conf() -> + emqx_config:put(default_zone_conf()). %%-------------------------------------------------------------------- %% CT Callbacks @@ -36,8 +181,8 @@ init_per_suite(Config) -> %% Access Control Meck ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), ok = meck:expect(emqx_access_control, authenticate, - fun(_) -> {ok, #{auth_result => success}} end), - ok = meck:expect(emqx_access_control, check_acl, fun(_, _, _) -> allow end), + fun(_) -> ok end), + ok = meck:expect(emqx_access_control, authorize, fun(_, _, _) -> allow end), %% Broker Meck ok = meck:new(emqx_broker, [passthrough, no_history, no_link]), %% Hooks Meck @@ -50,6 +195,9 @@ init_per_suite(Config) -> ok = meck:new(emqx_metrics, [passthrough, no_history, no_link]), ok = meck:expect(emqx_metrics, inc, fun(_) -> ok end), ok = meck:expect(emqx_metrics, inc, fun(_, _) -> ok end), + %% Ban + meck:new(emqx_banned, [passthrough, no_history, no_link]), + ok = meck:expect(emqx_banned, check, fun(_ConnInfo) -> false end), Config. end_per_suite(_Config) -> @@ -58,15 +206,15 @@ end_per_suite(_Config) -> emqx_session, emqx_broker, emqx_hooks, - emqx_cm + emqx_cm, + emqx_banned ]). init_per_testcase(_TestCase, Config) -> - meck:new(emqx_zone, [passthrough, no_history, no_link]), + set_default_zone_conf(), Config. end_per_testcase(_TestCase, Config) -> - meck:unload([emqx_zone]), Config. %%-------------------------------------------------------------------- @@ -83,7 +231,7 @@ t_chan_caps(_) -> #{max_clientid_len := 65535, max_qos_allowed := 2, max_topic_alias := 65535, - max_topic_levels := 0, + max_topic_levels := 65535, retain_available := true, shared_subscription := true, subscription_identifiers := true, @@ -120,35 +268,40 @@ t_handle_in_unexpected_packet(_) -> {ok, [{outgoing, Packet}, {close, protocol_error}], Channel} = emqx_channel:handle_in(?PUBLISH_PACKET(?QOS_0), Channel). -t_handle_in_connect_auth_failed(_) -> - ConnPkt = #mqtt_packet_connect{ - proto_name = <<"MQTT">>, - proto_ver = ?MQTT_PROTO_V5, - is_bridge = false, - clean_start = true, - keepalive = 30, - properties = #{ - 'Authentication-Method' => <<"failed_auth_method">>, - 'Authentication-Data' => <<"failed_auth_data">> - }, - clientid = <<"clientid">>, - username = <<"username">> - }, - {shutdown, not_authorized, ?CONNACK_PACKET(?RC_NOT_AUTHORIZED), _} = - emqx_channel:handle_in(?CONNECT_PACKET(ConnPkt), channel(#{conn_state => idle})). +% t_handle_in_connect_auth_failed(_) -> +% ConnPkt = #mqtt_packet_connect{ +% proto_name = <<"MQTT">>, +% proto_ver = ?MQTT_PROTO_V5, +% is_bridge = false, +% clean_start = true, +% keepalive = 30, +% properties = #{ +% 'Authentication-Method' => <<"failed_auth_method">>, +% 'Authentication-Data' => <<"failed_auth_data">> +% }, +% clientid = <<"clientid">>, +% username = <<"username">> +% }, +% {shutdown, not_authorized, ?CONNACK_PACKET(?RC_NOT_AUTHORIZED), _} = +% emqx_channel:handle_in(?CONNECT_PACKET(ConnPkt), channel(#{conn_state => idle})). t_handle_in_continue_auth(_) -> Properties = #{ 'Authentication-Method' => <<"failed_auth_method">>, 'Authentication-Data' => <<"failed_auth_data">> }, - {shutdown, bad_authentication_method, ?CONNACK_PACKET(?RC_BAD_AUTHENTICATION_METHOD), _} = - emqx_channel:handle_in(?AUTH_PACKET(?RC_CONTINUE_AUTHENTICATION,Properties), channel()), - {shutdown, not_authorized, ?CONNACK_PACKET(?RC_NOT_AUTHORIZED), _} = + + Channel1 = channel(#{conn_state => connected}), + {ok, [{outgoing, ?DISCONNECT_PACKET(?RC_PROTOCOL_ERROR)}, {close, protocol_error}], Channel1} = + emqx_channel:handle_in(?AUTH_PACKET(?RC_CONTINUE_AUTHENTICATION, Properties), Channel1), + + Channel2 = channel(#{conn_state => connecting}), + ConnInfo = emqx_channel:info(conninfo, Channel2), + Channel3 = emqx_channel:set_field(conninfo, ConnInfo#{conn_props => Properties}, Channel2), + + {ok, [{event, connected}, {connack, ?CONNACK_PACKET(?RC_SUCCESS)}], _} = emqx_channel:handle_in( - ?AUTH_PACKET(?RC_CONTINUE_AUTHENTICATION,Properties), - channel(#{conninfo => #{proto_ver => ?MQTT_PROTO_V5, conn_props => Properties}}) - ). + ?AUTH_PACKET(?RC_CONTINUE_AUTHENTICATION, Properties), Channel3). t_handle_in_re_auth(_) -> Properties = #{ @@ -167,10 +320,14 @@ t_handle_in_re_auth(_) -> ?AUTH_PACKET(?RC_RE_AUTHENTICATE,Properties), channel(#{conninfo => #{proto_ver => ?MQTT_PROTO_V5, conn_props => undefined}}) ), - {ok, [{outgoing, ?DISCONNECT_PACKET(?RC_NOT_AUTHORIZED)}, {close, not_authorized}], _} = + + Channel1 = channel(), + ConnInfo = emqx_channel:info(conninfo, Channel1), + Channel2 = emqx_channel:set_field(conninfo, ConnInfo#{conn_props => Properties}, Channel1), + + {ok, ?AUTH_PACKET(?RC_SUCCESS), _} = emqx_channel:handle_in( - ?AUTH_PACKET(?RC_RE_AUTHENTICATE,Properties), - channel(#{conninfo => #{proto_ver => ?MQTT_PROTO_V5, conn_props => Properties}}) + ?AUTH_PACKET(?RC_RE_AUTHENTICATE,Properties), Channel2 ). t_handle_in_qos0_publish(_) -> @@ -241,7 +398,7 @@ t_bad_receive_maximum(_) -> fun(true, _ClientInfo, _ConnInfo) -> {ok, #{session => session(), present => false}} end), - ok = meck:expect(emqx_zone, response_information, fun(_) -> test end), + emqx_config:put_zone_conf(default, [mqtt, response_information], test), C1 = channel(#{conn_state => idle}), {shutdown, protocol_error, _, _} = emqx_channel:handle_in( @@ -254,8 +411,8 @@ t_override_client_receive_maximum(_) -> fun(true, _ClientInfo, _ConnInfo) -> {ok, #{session => session(), present => false}} end), - ok = meck:expect(emqx_zone, response_information, fun(_) -> test end), - ok = meck:expect(emqx_zone, max_inflight, fun(_) -> 0 end), + emqx_config:put_zone_conf(default, [mqtt, response_information], test), + emqx_config:put_zone_conf(default, [mqtt, max_inflight], 0), C1 = channel(#{conn_state => idle}), ClientCapacity = 2, {ok, [{event, connected}, _ConnAck], C2} = @@ -346,8 +503,8 @@ t_handle_in_disconnect(_) -> t_handle_in_auth(_) -> Channel = channel(#{conn_state => connected}), - Packet = ?DISCONNECT_PACKET(?RC_IMPLEMENTATION_SPECIFIC_ERROR), - {ok, [{outgoing, Packet}, {close, implementation_specific_error}], Channel} = + Packet = ?DISCONNECT_PACKET(?RC_PROTOCOL_ERROR), + {ok, [{outgoing, Packet}, {close, protocol_error}], Channel} = emqx_channel:handle_in(?AUTH_PACKET(), Channel). t_handle_in_frame_error(_) -> @@ -477,7 +634,7 @@ t_handle_deliver_nl(_) -> Channel = channel(#{clientinfo => ClientInfo, session => Session}), Msg = emqx_message:make(<<"clientid">>, ?QOS_1, <<"t1">>, <<"qos1">>), NMsg = emqx_message:set_flag(nl, Msg), - {ok, Channel} = emqx_channel:handle_deliver([{deliver, <<"t1">>, NMsg}], Channel). + {ok, _} = emqx_channel:handle_deliver([{deliver, <<"t1">>, NMsg}], Channel). %%-------------------------------------------------------------------- %% Test cases for handle_out @@ -506,7 +663,7 @@ t_handle_out_connack_response_information(_) -> fun(true, _ClientInfo, _ConnInfo) -> {ok, #{session => session(), present => false}} end), - ok = meck:expect(emqx_zone, response_information, fun(_) -> test end), + emqx_config:put_zone_conf(default, [mqtt, response_information], test), IdleChannel = channel(#{conn_state => idle}), {ok, [{event, connected}, {connack, ?CONNACK_PACKET(?RC_SUCCESS, 0, #{'Response-Information' := test})}], @@ -520,7 +677,7 @@ t_handle_out_connack_not_response_information(_) -> fun(true, _ClientInfo, _ConnInfo) -> {ok, #{session => session(), present => false}} end), - ok = meck:expect(emqx_zone, response_information, fun(_) -> test end), + emqx_config:put_zone_conf(default, [mqtt, response_information], test), IdleChannel = channel(#{conn_state => idle}), {ok, [{event, connected}, {connack, ?CONNACK_PACKET(?RC_SUCCESS, 0, AckProps)}], _} = emqx_channel:handle_in( @@ -660,11 +817,8 @@ t_enrich_conninfo(_) -> t_enrich_client(_) -> {ok, _ConnPkt, _Chan} = emqx_channel:enrich_client(connpkt(), channel()). -t_check_banned(_) -> - ok = emqx_channel:check_banned(connpkt(), channel()). - t_auth_connect(_) -> - {ok, _Chan} = emqx_channel:auth_connect(connpkt(), channel()). + {ok, _, _Chan} = emqx_channel:authenticate(?CONNECT_PACKET(connpkt()), channel()). t_process_alias(_) -> Publish = #mqtt_packet_publish{topic_name = <<>>, properties = #{'Topic-Alias' => 1}}, @@ -708,20 +862,20 @@ t_packing_alias(_) -> #mqtt_packet{variable = #mqtt_packet_publish{topic_name = <<"z">>}}, channel())). -t_check_pub_acl(_) -> - ok = meck:expect(emqx_zone, enable_acl, fun(_) -> true end), +t_check_pub_authz(_) -> + emqx_config:put_zone_conf(default, [authorization, enable], true), Publish = ?PUBLISH_PACKET(?QOS_0, <<"t">>, 1, <<"payload">>), - ok = emqx_channel:check_pub_acl(Publish, channel()). + ok = emqx_channel:check_pub_authz(Publish, channel()). t_check_pub_alias(_) -> Publish = #mqtt_packet_publish{topic_name = <<>>, properties = #{'Topic-Alias' => 1}}, Channel = emqx_channel:set_field(alias_maximum, #{inbound => 10}, channel()), ok = emqx_channel:check_pub_alias(#mqtt_packet{variable = Publish}, Channel). -t_check_sub_acls(_) -> - ok = meck:expect(emqx_zone, enable_acl, fun(_) -> true end), +t_check_sub_authzs(_) -> + emqx_config:put_zone_conf(default, [authorization, enable], true), TopicFilter = {<<"t">>, ?DEFAULT_SUBOPTS}, - [{TopicFilter, 0}] = emqx_channel:check_sub_acls([TopicFilter], channel()). + [{TopicFilter, 0}] = emqx_channel:check_sub_authzs([TopicFilter], channel()). t_enrich_connack_caps(_) -> ok = meck:new(emqx_mqtt_caps, [passthrough, no_history]), @@ -763,7 +917,7 @@ t_ws_cookie_init(_) -> conn_mod => emqx_ws_connection, ws_cookie => WsCookie }, - Channel = emqx_channel:init(ConnInfo, [{zone, zone}]), + Channel = emqx_channel:init(ConnInfo, #{zone => default, listener => mqtt_tcp}), ?assertMatch(#{ws_cookie := WsCookie}, emqx_channel:info(clientinfo, Channel)). %%-------------------------------------------------------------------- @@ -788,7 +942,7 @@ channel(InitFields) -> maps:fold(fun(Field, Value, Channel) -> emqx_channel:set_field(Field, Value, Channel) end, - emqx_channel:init(ConnInfo, [{zone, zone}]), + emqx_channel:init(ConnInfo, #{zone => default, listener => mqtt_tcp}), maps:merge(#{clientinfo => clientinfo(), session => session(), conn_state => connected @@ -796,7 +950,8 @@ channel(InitFields) -> clientinfo() -> clientinfo(#{}). clientinfo(InitProps) -> - maps:merge(#{zone => zone, + maps:merge(#{zone => default, + listener => mqtt_tcp, protocol => mqtt, peerhost => {127,0,0,1}, clientid => <<"clientid">>, @@ -828,7 +983,7 @@ session(InitFields) when is_map(InitFields) -> maps:fold(fun(Field, Value, Session) -> emqx_session:set_field(Field, Value, Session) end, - emqx_session:init(#{zone => channel}, #{receive_maximum => 0}), + emqx_session:init(#{max_inflight => 0}), InitFields). %% conn: 5/s; overall: 10/s diff --git a/apps/emqx/test/emqx_client_SUITE.erl b/apps/emqx/test/emqx_client_SUITE.erl index 73a92024b..c6a450471 100644 --- a/apps/emqx/test/emqx_client_SUITE.erl +++ b/apps/emqx/test/emqx_client_SUITE.erl @@ -78,17 +78,14 @@ groups() -> init_per_suite(Config) -> emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([], fun set_special_confs/1), + emqx_ct_helpers:start_apps([]), + emqx_config:put_listener_conf(default, mqtt_ssl, [ssl, verify], verify_peer), + emqx_listeners:restart_listener('default:mqtt_ssl'), Config. end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([]). -set_special_confs(emqx) -> - emqx_ct_helpers:change_emqx_opts(ssl_twoway, [{peer_cert_as_username, cn}]); -set_special_confs(_) -> - ok. - %%-------------------------------------------------------------------- %% Test cases for MQTT v3 %%-------------------------------------------------------------------- @@ -104,8 +101,7 @@ t_basic_v4(_Config) -> t_basic([{proto_ver, v4}]). t_cm(_) -> - IdleTimeout = emqx_zone:get_env(external, idle_timeout, 30000), - emqx_zone:set_env(external, idle_timeout, 1000), + emqx_config:put_zone_conf(default, [mqtt, idle_timeout], 1000), ClientId = <<"myclient">>, {ok, C} = emqtt:start_link([{clientid, ClientId}]), {ok, _} = emqtt:connect(C), @@ -115,7 +111,7 @@ t_cm(_) -> ct:sleep(1200), Stats = emqx_cm:get_chan_stats(ClientId), ?assertEqual(1, proplists:get_value(subscriptions_cnt, Stats)), - emqx_zone:set_env(external, idle_timeout, IdleTimeout). + emqx_config:put_zone_conf(default, [mqtt, idle_timeout], 15000). t_cm_registry(_) -> Info = supervisor:which_children(emqx_cm_sup), @@ -273,15 +269,13 @@ t_basic(_Opts) -> ok = emqtt:disconnect(C). t_username_as_clientid(_) -> - emqx_zone:set_env(external, use_username_as_clientid, true), + emqx_config:put_zone_conf(default, [mqtt, use_username_as_clientid], true), Username = <<"usera">>, {ok, C} = emqtt:start_link([{username, Username}]), {ok, _} = emqtt:connect(C), #{clientinfo := #{clientid := Username}} = emqx_cm:get_chan_info(Username), emqtt:disconnect(C). - - t_certcn_as_clientid_default_config_tls(_) -> tls_certcn_as_clientid(default). @@ -329,7 +323,7 @@ tls_certcn_as_clientid(TLSVsn) -> tls_certcn_as_clientid(TLSVsn, RequiredTLSVsn) -> CN = <<"Client">>, - emqx_zone:set_env(external, use_username_as_clientid, true), + emqx_config:put_zone_conf(default, [mqtt, peer_cert_as_clientid], cn), SslConf = emqx_ct_helpers:client_ssl_twoway(TLSVsn), {ok, Client} = emqtt:start_link([{port, 8883}, {ssl, true}, {ssl_opts, SslConf}]), {ok, _} = emqtt:connect(Client), diff --git a/apps/emqx/test/emqx_cm_SUITE.erl b/apps/emqx/test/emqx_cm_SUITE.erl index 3f6950b3b..75d0a899c 100644 --- a/apps/emqx/test/emqx_cm_SUITE.erl +++ b/apps/emqx/test/emqx_cm_SUITE.erl @@ -89,7 +89,7 @@ t_open_session(_) -> ok = meck:expect(emqx_connection, call, fun(_, _) -> ok end), ok = meck:expect(emqx_connection, call, fun(_, _, _) -> ok end), - ClientInfo = #{zone => external, + ClientInfo = #{zone => default, listener => mqtt_tcp, clientid => <<"clientid">>, username => <<"username">>, peerhost => {127,0,0,1}}, @@ -114,7 +114,7 @@ rand_client_id() -> t_open_session_race_condition(_) -> ClientId = rand_client_id(), - ClientInfo = #{zone => external, + ClientInfo = #{zone => default, listener => mqtt_tcp, clientid => ClientId, username => <<"username">>, peerhost => {127,0,0,1}}, diff --git a/apps/emqx/test/emqx_cm_registry_SUITE.erl b/apps/emqx/test/emqx_cm_registry_SUITE.erl index 097bfc7b4..35f748477 100644 --- a/apps/emqx/test/emqx_cm_registry_SUITE.erl +++ b/apps/emqx/test/emqx_cm_registry_SUITE.erl @@ -42,26 +42,26 @@ end_per_testcase(_TestCase, Config) -> Config. t_is_enabled(_) -> - application:set_env(emqx, enable_session_registry, false), + emqx_config:put([broker, enable_session_registry], false), ?assertEqual(false, emqx_cm_registry:is_enabled()), - application:set_env(emqx, enable_session_registry, true), + emqx_config:put([broker, enable_session_registry], true), ?assertEqual(true, emqx_cm_registry:is_enabled()). t_register_unregister_channel(_) -> ClientId = <<"clientid">>, - application:set_env(emqx, enable_session_registry, false), + emqx_config:put([broker, enable_session_registry], false), emqx_cm_registry:register_channel(ClientId), ?assertEqual([], emqx_cm_registry:lookup_channels(ClientId)), - application:set_env(emqx, enable_session_registry, true), + emqx_config:put([broker, enable_session_registry], true), emqx_cm_registry:register_channel(ClientId), ?assertEqual([self()], emqx_cm_registry:lookup_channels(ClientId)), - application:set_env(emqx, enable_session_registry, false), + emqx_config:put([broker, enable_session_registry], false), emqx_cm_registry:unregister_channel(ClientId), ?assertEqual([self()], emqx_cm_registry:lookup_channels(ClientId)), - application:set_env(emqx, enable_session_registry, true), + emqx_config:put([broker, enable_session_registry], true), emqx_cm_registry:unregister_channel(ClientId), ?assertEqual([], emqx_cm_registry:lookup_channels(ClientId)). diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index a6b2b614a..0d5114325 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -57,6 +57,7 @@ init_per_suite(Config) -> ok = meck:expect(emqx_alarm, deactivate, fun(_) -> ok end), ok = meck:expect(emqx_alarm, deactivate, fun(_, _) -> ok end), + emqx_channel_SUITE:set_default_zone_conf(), Config. end_per_suite(_Config) -> @@ -120,14 +121,13 @@ t_info(_) -> end end), #{sockinfo := SockInfo} = emqx_connection:info(CPid), - ?assertMatch(#{active_n := 100, - peername := {{127,0,0,1},3456}, + ?assertMatch(#{ peername := {{127,0,0,1},3456}, sockname := {{127,0,0,1},1883}, sockstate := idle, socktype := tcp}, SockInfo). t_info_limiter(_) -> - St = st(#{limiter => emqx_limiter:init(external, [])}), + St = st(#{limiter => emqx_limiter:init(default, [])}), ?assertEqual(undefined, emqx_connection:info(limiter, St)). t_stats(_) -> @@ -219,8 +219,10 @@ t_handle_msg_deliver(_) -> t_handle_msg_inet_reply(_) -> ok = meck:expect(emqx_pd, get_counter, fun(_) -> 10 end), - ?assertMatch({ok, _St}, handle_msg({inet_reply, for_testing, ok}, st(#{active_n => 0}))), - ?assertEqual(ok, handle_msg({inet_reply, for_testing, ok}, st(#{active_n => 100}))), + emqx_config:put_listener_conf(default, mqtt_tcp, [tcp, active_n], 0), + ?assertMatch({ok, _St}, handle_msg({inet_reply, for_testing, ok}, st())), + emqx_config:put_listener_conf(default, mqtt_tcp, [tcp, active_n], 100), + ?assertEqual(ok, handle_msg({inet_reply, for_testing, ok}, st())), ?assertMatch({stop, {shutdown, for_testing}, _St}, handle_msg({inet_reply, for_testing, {error, for_testing}}, st())). @@ -331,12 +333,12 @@ t_ensure_rate_limit(_) -> ?assertEqual(undefined, emqx_connection:info(limiter, State)), ok = meck:expect(emqx_limiter, check, - fun(_, _) -> {ok, emqx_limiter:init(external, [])} end), + fun(_, _) -> {ok, emqx_limiter:init(default, [])} end), State1 = emqx_connection:ensure_rate_limit(#{}, st(#{limiter => #{}})), ?assertEqual(undefined, emqx_connection:info(limiter, State1)), ok = meck:expect(emqx_limiter, check, - fun(_, _) -> {pause, 3000, emqx_limiter:init(external, [])} end), + fun(_, _) -> {pause, 3000, emqx_limiter:init(default, [])} end), State2 = emqx_connection:ensure_rate_limit(#{}, st(#{limiter => #{}})), ?assertEqual(undefined, emqx_connection:info(limiter, State2)), ?assertEqual(blocked, emqx_connection:info(sockstate, State2)). @@ -386,8 +388,7 @@ t_start_link_exit_on_activate(_) -> t_get_conn_info(_) -> with_conn(fun(CPid) -> #{sockinfo := SockInfo} = emqx_connection:info(CPid), - ?assertEqual(#{active_n => 100, - peername => {{127,0,0,1},3456}, + ?assertEqual(#{peername => {{127,0,0,1},3456}, sockname => {{127,0,0,1},1883}, sockstate => running, socktype => tcp @@ -397,16 +398,12 @@ t_get_conn_info(_) -> t_oom_shutdown(init, Config) -> ok = snabbkaffe:start_trace(), ok = meck:new(emqx_misc, [non_strict, passthrough, no_history, no_link]), - ok = meck:new(emqx_zone, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_zone, oom_policy, - fun(_Zone) -> #{message_queue_len => 10, max_heap_size => 8000000} end), meck:expect(emqx_misc, check_oom, fun(_) -> {shutdown, "fake_oom"} end), Config; t_oom_shutdown('end', _Config) -> snabbkaffe:stop(), meck:unload(emqx_misc), - meck:unload(emqx_zone), ok. t_oom_shutdown(_) -> @@ -455,13 +452,11 @@ exit_on_activate_error(SockErr, Reason) -> with_conn(TestFun) -> with_conn(TestFun, #{trap_exit => false}). -with_conn(TestFun, Options) when is_map(Options) -> - with_conn(TestFun, maps:to_list(Options)); - -with_conn(TestFun, Options) -> - TrapExit = proplists:get_value(trap_exit, Options, false), +with_conn(TestFun, Opts) when is_map(Opts) -> + TrapExit = maps:get(trap_exit, Opts, false), process_flag(trap_exit, TrapExit), - {ok, CPid} = emqx_connection:start_link(emqx_transport, sock, Options), + {ok, CPid} = emqx_connection:start_link(emqx_transport, sock, + maps:merge(Opts, #{zone => default, listener => mqtt_tcp})), TestFun(CPid), TrapExit orelse emqx_connection:stop(CPid), ok. @@ -483,7 +478,8 @@ st() -> st(#{}, #{}). st(InitFields) when is_map(InitFields) -> st(InitFields, #{}). st(InitFields, ChannelFields) when is_map(InitFields) -> - St = emqx_connection:init_state(emqx_transport, sock, [#{zone => external}]), + St = emqx_connection:init_state(emqx_transport, sock, #{zone => default, + listener => mqtt_tcp}), maps:fold(fun(N, V, S) -> emqx_connection:set_field(N, V, S) end, emqx_connection:set_field(channel, channel(ChannelFields), St), InitFields @@ -503,7 +499,8 @@ channel(InitFields) -> receive_maximum => 100, expiry_interval => 0 }, - ClientInfo = #{zone => zone, + ClientInfo = #{zone => default, + listener => mqtt_tcp, protocol => mqtt, peerhost => {127,0,0,1}, clientid => <<"clientid">>, @@ -512,13 +509,11 @@ channel(InitFields) -> peercert => undefined, mountpoint => undefined }, - Session = emqx_session:init(#{zone => external}, - #{receive_maximum => 0} - ), + Session = emqx_session:init(#{max_inflight => 0}), maps:fold(fun(Field, Value, Channel) -> - emqx_channel:set_field(Field, Value, Channel) + emqx_channel:set_field(Field, Value, Channel) end, - emqx_channel:init(ConnInfo, [{zone, zone}]), + emqx_channel:init(ConnInfo, #{zone => default, listener => mqtt_tcp}), maps:merge(#{clientinfo => ClientInfo, session => Session, conn_state => connected diff --git a/apps/emqx/test/emqx_flapping_SUITE.erl b/apps/emqx/test/emqx_flapping_SUITE.erl index 8f069b747..eca276b84 100644 --- a/apps/emqx/test/emqx_flapping_SUITE.erl +++ b/apps/emqx/test/emqx_flapping_SUITE.erl @@ -25,26 +25,23 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([], fun set_special_configs/1), + emqx_ct_helpers:start_apps([]), + emqx_config:put_zone_conf(default, [flapping_detect], + #{max_count => 3, + window_time => 100, % 0.1s + ban_time => 2000 %% 2s + }), Config. -set_special_configs(emqx) -> - emqx_zone:set_env(external, enable_flapping_detect, true), - application:set_env(emqx, flapping_detect_policy, - #{threshold => 3, - duration => 100, - banned_interval => 2 - }); -set_special_configs(_App) -> ok. - end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([]), ekka_mnesia:delete_schema(), %% Clean emqx_banned table ok. t_detect_check(_) -> - ClientInfo = #{zone => external, - clientid => <<"clientid">>, + ClientInfo = #{zone => default, + listener => mqtt_tcp, + clientid => <<"client007">>, peerhost => {127,0,0,1} }, false = emqx_flapping:detect(ClientInfo), @@ -53,6 +50,8 @@ t_detect_check(_) -> false = emqx_banned:check(ClientInfo), true = emqx_flapping:detect(ClientInfo), timer:sleep(50), + ct:pal("the table emqx_banned: ~p, nowsec: ~p", [ets:tab2list(emqx_banned), + erlang:system_time(second)]), true = emqx_banned:check(ClientInfo), timer:sleep(3000), false = emqx_banned:check(ClientInfo), @@ -64,12 +63,13 @@ t_detect_check(_) -> ok = emqx_flapping:stop(). t_expired_detecting(_) -> - ClientInfo = #{zone => external, - clientid => <<"clientid">>, + ClientInfo = #{zone => default, + listener => mqtt_tcp, + clientid => <<"client008">>, peerhost => {127,0,0,1}}, false = emqx_flapping:detect(ClientInfo), - ?assertEqual(true, lists:any(fun({flapping, <<"clientid">>, _, _, _}) -> true; + ?assertEqual(true, lists:any(fun({flapping, <<"client008">>, _, _, _}) -> true; (_) -> false end, ets:tab2list(emqx_flapping))), timer:sleep(200), - ?assertEqual(true, lists:all(fun({flapping, <<"clientid">>, _, _, _}) -> false; + ?assertEqual(true, lists:all(fun({flapping, <<"client008">>, _, _, _}) -> false; (_) -> true end, ets:tab2list(emqx_flapping))). \ No newline at end of file diff --git a/apps/emqx/test/emqx_global_gc_SUITE.erl b/apps/emqx/test/emqx_global_gc_SUITE.erl index 92d6a5251..dcdeeae57 100644 --- a/apps/emqx/test/emqx_global_gc_SUITE.erl +++ b/apps/emqx/test/emqx_global_gc_SUITE.erl @@ -24,7 +24,7 @@ all() -> emqx_ct:all(?MODULE). t_run_gc(_) -> - ok = application:set_env(emqx, global_gc_interval, 1), + ok = emqx_config:put([node, global_gc_interval], 1000), {ok, _} = emqx_global_gc:start_link(), ok = timer:sleep(1500), {ok, MilliSecs} = emqx_global_gc:run(), diff --git a/apps/emqx/test/emqx_hooks_SUITE.erl b/apps/emqx/test/emqx_hooks_SUITE.erl index 49ba88934..be8814ca4 100644 --- a/apps/emqx/test/emqx_hooks_SUITE.erl +++ b/apps/emqx/test/emqx_hooks_SUITE.erl @@ -38,15 +38,22 @@ all() -> emqx_ct:all(?MODULE). % t_add(_) -> % error('TODO'). -t_add_del_hook(_) -> +t_add_put_del_hook(_) -> {ok, _} = emqx_hooks:start_link(), ok = emqx:hook(test_hook, {?MODULE, hook_fun1, []}), ok = emqx:hook(test_hook, {?MODULE, hook_fun2, []}), ?assertEqual({error, already_exists}, emqx:hook(test_hook, {?MODULE, hook_fun2, []})), - Callbacks = [{callback, {?MODULE, hook_fun1, []}, undefined, 0}, - {callback, {?MODULE, hook_fun2, []}, undefined, 0}], - ?assertEqual(Callbacks, emqx_hooks:lookup(test_hook)), + Callbacks0 = [{callback, {?MODULE, hook_fun1, []}, undefined, 0}, + {callback, {?MODULE, hook_fun2, []}, undefined, 0}], + ?assertEqual(Callbacks0, emqx_hooks:lookup(test_hook)), + + ok = emqx_hooks:put(test_hook, {?MODULE, hook_fun1, [test]}), + ok = emqx_hooks:put(test_hook, {?MODULE, hook_fun2, [test]}), + Callbacks1 = [{callback, {?MODULE, hook_fun1, [test]}, undefined, 0}, + {callback, {?MODULE, hook_fun2, [test]}, undefined, 0}], + ?assertEqual(Callbacks1, emqx_hooks:lookup(test_hook)), + ok = emqx:unhook(test_hook, {?MODULE, hook_fun1}), ok = emqx:unhook(test_hook, {?MODULE, hook_fun2}), timer:sleep(200), @@ -61,10 +68,21 @@ t_add_del_hook(_) -> {callback, {?MODULE, hook_fun8, []}, undefined, 8}, {callback, {?MODULE, hook_fun2, []}, undefined, 2}], ?assertEqual(Callbacks2, emqx_hooks:lookup(emqx_hook)), - ok = emqx:unhook(emqx_hook, {?MODULE, hook_fun2, []}), - ok = emqx:unhook(emqx_hook, {?MODULE, hook_fun8, []}), - ok = emqx:unhook(emqx_hook, {?MODULE, hook_fun9, []}), - ok = emqx:unhook(emqx_hook, {?MODULE, hook_fun10, []}), + + ok = emqx_hooks:put(emqx_hook, {?MODULE, hook_fun8, [test]}, 3), + ok = emqx_hooks:put(emqx_hook, {?MODULE, hook_fun2, [test]}, 4), + ok = emqx_hooks:put(emqx_hook, {?MODULE, hook_fun10, [test]}, 1), + ok = emqx_hooks:put(emqx_hook, {?MODULE, hook_fun9, [test]}, 2), + Callbacks3 = [{callback, {?MODULE, hook_fun2, [test]}, undefined, 4}, + {callback, {?MODULE, hook_fun8, [test]}, undefined, 3}, + {callback, {?MODULE, hook_fun9, [test]}, undefined, 2}, + {callback, {?MODULE, hook_fun10, [test]}, undefined, 1}], + ?assertEqual(Callbacks3, emqx_hooks:lookup(emqx_hook)), + + ok = emqx:unhook(emqx_hook, {?MODULE, hook_fun2, [test]}), + ok = emqx:unhook(emqx_hook, {?MODULE, hook_fun8, [test]}), + ok = emqx:unhook(emqx_hook, {?MODULE, hook_fun9, [test]}), + ok = emqx:unhook(emqx_hook, {?MODULE, hook_fun10, [test]}), timer:sleep(200), ?assertEqual([], emqx_hooks:lookup(emqx_hook)), ok = emqx_hooks:stop(). diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index 53f388dfa..a8760c7e8 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -28,6 +28,7 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> NewConfig = generate_config(), application:ensure_all_started(esockd), + application:ensure_all_started(quicer), application:ensure_all_started(cowboy), lists:foreach(fun set_app_env/1, NewConfig), Config. @@ -70,8 +71,8 @@ generate_config() -> hocon_schema:generate(emqx_schema, Conf). set_app_env({App, Lists}) -> - lists:foreach(fun({acl_file, _Var}) -> - application:set_env(App, acl_file, local_path(["etc", "acl.conf"])); + lists:foreach(fun({authz_file, _Var}) -> + application:set_env(App, authz_file, local_path(["etc", "authz.conf"])); ({plugins_loaded_file, _Var}) -> application:set_env(App, plugins_loaded_file, diff --git a/apps/emqx/test/emqx_misc_SUITE.erl b/apps/emqx/test/emqx_misc_SUITE.erl index f933fb498..c3580545a 100644 --- a/apps/emqx/test/emqx_misc_SUITE.erl +++ b/apps/emqx/test/emqx_misc_SUITE.erl @@ -119,8 +119,9 @@ t_index_of(_) -> ?assertEqual(3, emqx_misc:index_of(a, [b, c, a, e, f])). t_check(_) -> - Policy = #{message_queue_len => 10, - max_heap_size => 1024 * 1024 * 8}, + Policy = #{max_message_queue_len => 10, + max_heap_size => 1024 * 1024 * 8, + enable => true}, [self() ! {msg, I} || I <- lists:seq(1, 5)], ?assertEqual(ok, emqx_misc:check_oom(Policy)), [self() ! {msg, I} || I <- lists:seq(1, 6)], diff --git a/apps/emqx/test/emqx_mqtt_SUITE.erl b/apps/emqx/test/emqx_mqtt_SUITE.erl index c86d6334a..42a4e5780 100644 --- a/apps/emqx/test/emqx_mqtt_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_SUITE.erl @@ -156,6 +156,15 @@ t_async_set_keepalive('end', _Config) -> ok. t_async_set_keepalive(_) -> + case os:type() of + {unix, darwin} -> + %% Mac OSX don't support the feature + ok; + _ -> + do_async_set_keepalive() + end. + +do_async_set_keepalive() -> ClientID = <<"client-tcp-keepalive">>, {ok, Client} = emqtt:start_link([{host, "localhost"}, {proto_ver,v5}, diff --git a/apps/emqx/test/emqx_mqtt_caps_SUITE.erl b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl index d6cd5925b..c01420f49 100644 --- a/apps/emqx/test/emqx_mqtt_caps_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl @@ -25,39 +25,36 @@ all() -> emqx_ct:all(?MODULE). t_check_pub(_) -> - PubCaps = #{max_qos_allowed => ?QOS_1, - retain_available => false - }, - emqx_zone:set_env(zone, '$mqtt_pub_caps', PubCaps), + OldConf = emqx_config:get([zones]), + emqx_config:put_zone_conf(default, [mqtt, max_qos_allowed], ?QOS_1), + emqx_config:put_zone_conf(default, [mqtt, retain_available], false), timer:sleep(50), - ok = emqx_mqtt_caps:check_pub(zone, #{qos => ?QOS_1, - retain => false}), + ok = emqx_mqtt_caps:check_pub(default, #{qos => ?QOS_1, retain => false}), PubFlags1 = #{qos => ?QOS_2, retain => false}, ?assertEqual({error, ?RC_QOS_NOT_SUPPORTED}, - emqx_mqtt_caps:check_pub(zone, PubFlags1)), + emqx_mqtt_caps:check_pub(default, PubFlags1)), PubFlags2 = #{qos => ?QOS_1, retain => true}, ?assertEqual({error, ?RC_RETAIN_NOT_SUPPORTED}, - emqx_mqtt_caps:check_pub(zone, PubFlags2)), - emqx_zone:unset_env(zone, '$mqtt_pub_caps'). + emqx_mqtt_caps:check_pub(default, PubFlags2)), + emqx_config:put([zones], OldConf). t_check_sub(_) -> + OldConf = emqx_config:get([zones]), SubOpts = #{rh => 0, rap => 0, nl => 0, qos => ?QOS_2 }, - SubCaps = #{max_topic_levels => 2, - max_qos_allowed => ?QOS_2, - shared_subscription => false, - wildcard_subscription => false - }, - emqx_zone:set_env(zone, '$mqtt_sub_caps', SubCaps), + emqx_config:put_zone_conf(default, [mqtt, max_topic_levels], 2), + emqx_config:put_zone_conf(default, [mqtt, max_qos_allowed], ?QOS_1), + emqx_config:put_zone_conf(default, [mqtt, shared_subscription], false), + emqx_config:put_zone_conf(default, [mqtt, wildcard_subscription], false), timer:sleep(50), - ok = emqx_mqtt_caps:check_sub(zone, <<"topic">>, SubOpts), + ok = emqx_mqtt_caps:check_sub(default, <<"topic">>, SubOpts), ?assertEqual({error, ?RC_TOPIC_FILTER_INVALID}, - emqx_mqtt_caps:check_sub(zone, <<"a/b/c/d">>, SubOpts)), + emqx_mqtt_caps:check_sub(default, <<"a/b/c/d">>, SubOpts)), ?assertEqual({error, ?RC_WILDCARD_SUBSCRIPTIONS_NOT_SUPPORTED}, - emqx_mqtt_caps:check_sub(zone, <<"+/#">>, SubOpts)), + emqx_mqtt_caps:check_sub(default, <<"+/#">>, SubOpts)), ?assertEqual({error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED}, - emqx_mqtt_caps:check_sub(zone, <<"topic">>, SubOpts#{share => true})), - emqx_zone:unset_env(zone, '$mqtt_pub_caps'). + emqx_mqtt_caps:check_sub(default, <<"topic">>, SubOpts#{share => true})), + emqx_config:put([zones], OldConf). diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl index 8ce35b50c..8f82d83bb 100644 --- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl @@ -23,6 +23,7 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("common_test/include/ct.hrl"). -import(lists, [nth/2]). @@ -32,18 +33,37 @@ -define(WILD_TOPICS, [<<"TopicA/+">>, <<"+/C">>, <<"#">>, <<"/#">>, <<"/+">>, <<"+/+">>, <<"TopicA/#">>]). -all() -> emqx_ct:all(?MODULE). +all() -> + [ {group, tcp} + , {group, quic} + ]. + +groups() -> + TCs = emqx_ct:all(?MODULE), + [ {tcp, [], TCs} + , {quic, [], TCs} + ]. + +init_per_group(tcp, Config) -> + emqx_ct_helpers:start_apps([]), + [ {port, 1883}, {conn_fun, connect} | Config]; +init_per_group(quic, Config) -> + emqx_ct_helpers:start_apps([]), + [ {port, 14567}, {conn_fun, quic_connect} | Config]; +init_per_group(_, Config) -> + emqx_ct_helpers:stop_apps([]), + Config. + +end_per_group(_Group, _Config) -> + ok. init_per_suite(Config) -> - %% Meck emqtt - ok = meck:new(emqtt, [non_strict, passthrough, no_history, no_link]), %% Start Apps emqx_ct_helpers:boot_modules(all), emqx_ct_helpers:start_apps([]), Config. end_per_suite(_Config) -> - ok = meck:unload(emqtt), emqx_ct_helpers:stop_apps([]). init_per_testcase(TestCase, Config) -> @@ -97,9 +117,10 @@ waiting_client_process_exit(C) -> 1000 -> error({waiting_timeout, C}) end. -clean_retained(Topic) -> - {ok, Clean} = emqtt:start_link([{clean_start, true}]), - {ok, _} = emqtt:connect(Clean), +clean_retained(Topic, Config) -> + ConnFun = ?config(conn_fun, Config), + {ok, Clean} = emqtt:start_link([{clean_start, true} | Config]), + {ok, _} = emqtt:ConnFun(Clean), {ok, _} = emqtt:publish(Clean, Topic, #{}, <<"">>, [{qos, ?QOS_1}, {retain, true}]), ok = emqtt:disconnect(Clean). @@ -107,11 +128,12 @@ clean_retained(Topic) -> %% Test Cases %%-------------------------------------------------------------------- -t_basic_test(_) -> +t_basic_test(Config) -> + ConnFun = ?config(conn_fun, Config), Topic = nth(1, ?TOPICS), ct:print("Basic test starting"), - {ok, C} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(C), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(C), {ok, _, [1]} = emqtt:subscribe(C, Topic, qos1), {ok, _, [2]} = emqtt:subscribe(C, Topic, qos2), {ok, _} = emqtt:publish(C, Topic, <<"qos 2">>, 2), @@ -124,16 +146,17 @@ t_basic_test(_) -> %% Connection %%-------------------------------------------------------------------- -t_connect_clean_start(_) -> +t_connect_clean_start(Config) -> + ConnFun = ?config(conn_fun, Config), process_flag(trap_exit, true), {ok, Client1} = emqtt:start_link([{clientid, <<"t_connect_clean_start">>}, - {proto_ver, v5},{clean_start, true}]), - {ok, _} = emqtt:connect(Client1), + {proto_ver, v5},{clean_start, true} | Config]), + {ok, _} = emqtt:ConnFun(Client1), ?assertEqual(0, client_info(session_present, Client1)), %% [MQTT-3.1.2-4] ok = emqtt:pause(Client1), {ok, Client2} = emqtt:start_link([{clientid, <<"t_connect_clean_start">>}, - {proto_ver, v5},{clean_start, false}]), - {ok, _} = emqtt:connect(Client2), + {proto_ver, v5},{clean_start, false} | Config]), + {ok, _} = emqtt:ConnFun(Client2), ?assertEqual(1, client_info(session_present, Client2)), %% [MQTT-3.1.2-5] ?assertEqual(142, receive_disconnect_reasoncode()), waiting_client_process_exit(Client1), @@ -142,32 +165,32 @@ t_connect_clean_start(_) -> waiting_client_process_exit(Client2), {ok, Client3} = emqtt:start_link([{clientid, <<"new_client">>}, - {proto_ver, v5},{clean_start, false}]), - {ok, _} = emqtt:connect(Client3), + {proto_ver, v5},{clean_start, false} | Config]), + {ok, _} = emqtt:ConnFun(Client3), ?assertEqual(0, client_info(session_present, Client3)), %% [MQTT-3.1.2-6] ok = emqtt:disconnect(Client3), waiting_client_process_exit(Client3), process_flag(trap_exit, false). -t_connect_will_message(_) -> +t_connect_will_message(Config) -> + ConnFun = ?config(conn_fun, Config), Topic = nth(1, ?TOPICS), Payload = "will message", - {ok, Client1} = emqtt:start_link([ - {proto_ver, v5}, - {clean_start, true}, - {will_flag, true}, - {will_topic, Topic}, - {will_payload, Payload} - ]), - {ok, _} = emqtt:connect(Client1), + {ok, Client1} = emqtt:start_link([ {proto_ver, v5}, + {clean_start, true}, + {will_flag, true}, + {will_topic, Topic}, + {will_payload, Payload} | Config + ]), + {ok, _} = emqtt:ConnFun(Client1), [ClientPid] = emqx_cm:lookup_channels(client_info(clientid, Client1)), Info = emqx_connection:info(sys:get_state(ClientPid)), ?assertNotEqual(undefined, maps:find(will_msg, Info)), %% [MQTT-3.1.2-7] - {ok, Client2} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client2), + {ok, Client2} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client2), {ok, _, [2]} = emqtt:subscribe(Client2, Topic, qos2), ok = emqtt:disconnect(Client1, 4), %% [MQTT-3.14.2-1] @@ -178,27 +201,34 @@ t_connect_will_message(_) -> ?assertEqual({ok, 0}, maps:find(qos, Msg)), ok = emqtt:disconnect(Client2), - {ok, Client3} = emqtt:start_link([ - {proto_ver, v5}, - {clean_start, true}, - {will_flag, true}, - {will_topic, Topic}, - {will_payload, Payload} - ]), - {ok, _} = emqtt:connect(Client3), + {ok, Client3} = emqtt:start_link([ {proto_ver, v5}, + {clean_start, true}, + {will_flag, true}, + {will_topic, Topic}, + {will_payload, Payload} | Config + ]), + {ok, _} = emqtt:ConnFun(Client3), - {ok, Client4} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client4), + {ok, Client4} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client4), {ok, _, [2]} = emqtt:subscribe(Client4, Topic, qos2), ok = emqtt:disconnect(Client3), ?assertEqual(0, length(receive_messages(1))), %% [MQTT-3.1.2-10] ok = emqtt:disconnect(Client4). -t_batch_subscribe(_) -> - {ok, Client} = emqtt:start_link([{proto_ver, v5}, {clientid, <<"batch_test">>}]), - {ok, _} = emqtt:connect(Client), - application:set_env(emqx, enable_acl_cache, false), - application:set_env(emqx, acl_nomatch, deny), +t_batch_subscribe(init, Config) -> + emqx_config:put_zone_conf(default, [authorization, enable], true), + ok = meck:new(emqx_access_control, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_access_control, authorize, fun(_, _, _) -> deny end), + Config; +t_batch_subscribe('end', _Config) -> + emqx_config:put_zone_conf(default, [authorization, enable], false), + meck:unload(emqx_access_control). + +t_batch_subscribe(Config) -> + ConnFun = ?config(conn_fun, Config), + {ok, Client} = emqtt:start_link([{proto_ver, v5}, {clientid, <<"batch_test">>} | Config]), + {ok, _} = emqtt:ConnFun(Client), {ok, _, [?RC_NOT_AUTHORIZED, ?RC_NOT_AUTHORIZED, ?RC_NOT_AUTHORIZED]} = emqtt:subscribe(Client, [{<<"t1">>, qos1}, @@ -209,25 +239,25 @@ t_batch_subscribe(_) -> ?RC_NO_SUBSCRIPTION_EXISTED]} = emqtt:unsubscribe(Client, [<<"t1">>, <<"t2">>, <<"t3">>]), - application:set_env(emqx, acl_nomatch, allow), emqtt:disconnect(Client). -t_connect_will_retain(_) -> +t_connect_will_retain(Config) -> + ConnFun = ?config(conn_fun, Config), + process_flag(trap_exit, true), Topic = nth(1, ?TOPICS), Payload = "will message", - {ok, Client1} = emqtt:start_link([ - {proto_ver, v5}, - {clean_start, true}, - {will_flag, true}, - {will_topic, Topic}, - {will_payload, Payload}, - {will_retain, false} - ]), - {ok, _} = emqtt:connect(Client1), + {ok, Client1} = emqtt:start_link([ {proto_ver, v5}, + {clean_start, true}, + {will_flag, true}, + {will_topic, Topic}, + {will_payload, Payload}, + {will_retain, false} | Config + ]), + {ok, _} = emqtt:ConnFun(Client1), - {ok, Client2} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client2), + {ok, Client2} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client2), {ok, _, [2]} = emqtt:subscribe(Client2, #{}, [{Topic, [{rap, true}, {qos, 2}]}]), ok = emqtt:disconnect(Client1, 4), @@ -235,75 +265,50 @@ t_connect_will_retain(_) -> ?assertEqual({ok, false}, maps:find(retain, Msg1)), %% [MQTT-3.1.2-14] ok = emqtt:disconnect(Client2), - {ok, Client3} = emqtt:start_link([ - {proto_ver, v5}, - {clean_start, true}, - {will_flag, true}, - {will_topic, Topic}, - {will_payload, Payload}, - {will_retain, true} - ]), - {ok, _} = emqtt:connect(Client3), + {ok, Client3} = emqtt:start_link([ {proto_ver, v5}, + {clean_start, true}, + {will_flag, true}, + {will_topic, Topic}, + {will_payload, Payload}, + {will_retain, true} | Config + ]), + {ok, _} = emqtt:ConnFun(Client3), - {ok, Client4} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client4), + {ok, Client4} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client4), {ok, _, [2]} = emqtt:subscribe(Client4, #{}, [{Topic, [{rap, true}, {qos, 2}]}]), ok = emqtt:disconnect(Client3, 4), [Msg2 | _ ] = receive_messages(1), ?assertEqual({ok, true}, maps:find(retain, Msg2)), %% [MQTT-3.1.2-15] ok = emqtt:disconnect(Client4), - clean_retained(Topic). + clean_retained(Topic, Config). -t_connect_idle_timeout(_) -> +t_connect_idle_timeout(_Config) -> IdleTimeout = 2000, - emqx_zone:set_env(external, idle_timeout, IdleTimeout), - + emqx_config:put_zone_conf(default, [mqtt, idle_timeout], IdleTimeout), + emqx_config:put_zone_conf(default, [mqtt, idle_timeout], IdleTimeout), {ok, Sock} = emqtt_sock:connect({127,0,0,1}, 1883, [], 60000), timer:sleep(IdleTimeout), ?assertMatch({error, closed}, emqtt_sock:recv(Sock,1024)). -t_connect_limit_timeout(_) -> - ok = meck:new(proplists, [non_strict, passthrough, no_history, no_link, unstick]), - meck:expect(proplists, get_value, fun(active_n, _Options, _Default) -> 1; - (Arg1, ARg2, Arg3) -> meck:passthrough([Arg1, ARg2, Arg3]) - end), - - Topic = nth(1, ?TOPICS), - emqx_zone:set_env(external, publish_limit, {3, 5}), - - {ok, Client} = emqtt:start_link([{proto_ver, v5},{keepalive, 60}]), - {ok, _} = emqtt:connect(Client), - [ClientPid] = emqx_cm:lookup_channels(client_info(clientid, Client)), - - ?assertEqual(undefined, emqx_connection:info(limit_timer, sys:get_state(ClientPid))), - Payload = <<"t_shared_subscriptions_client_terminates_when_qos_eq_2">>, - ok = emqtt:publish(Client, Topic, Payload, 0), - ok = emqtt:publish(Client, Topic, Payload, 0), - ok = emqtt:publish(Client, Topic, Payload, 0), - timer:sleep(200), - ?assert(is_reference(emqx_connection:info(limit_timer, sys:get_state(ClientPid)))), - - ok = emqtt:disconnect(Client), - emqx_zone:set_env(external, publish_limit, undefined), - meck:unload(proplists). - t_connect_emit_stats_timeout(init, Config) -> NewIdleTimeout = 1000, - OldIdleTimeout = emqx_zone:get_env(external, idle_timeout), - emqx_zone:set_env(external, idle_timeout, NewIdleTimeout), + emqx_config:put_zone_conf(default, [mqtt, idle_timeout], NewIdleTimeout), + emqx_config:put_zone_conf(default, [mqtt, idle_timeout], NewIdleTimeout), ok = snabbkaffe:start_trace(), - [{idle_timeout, NewIdleTimeout}, {old_idle_timeout, OldIdleTimeout} | Config]; -t_connect_emit_stats_timeout('end', Config) -> + [{idle_timeout, NewIdleTimeout} | Config]; +t_connect_emit_stats_timeout('end', _Config) -> snabbkaffe:stop(), - {_, OldIdleTimeout} = lists:keyfind(old_idle_timeout, 1, Config), - emqx_zone:set_env(external, idle_timeout, OldIdleTimeout), + emqx_config:put_zone_conf(default, [mqtt, idle_timeout], 15000), + emqx_config:put_zone_conf(default, [mqtt, idle_timeout], 15000), ok. t_connect_emit_stats_timeout(Config) -> + ConnFun = ?config(conn_fun, Config), {_, IdleTimeout} = lists:keyfind(idle_timeout, 1, Config), - {ok, Client} = emqtt:start_link([{proto_ver, v5},{keepalive, 60}]), - {ok, _} = emqtt:connect(Client), + {ok, Client} = emqtt:start_link([{proto_ver, v5},{keepalive, 60} | Config]), + {ok, _} = emqtt:ConnFun(Client), [ClientPid] = emqx_cm:lookup_channels(client_info(clientid, Client)), ?assert(is_reference(emqx_connection:info(stats_timer, sys:get_state(ClientPid)))), ?block_until(#{?snk_kind := cancel_stats_timer}, IdleTimeout * 2, _BackInTime = 0), @@ -311,15 +316,16 @@ t_connect_emit_stats_timeout(Config) -> ok = emqtt:disconnect(Client). %% [MQTT-3.1.2-22] -t_connect_keepalive_timeout(_) -> +t_connect_keepalive_timeout(Config) -> + ConnFun = ?config(conn_fun, Config), %% Prevent the emqtt client bringing us down on the disconnect. process_flag(trap_exit, true), Keepalive = 2, {ok, Client} = emqtt:start_link([{proto_ver, v5}, - {keepalive, Keepalive}]), - {ok, _} = emqtt:connect(Client), + {keepalive, Keepalive} | Config]), + {ok, _} = emqtt:ConnFun(Client), emqtt:pause(Client), receive {disconnected, ReasonCode, _Channel} -> ?assertEqual(141, ReasonCode) @@ -328,30 +334,30 @@ t_connect_keepalive_timeout(_) -> end. %% [MQTT-3.1.2-23] -t_connect_session_expiry_interval(_) -> +t_connect_session_expiry_interval(Config) -> + ConnFun = ?config(conn_fun, Config), Topic = nth(1, ?TOPICS), Payload = "test message", - {ok, Client1} = emqtt:start_link([ - {clientid, <<"t_connect_session_expiry_interval">>}, - {proto_ver, v5}, - {properties, #{'Session-Expiry-Interval' => 7200}} + {ok, Client1} = emqtt:start_link([ {clientid, <<"t_connect_session_expiry_interval">>}, + {proto_ver, v5}, + {properties, #{'Session-Expiry-Interval' => 7200}} + | Config ]), - {ok, _} = emqtt:connect(Client1), + {ok, _} = emqtt:ConnFun(Client1), {ok, _, [2]} = emqtt:subscribe(Client1, Topic, qos2), ok = emqtt:disconnect(Client1), - {ok, Client2} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client2), + {ok, Client2} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client2), {ok, 2} = emqtt:publish(Client2, Topic, Payload, 2), ok = emqtt:disconnect(Client2), - {ok, Client3} = emqtt:start_link([ - {clientid, <<"t_connect_session_expiry_interval">>}, - {proto_ver, v5}, - {clean_start, false} + {ok, Client3} = emqtt:start_link([ {clientid, <<"t_connect_session_expiry_interval">>}, + {proto_ver, v5}, + {clean_start, false} | Config ]), - {ok, _} = emqtt:connect(Client3), + {ok, _} = emqtt:ConnFun(Client3), [Msg | _ ] = receive_messages(1), ?assertEqual({ok, iolist_to_binary(Topic)}, maps:find(topic, Msg)), ?assertEqual({ok, iolist_to_binary(Payload)}, maps:find(payload, Msg)), @@ -360,13 +366,13 @@ t_connect_session_expiry_interval(_) -> %% [MQTT-3.1.3-9] %% !!!REFACTOR NEED: -%t_connect_will_delay_interval(_) -> +%t_connect_will_delay_interval(Config) -> % process_flag(trap_exit, true), % Topic = nth(1, ?TOPICS), % Payload = "will message", % -% {ok, Client1} = emqtt:start_link([{proto_ver, v5}]), -% {ok, _} = emqtt:connect(Client1), +% {ok, Client1} = emqtt:start_link([{proto_ver, v5} | Config]), +% {ok, _} = emqtt:ConnFun(Client1), % {ok, _, [2]} = emqtt:subscribe(Client1, Topic, qos2), % % {ok, Client2} = emqtt:start_link([ @@ -379,9 +385,9 @@ t_connect_session_expiry_interval(_) -> % {will_payload, Payload}, % {will_props, #{'Will-Delay-Interval' => 3}}, % {properties, #{'Session-Expiry-Interval' => 7200}}, -% {keepalive, 2} +% {keepalive, 2} | Config % ]), -% {ok, _} = emqtt:connect(Client2), +% {ok, _} = emqtt:ConnFun(Client2), % timer:sleep(50), % erlang:exit(Client2, kill), % timer:sleep(2000), @@ -399,9 +405,9 @@ t_connect_session_expiry_interval(_) -> % {will_payload, Payload}, % {will_props, #{'Will-Delay-Interval' => 7200}}, % {properties, #{'Session-Expiry-Interval' => 3}}, -% {keepalive, 2} +% {keepalive, 2} | Config % ]), -% {ok, _} = emqtt:connect(Client3), +% {ok, _} = emqtt:ConnFun(Client3), % timer:sleep(50), % erlang:exit(Client3, kill), % @@ -418,18 +424,17 @@ t_connect_session_expiry_interval(_) -> % process_flag(trap_exit, false). %% [MQTT-3.1.4-3] -t_connect_duplicate_clientid(_) -> +t_connect_duplicate_clientid(Config) -> + ConnFun = ?config(conn_fun, Config), process_flag(trap_exit, true), - {ok, Client1} = emqtt:start_link([ - {clientid, <<"t_connect_duplicate_clientid">>}, - {proto_ver, v5} - ]), - {ok, _} = emqtt:connect(Client1), - {ok, Client2} = emqtt:start_link([ - {clientid, <<"t_connect_duplicate_clientid">>}, - {proto_ver, v5} - ]), - {ok, _} = emqtt:connect(Client2), + {ok, Client1} = emqtt:start_link([ {clientid, <<"t_connect_duplicate_clientid">>}, + {proto_ver, v5} | Config + ]), + {ok, _} = emqtt:ConnFun(Client1), + {ok, Client2} = emqtt:start_link([ {clientid, <<"t_connect_duplicate_clientid">>}, + {proto_ver, v5} | Config + ]), + {ok, _} = emqtt:ConnFun(Client2), ?assertEqual(142, receive_disconnect_reasoncode()), waiting_client_process_exit(Client1), @@ -441,38 +446,43 @@ t_connect_duplicate_clientid(_) -> %% Connack %%-------------------------------------------------------------------- -t_connack_session_present(_) -> - {ok, Client1} = emqtt:start_link([ - {clientid, <<"t_connect_duplicate_clientid">>}, - {proto_ver, v5}, - {properties, #{'Session-Expiry-Interval' => 7200}}, - {clean_start, true} - ]), - {ok, _} = emqtt:connect(Client1), +t_connack_session_present(Config) -> + ConnFun = ?config(conn_fun, Config), + {ok, Client1} = emqtt:start_link([ {clientid, <<"t_connect_duplicate_clientid">>}, + {proto_ver, v5}, + {properties, #{'Session-Expiry-Interval' => 7200}}, + {clean_start, true} | Config + ]), + {ok, _} = emqtt:ConnFun(Client1), ?assertEqual(0, client_info(session_present, Client1)), %% [MQTT-3.2.2-2] ok = emqtt:disconnect(Client1), - {ok, Client2} = emqtt:start_link([ - {clientid, <<"t_connect_duplicate_clientid">>}, - {proto_ver, v5}, - {properties, #{'Session-Expiry-Interval' => 7200}}, - {clean_start, false} - ]), - {ok, _} = emqtt:connect(Client2), + {ok, Client2} = emqtt:start_link([ {clientid, <<"t_connect_duplicate_clientid">>}, + {proto_ver, v5}, + {properties, #{'Session-Expiry-Interval' => 7200}}, + {clean_start, false} | Config + ]), + {ok, _} = emqtt:ConnFun(Client2), ?assertEqual(1, client_info(session_present, Client2)), %% [[MQTT-3.2.2-3]] ok = emqtt:disconnect(Client2). -t_connack_max_qos_allowed(_) -> +t_connack_max_qos_allowed(init, Config) -> + Config; +t_connack_max_qos_allowed('end', _Config) -> + emqx_config:put_zone_conf(default, [mqtt, max_qos_allowed], 2), + emqx_config:put_zone_conf(default, [mqtt, max_qos_allowed], 2), + ok. +t_connack_max_qos_allowed(Config) -> + ConnFun = ?config(conn_fun, Config), process_flag(trap_exit, true), Topic = nth(1, ?TOPICS), %% max_qos_allowed = 0 - emqx_zone:set_env(external, max_qos_allowed, 0), - persistent_term:erase({emqx_zone, external, '$mqtt_caps'}), - persistent_term:erase({emqx_zone, external, '$mqtt_pub_caps'}), + emqx_config:put_zone_conf(default, [mqtt, max_qos_allowed], 0), + emqx_config:put_zone_conf(default, [mqtt, max_qos_allowed], 0), - {ok, Client1} = emqtt:start_link([{proto_ver, v5}]), - {ok, Connack1} = emqtt:connect(Client1), + {ok, Client1} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, Connack1} = emqtt:ConnFun(Client1), ?assertEqual(0, maps:get('Maximum-QoS', Connack1)), %% [MQTT-3.2.2-9] {ok, _, [0]} = emqtt:subscribe(Client1, Topic, 0), %% [MQTT-3.2.2-10] @@ -483,24 +493,22 @@ t_connack_max_qos_allowed(_) -> ?assertEqual(155, receive_disconnect_reasoncode()), %% [MQTT-3.2.2-11] waiting_client_process_exit(Client1), - {ok, Client2} = emqtt:start_link([ - {proto_ver, v5}, - {will_flag, true}, - {will_topic, Topic}, - {will_payload, <<"Unsupported Qos">>}, - {will_qos, 2} - ]), - {error, Connack2} = emqtt:connect(Client2), + {ok, Client2} = emqtt:start_link([ {proto_ver, v5}, + {will_flag, true}, + {will_topic, Topic}, + {will_payload, <<"Unsupported Qos">>}, + {will_qos, 2} | Config + ]), + {error, Connack2} = emqtt:ConnFun(Client2), ?assertMatch({qos_not_supported, _}, Connack2), %% [MQTT-3.2.2-12] waiting_client_process_exit(Client2), %% max_qos_allowed = 1 - emqx_zone:set_env(external, max_qos_allowed, 1), - persistent_term:erase({emqx_zone, external, '$mqtt_caps'}), - persistent_term:erase({emqx_zone, external, '$mqtt_pub_caps'}), + emqx_config:put_zone_conf(default, [mqtt, max_qos_allowed], 1), + emqx_config:put_zone_conf(default, [mqtt, max_qos_allowed], 1), - {ok, Client3} = emqtt:start_link([{proto_ver, v5}]), - {ok, Connack3} = emqtt:connect(Client3), + {ok, Client3} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, Connack3} = emqtt:ConnFun(Client3), ?assertEqual(1, maps:get('Maximum-QoS', Connack3)), %% [MQTT-3.2.2-9] {ok, _, [0]} = emqtt:subscribe(Client3, Topic, 0), %% [MQTT-3.2.2-10] @@ -511,33 +519,32 @@ t_connack_max_qos_allowed(_) -> ?assertEqual(155, receive_disconnect_reasoncode()), %% [MQTT-3.2.2-11] waiting_client_process_exit(Client3), - {ok, Client4} = emqtt:start_link([ - {proto_ver, v5}, - {will_flag, true}, - {will_topic, Topic}, - {will_payload, <<"Unsupported Qos">>}, - {will_qos, 2} - ]), - {error, Connack4} = emqtt:connect(Client4), + {ok, Client4} = emqtt:start_link([ {proto_ver, v5}, + {will_flag, true}, + {will_topic, Topic}, + {will_payload, <<"Unsupported Qos">>}, + {will_qos, 2} | Config + ]), + {error, Connack4} = emqtt:ConnFun(Client4), ?assertMatch({qos_not_supported, _}, Connack4), %% [MQTT-3.2.2-12] waiting_client_process_exit(Client4), %% max_qos_allowed = 2 - emqx_zone:set_env(external, max_qos_allowed, 2), - persistent_term:erase({emqx_zone, external, '$mqtt_caps'}), - persistent_term:erase({emqx_zone, external, '$mqtt_pub_caps'}), + emqx_config:put_zone_conf(default, [mqtt, max_qos_allowed], 2), + emqx_config:put_zone_conf(default, [mqtt, max_qos_allowed], 2), - {ok, Client5} = emqtt:start_link([{proto_ver, v5}]), - {ok, Connack5} = emqtt:connect(Client5), + {ok, Client5} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, Connack5} = emqtt:ConnFun(Client5), ?assertEqual(undefined, maps:get('Maximum-QoS', Connack5, undefined)), %% [MQTT-3.2.2-9] ok = emqtt:disconnect(Client5), waiting_client_process_exit(Client5), process_flag(trap_exit, false). -t_connack_assigned_clienid(_) -> - {ok, Client1} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client1), +t_connack_assigned_clienid(Config) -> + ConnFun = ?config(conn_fun, Config), + {ok, Client1} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client1), ?assert(is_binary(client_info(clientid, Client1))), %% [MQTT-3.2.2-16] ok = emqtt:disconnect(Client1). @@ -545,11 +552,12 @@ t_connack_assigned_clienid(_) -> %% Publish %%-------------------------------------------------------------------- -t_publish_rap(_) -> +t_publish_rap(Config) -> + ConnFun = ?config(conn_fun, Config), Topic = nth(1, ?TOPICS), - {ok, Client1} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client1), + {ok, Client1} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client1), {ok, _, [2]} = emqtt:subscribe(Client1, #{}, [{Topic, [{rap, true}, {qos, 2}]}]), {ok, _} = emqtt:publish(Client1, Topic, #{}, <<"retained message">>, [{qos, ?QOS_1}, {retain, true}]), @@ -557,8 +565,8 @@ t_publish_rap(_) -> ?assertEqual(true, maps:get(retain, Msg1)), %% [MQTT-3.3.1-12] ok = emqtt:disconnect(Client1), - {ok, Client2} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client2), + {ok, Client2} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client2), {ok, _, [2]} = emqtt:subscribe(Client2, #{}, [{Topic, [{rap, false}, {qos, 2}]}]), {ok, _} = emqtt:publish(Client2, Topic, #{}, <<"retained message">>, [{qos, ?QOS_1}, {retain, true}]), @@ -566,44 +574,47 @@ t_publish_rap(_) -> ?assertEqual(false, maps:get(retain, Msg2)), %% [MQTT-3.3.1-13] ok = emqtt:disconnect(Client2), - clean_retained(Topic). + clean_retained(Topic, Config). -t_publish_wildtopic(_) -> +t_publish_wildtopic(Config) -> + ConnFun = ?config(conn_fun, Config), process_flag(trap_exit, true), Topic = nth(1, ?WILD_TOPICS), - {ok, Client1} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client1), + {ok, Client1} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client1), ok = emqtt:publish(Client1, Topic, <<"error topic">>), ?assertEqual(144, receive_disconnect_reasoncode()), waiting_client_process_exit(Client1), process_flag(trap_exit, false). -t_publish_payload_format_indicator(_) -> +t_publish_payload_format_indicator(Config) -> + ConnFun = ?config(conn_fun, Config), Topic = nth(1, ?TOPICS), Properties = #{'Payload-Format-Indicator' => 233}, - {ok, Client1} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client1), + {ok, Client1} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client1), {ok, _, [2]} = emqtt:subscribe(Client1, Topic, qos2), ok = emqtt:publish(Client1, Topic, Properties, <<"Payload Format Indicator">>, [{qos, ?QOS_0}]), [Msg1 | _] = receive_messages(1), ?assertEqual(Properties, maps:get(properties, Msg1)), %% [MQTT-3.3.2-6] ok = emqtt:disconnect(Client1). -t_publish_topic_alias(_) -> +t_publish_topic_alias(Config) -> + ConnFun = ?config(conn_fun, Config), process_flag(trap_exit, true), Topic = nth(1, ?TOPICS), - {ok, Client1} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client1), + {ok, Client1} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client1), ok = emqtt:publish(Client1, Topic, #{'Topic-Alias' => 0}, <<"Topic-Alias">>, [{qos, ?QOS_0}]), ?assertEqual(148, receive_disconnect_reasoncode()), %% [MQTT-3.3.2-8] waiting_client_process_exit(Client1), - {ok, Client2} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client2), + {ok, Client2} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client2), {ok, _, [2]} = emqtt:subscribe(Client2, Topic, qos2), ok = emqtt:publish(Client2, Topic, #{'Topic-Alias' => 233}, <<"Topic-Alias">>, [{qos, ?QOS_0}]), @@ -615,12 +626,13 @@ t_publish_topic_alias(_) -> process_flag(trap_exit, false). -t_publish_response_topic(_) -> +t_publish_response_topic(Config) -> + ConnFun = ?config(conn_fun, Config), process_flag(trap_exit, true), Topic = nth(1, ?TOPICS), - {ok, Client1} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client1), + {ok, Client1} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client1), ok = emqtt:publish(Client1, Topic, #{'Response-Topic' => nth(1, ?WILD_TOPICS)}, <<"Response-Topic">>, [{qos, ?QOS_0}]), ?assertEqual(130, receive_disconnect_reasoncode()), %% [MQTT-3.3.2-14] @@ -628,7 +640,8 @@ t_publish_response_topic(_) -> process_flag(trap_exit, false). -t_publish_properties(_) -> +t_publish_properties(Config) -> + ConnFun = ?config(conn_fun, Config), Topic = nth(1, ?TOPICS), Properties = #{ 'Response-Topic' => Topic, %% [MQTT-3.3.2-15] @@ -637,20 +650,21 @@ t_publish_properties(_) -> 'Content-Type' => <<"2333">> %% [MQTT-3.3.2-20] }, - {ok, Client1} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client1), + {ok, Client1} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client1), {ok, _, [2]} = emqtt:subscribe(Client1, Topic, qos2), ok = emqtt:publish(Client1, Topic, Properties, <<"Publish Properties">>, [{qos, ?QOS_0}]), [Msg1 | _] = receive_messages(1), ?assertEqual(Properties, maps:get(properties, Msg1)), %% [MQTT-3.3.2-16] ok = emqtt:disconnect(Client1). -t_publish_overlapping_subscriptions(_) -> +t_publish_overlapping_subscriptions(Config) -> + ConnFun = ?config(conn_fun, Config), Topic = nth(1, ?TOPICS), Properties = #{'Subscription-Identifier' => 2333}, - {ok, Client1} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client1), + {ok, Client1} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client1), {ok, _, [1]} = emqtt:subscribe(Client1, Properties, nth(1, ?WILD_TOPICS), qos1), {ok, _, [0]} = emqtt:subscribe(Client1, Properties, nth(3, ?WILD_TOPICS), qos0), {ok, _} = emqtt:publish(Client1, Topic, #{}, @@ -665,13 +679,15 @@ t_publish_overlapping_subscriptions(_) -> %% Subsctibe %%-------------------------------------------------------------------- -t_subscribe_topic_alias(_) -> +t_subscribe_topic_alias(Config) -> + ConnFun = ?config(conn_fun, Config), Topic1 = nth(1, ?TOPICS), Topic2 = nth(2, ?TOPICS), - {ok, Client1} = emqtt:start_link([{proto_ver, v5}, - {properties, #{'Topic-Alias-Maximum' => 1}} + {ok, Client1} = emqtt:start_link([ {proto_ver, v5}, + {properties, #{'Topic-Alias-Maximum' => 1}} + | Config ]), - {ok, _} = emqtt:connect(Client1), + {ok, _} = emqtt:ConnFun(Client1), {ok, _, [2]} = emqtt:subscribe(Client1, Topic1, qos2), {ok, _, [2]} = emqtt:subscribe(Client1, Topic2, qos2), @@ -692,27 +708,29 @@ t_subscribe_topic_alias(_) -> ok = emqtt:disconnect(Client1). -t_subscribe_no_local(_) -> +t_subscribe_no_local(Config) -> + ConnFun = ?config(conn_fun, Config), Topic = nth(1, ?TOPICS), - {ok, Client1} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client1), + {ok, Client1} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client1), {ok, _, [2]} = emqtt:subscribe(Client1, #{}, [{Topic, [{nl, true}, {qos, 2}]}]), - {ok, Client2} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client2), + {ok, Client2} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client2), {ok, _, [2]} = emqtt:subscribe(Client2, #{}, [{Topic, [{nl, true}, {qos, 2}]}]), ok = emqtt:publish(Client1, Topic, <<"t_subscribe_no_local">>, 0), ?assertEqual(1, length(receive_messages(2))), %% [MQTT-3.8.3-3] ok = emqtt:disconnect(Client1). -t_subscribe_actions(_) -> +t_subscribe_actions(Config) -> + ConnFun = ?config(conn_fun, Config), Topic = nth(1, ?TOPICS), Properties = #{'Subscription-Identifier' => 2333}, - {ok, Client1} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client1), + {ok, Client1} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client1), {ok, _, [2]} = emqtt:subscribe(Client1, Properties, Topic, qos2), {ok, _, [1]} = emqtt:subscribe(Client1, Properties, Topic, qos1), {ok, _} = emqtt:publish(Client1, Topic, <<"t_subscribe_actions">>, 2), @@ -726,12 +744,13 @@ t_subscribe_actions(_) -> %% Unsubsctibe Unsuback %%-------------------------------------------------------------------- -t_unscbsctibe(_) -> +t_unscbsctibe(Config) -> + ConnFun = ?config(conn_fun, Config), Topic1 = nth(1, ?TOPICS), Topic2 = nth(2, ?TOPICS), - {ok, Client1} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client1), + {ok, Client1} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client1), {ok, _, [2]} = emqtt:subscribe(Client1, Topic1, qos2), {ok, _, [0]} = emqtt:unsubscribe(Client1, Topic1), %% [MQTT-3.10.4-4] {ok, _, [17]} = emqtt:unsubscribe(Client1, <<"noExistTopic">>), %% [MQTT-3.10.4-5] @@ -745,9 +764,10 @@ t_unscbsctibe(_) -> %% Pingreq %%-------------------------------------------------------------------- -t_pingreq(_) -> - {ok, Client1} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(Client1), +t_pingreq(Config) -> + ConnFun = ?config(conn_fun, Config), + {ok, Client1} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client1), pong = emqtt:ping(Client1), %% [MQTT-3.12.4-1] ok = emqtt:disconnect(Client1). @@ -755,9 +775,16 @@ t_pingreq(_) -> %% Shared Subscriptions %%-------------------------------------------------------------------- -t_shared_subscriptions_client_terminates_when_qos_eq_2(_) -> +t_shared_subscriptions_client_terminates_when_qos_eq_2(init, Config) -> + ok = meck:new(emqtt, [non_strict, passthrough, no_history, no_link]), + Config; +t_shared_subscriptions_client_terminates_when_qos_eq_2('end', _Config) -> + catch meck:unload(emqtt). + +t_shared_subscriptions_client_terminates_when_qos_eq_2(Config) -> + ConnFun = ?config(conn_fun, Config), process_flag(trap_exit, true), - application:set_env(emqx, shared_dispatch_ack_enabled, true), + emqx_config:put([broker, shared_dispatch_ack_enabled], true), Topic = nth(1, ?TOPICS), SharedTopic = list_to_binary("$share/sharename/" ++ binary_to_list(<<"TopicA">>)), @@ -766,32 +793,33 @@ t_shared_subscriptions_client_terminates_when_qos_eq_2(_) -> meck:expect(emqtt, connected, fun(cast, ?PUBLISH_PACKET(?QOS_2, _PacketId), _State) -> ok = counters:add(CRef, 1, 1), - {stop, {shutdown, for_testiong}}; + {stop, {shutdown, for_testing}}; (Arg1, ARg2, Arg3) -> meck:passthrough([Arg1, ARg2, Arg3]) end), - {ok, Sub1} = emqtt:start_link([{proto_ver, v5}, + {ok, Sub1} = emqtt:start_link([ {proto_ver, v5}, {clientid, <<"sub_client_1">>}, - {keepalive, 5}]), - {ok, _} = emqtt:connect(Sub1), + {keepalive, 5} | Config + ]), + {ok, _} = emqtt:ConnFun(Sub1), {ok, _, [2]} = emqtt:subscribe(Sub1, SharedTopic, qos2), {ok, Sub2} = emqtt:start_link([{proto_ver, v5}, {clientid, <<"sub_client_2">>}, - {keepalive, 5}]), - {ok, _} = emqtt:connect(Sub2), + {keepalive, 5} | Config]), + {ok, _} = emqtt:ConnFun(Sub2), {ok, _, [2]} = emqtt:subscribe(Sub2, SharedTopic, qos2), - {ok, Pub} = emqtt:start_link([{proto_ver, v5}, {clientid, <<"pub_client">>}]), - {ok, _} = emqtt:connect(Pub), + {ok, Pub} = emqtt:start_link([{proto_ver, v5}, {clientid, <<"pub_client">>} | Config]), + {ok, _} = emqtt:ConnFun(Pub), {ok, _} = emqtt:publish(Pub, Topic, <<"t_shared_subscriptions_client_terminates_when_qos_eq_2">>, 2), receive - {'EXIT', _,{shutdown, for_testiong}} -> + {'EXIT', _,{shutdown, for_testing}} -> ok after 1000 -> - error("disconnected timeout") + ct:fail("disconnected timeout") end, ?assertEqual(1, counters:get(CRef, 1)), diff --git a/apps/emqx/test/emqx_os_mon_SUITE.erl b/apps/emqx/test/emqx_os_mon_SUITE.erl index f7abd094b..674d005d3 100644 --- a/apps/emqx/test/emqx_os_mon_SUITE.erl +++ b/apps/emqx/test/emqx_os_mon_SUITE.erl @@ -24,47 +24,36 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> + emqx_config:put([sysmon, os], #{ + cpu_check_interval => 60000,cpu_high_watermark => 0.8, + cpu_low_watermark => 0.6,mem_check_interval => 60000, + procmem_high_watermark => 0.05,sysmem_high_watermark => 0.7}), application:ensure_all_started(os_mon), Config. end_per_suite(_Config) -> application:stop(os_mon). -% t_set_mem_check_interval(_) -> -% error('TODO'). - -% t_set_sysmem_high_watermark(_) -> -% error('TODO'). - -% t_set_procmem_high_watermark(_) -> -% error('TODO'). - t_api(_) -> gen_event:swap_handler(alarm_handler, {emqx_alarm_handler, swap}, {alarm_handler, []}), - {ok, _} = emqx_os_mon:start_link([{cpu_check_interval, 1}, - {cpu_high_watermark, 5}, - {cpu_low_watermark, 80}, - {mem_check_interval, 60}, - {sysmem_high_watermark, 70}, - {procmem_high_watermark, 5}]), - ?assertEqual(1, emqx_os_mon:get_cpu_check_interval()), - ?assertEqual(5, emqx_os_mon:get_cpu_high_watermark()), - ?assertEqual(80, emqx_os_mon:get_cpu_low_watermark()), - ?assertEqual(60, emqx_os_mon:get_mem_check_interval()), - ?assertEqual(70, emqx_os_mon:get_sysmem_high_watermark()), - ?assertEqual(5, emqx_os_mon:get_procmem_high_watermark()), - % timer:sleep(2000), - % ?assertEqual(true, lists:keymember(cpu_high_watermark, 1, alarm_handler:get_alarms())), + {ok, _} = emqx_os_mon:start_link(), - emqx_os_mon:set_cpu_check_interval(0.05), - emqx_os_mon:set_cpu_high_watermark(80), - emqx_os_mon:set_cpu_low_watermark(75), - ?assertEqual(0.05, emqx_os_mon:get_cpu_check_interval()), - ?assertEqual(80, emqx_os_mon:get_cpu_high_watermark()), - ?assertEqual(75, emqx_os_mon:get_cpu_low_watermark()), - % timer:sleep(3000), - % ?assertEqual(false, lists:keymember(cpu_high_watermark, 1, alarm_handler:get_alarms())), - ?assertEqual(ignored, gen_server:call(emqx_os_mon, ignored)), + ?assertEqual(60000, emqx_os_mon:get_mem_check_interval()), + ?assertEqual(ok, emqx_os_mon:set_mem_check_interval(30000)), + ?assertEqual(60000, emqx_os_mon:get_mem_check_interval()), + ?assertEqual(ok, emqx_os_mon:set_mem_check_interval(122000)), + ?assertEqual(120000, emqx_os_mon:get_mem_check_interval()), + + ?assertEqual(70, emqx_os_mon:get_sysmem_high_watermark()), + ?assertEqual(ok, emqx_os_mon:set_sysmem_high_watermark(0.8)), + ?assertEqual(80, emqx_os_mon:get_sysmem_high_watermark()), + + ?assertEqual(5, emqx_os_mon:get_procmem_high_watermark()), + ?assertEqual(ok, emqx_os_mon:set_procmem_high_watermark(0.11)), + ?assertEqual(11, emqx_os_mon:get_procmem_high_watermark()), + + ?assertEqual({error, {unexpected_call, ignored}}, + gen_server:call(emqx_os_mon, ignored)), ?assertEqual(ok, gen_server:cast(emqx_os_mon, ignored)), emqx_os_mon ! ignored, gen_server:stop(emqx_os_mon), diff --git a/apps/emqx/test/emqx_plugins_SUITE.erl b/apps/emqx/test/emqx_plugins_SUITE.erl index 6a76cb9d2..7a68ab8dd 100644 --- a/apps/emqx/test/emqx_plugins_SUITE.erl +++ b/apps/emqx/test/emqx_plugins_SUITE.erl @@ -40,17 +40,12 @@ init_per_suite(Config) -> ct:pal("Executing ~s~n", [CmdPath]), ct:pal("~n ~s~n", [os:cmd(CmdPath)]), - put(loaded_file, filename:join([DataPath, "loaded_plugins"])), emqx_ct_helpers:boot_modules([]), - emqx_ct_helpers:start_apps([], fun(_) -> set_special_cfg(DataPath) end), - + emqx_ct_helpers:start_apps([]), + emqx_config:put([plugins, expand_plugins_dir], DataPath), + ?assertEqual(ok, emqx_plugins:load()), Config. -set_special_cfg(PluginsDir) -> - application:set_env(emqx, plugins_loaded_file, get(loaded_file)), - application:set_env(emqx, expand_plugins_dir, PluginsDir), - ok. - end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([]). @@ -62,65 +57,28 @@ t_load(_) -> ?assertEqual({error, not_started}, emqx_plugins:unload(emqx_mini_plugin)), ?assertEqual({error, not_started}, emqx_plugins:unload(emqx_hocon_plugin)), - application:set_env(emqx, expand_plugins_dir, undefined), - application:set_env(emqx, plugins_loaded_file, undefined), - ?assertEqual(ignore, emqx_plugins:load()), - ?assertEqual(ignore, emqx_plugins:unload()). - - -t_init_config(_) -> - ConfFile = "emqx_mini_plugin.config", - Data = "[{emqx_mini_plugin,[{mininame ,test}]}].", - file:write_file(ConfFile, list_to_binary(Data)), - ?assertEqual(ok, emqx_plugins:init_config(ConfFile)), - file:delete(ConfFile), - ?assertEqual({ok,test}, application:get_env(emqx_mini_plugin, mininame)). + emqx_config:put([plugins, expand_plugins_dir], undefined). t_load_ext_plugin(_) -> ?assertError({plugin_app_file_not_found, _}, emqx_plugins:load_ext_plugin("./not_existed_path/")). t_list(_) -> - ?assertMatch([{plugin, _, _, _, _, _, _, _} | _ ], emqx_plugins:list()). + ?assertMatch([{plugin, _, _, _, _, _, _} | _ ], emqx_plugins:list()). t_find_plugin(_) -> - ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _, _}, emqx_plugins:find_plugin(emqx_mini_plugin)), - ?assertMatch({plugin, emqx_hocon_plugin, _, _, _, _, _, _}, emqx_plugins:find_plugin(emqx_hocon_plugin)). - -t_plugin_type(_) -> - ?assertEqual(auth, emqx_plugins:plugin_type(auth)), - ?assertEqual(protocol, emqx_plugins:plugin_type(protocol)), - ?assertEqual(backend, emqx_plugins:plugin_type(backend)), - ?assertEqual(bridge, emqx_plugins:plugin_type(bridge)), - ?assertEqual(feature, emqx_plugins:plugin_type(undefined)). - -t_with_loaded_file(_) -> - ?assertMatch({error, _}, emqx_plugins:with_loaded_file("./not_existed_path/", fun(_) -> ok end)). - -t_plugin_loaded(_) -> - ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_mini_plugin, false)), - ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_mini_plugin, true)), - ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_hocon_plugin, false)), - ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_hocon_plugin, true)). - -t_plugin_unloaded(_) -> - ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin, false)), - ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin, true)), - ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_hocon_plugin, false)), - ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_hocon_plugin, true)). + ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _}, emqx_plugins:find_plugin(emqx_mini_plugin)), + ?assertMatch({plugin, emqx_hocon_plugin, _, _, _, _, _}, emqx_plugins:find_plugin(emqx_hocon_plugin)). t_plugin(_) -> try - emqx_plugins:plugin(not_existed_plugin, undefined) + emqx_plugins:plugin(not_existed_plugin) catch _Error:Reason:_Stacktrace -> ?assertEqual({plugin_not_found,not_existed_plugin}, Reason) end, - ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _, _}, emqx_plugins:plugin(emqx_mini_plugin, undefined)), - ?assertMatch({plugin, emqx_hocon_plugin, _, _, _, _, _, _}, emqx_plugins:plugin(emqx_hocon_plugin, undefined)). - -t_filter_plugins(_) -> - ?assertEqual([name1, name2], emqx_plugins:filter_plugins([name1, {name2,true}, {name3, false}])). + ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _}, emqx_plugins:plugin(emqx_mini_plugin)), + ?assertMatch({plugin, emqx_hocon_plugin, _, _, _, _, _}, emqx_plugins:plugin(emqx_hocon_plugin)). t_load_plugin(_) -> ok = meck:new(application, [unstick, non_strict, passthrough, no_history]), @@ -133,9 +91,9 @@ t_load_plugin(_) -> ok = meck:new(emqx_plugins, [unstick, non_strict, passthrough, no_history]), ok = meck:expect(emqx_plugins, generate_configs, fun(_) -> ok end), ok = meck:expect(emqx_plugins, apply_configs, fun(_) -> ok end), - ?assertMatch({error, _}, emqx_plugins:load_plugin(already_loaded_app, true)), - ?assertMatch(ok, emqx_plugins:load_plugin(normal, true)), - ?assertMatch({error,_}, emqx_plugins:load_plugin(error_app, true)), + ?assertMatch({error, _}, emqx_plugins:load_plugin(already_loaded_app)), + ?assertMatch(ok, emqx_plugins:load_plugin(normal)), + ?assertMatch({error,_}, emqx_plugins:load_plugin(error_app)), ok = meck:unload(emqx_plugins), ok = meck:unload(application). @@ -146,8 +104,8 @@ t_unload_plugin(_) -> (error_app) -> {error, error}; (_) -> ok end), - ?assertEqual(ok, emqx_plugins:unload_plugin(not_started_app, true)), - ?assertEqual(ok, emqx_plugins:unload_plugin(normal, true)), - ?assertEqual({error,error}, emqx_plugins:unload_plugin(error_app, true)), + ?assertEqual(ok, emqx_plugins:unload_plugin(not_started_app)), + ?assertEqual(ok, emqx_plugins:unload_plugin(normal)), + ?assertEqual({error,error}, emqx_plugins:unload_plugin(error_app)), ok = meck:unload(application). diff --git a/apps/emqx/test/emqx_session_SUITE.erl b/apps/emqx/test/emqx_session_SUITE.erl index cb7c10cae..1aa5b0196 100644 --- a/apps/emqx/test/emqx_session_SUITE.erl +++ b/apps/emqx/test/emqx_session_SUITE.erl @@ -29,6 +29,7 @@ all() -> emqx_ct:all(?MODULE). %%-------------------------------------------------------------------- init_per_suite(Config) -> + emqx_channel_SUITE:set_default_zone_conf(), ok = meck:new([emqx_hooks, emqx_metrics, emqx_broker], [passthrough, no_history, no_link]), ok = meck:expect(emqx_metrics, inc, fun(_) -> ok end), @@ -50,19 +51,19 @@ end_per_testcase(_TestCase, Config) -> %%-------------------------------------------------------------------- t_session_init(_) -> - Session = emqx_session:init(#{zone => zone}, #{receive_maximum => 64}), + Session = emqx_session:init(#{max_inflight => 64}), ?assertEqual(#{}, emqx_session:info(subscriptions, Session)), ?assertEqual(0, emqx_session:info(subscriptions_cnt, Session)), - ?assertEqual(0, emqx_session:info(subscriptions_max, Session)), + ?assertEqual(infinity, emqx_session:info(subscriptions_max, Session)), ?assertEqual(false, emqx_session:info(upgrade_qos, Session)), ?assertEqual(0, emqx_session:info(inflight_cnt, Session)), ?assertEqual(64, emqx_session:info(inflight_max, Session)), ?assertEqual(1, emqx_session:info(next_pkt_id, Session)), - ?assertEqual(0, emqx_session:info(retry_interval, Session)), + ?assertEqual(30000, emqx_session:info(retry_interval, Session)), ?assertEqual(0, emqx_mqueue:len(emqx_session:info(mqueue, Session))), ?assertEqual(0, emqx_session:info(awaiting_rel_cnt, Session)), ?assertEqual(100, emqx_session:info(awaiting_rel_max, Session)), - ?assertEqual(300, emqx_session:info(await_rel_timeout, Session)), + ?assertEqual(300000, emqx_session:info(await_rel_timeout, Session)), ?assert(is_integer(emqx_session:info(created_at, Session))). %%-------------------------------------------------------------------- @@ -72,13 +73,13 @@ t_session_init(_) -> t_session_info(_) -> ?assertMatch(#{subscriptions := #{}, upgrade_qos := false, - retry_interval := 0, - await_rel_timeout := 300 + retry_interval := 30000, + await_rel_timeout := 300000 }, emqx_session:info(session())). t_session_stats(_) -> Stats = emqx_session:stats(session()), - ?assertMatch(#{subscriptions_max := 0, + ?assertMatch(#{subscriptions_max := infinity, inflight_max := 0, mqueue_len := 0, mqueue_max := 1000, @@ -99,7 +100,7 @@ t_subscribe(_) -> ?assertEqual(1, emqx_session:info(subscriptions_cnt, Session)). t_is_subscriptions_full_false(_) -> - Session = session(#{max_subscriptions => 0}), + Session = session(#{max_subscriptions => infinity}), ?assertNot(emqx_session:is_subscriptions_full(Session)). t_is_subscriptions_full_true(_) -> @@ -152,7 +153,7 @@ t_publish_qos2_with_error_return(_) -> {error, ?RC_RECEIVE_MAXIMUM_EXCEEDED} = emqx_session:publish(3, Msg, Session1). t_is_awaiting_full_false(_) -> - Session = session(#{max_awaiting_rel => 0}), + Session = session(#{max_awaiting_rel => infinity}), ?assertNot(emqx_session:is_awaiting_full(Session)). t_is_awaiting_full_true(_) -> @@ -308,9 +309,11 @@ t_enqueue(_) -> t_retry(_) -> Delivers = [delivery(?QOS_1, <<"t1">>), delivery(?QOS_2, <<"t2">>)], - Session = session(#{retry_interval => 100}), + RetryIntervalMs = 100, %% 0.1s + Session = session(#{retry_interval => RetryIntervalMs}), {ok, Pubs, Session1} = emqx_session:deliver(Delivers, Session), - ok = timer:sleep(200), + ElapseMs = 200, %% 0.2s + ok = timer:sleep(ElapseMs), Msgs1 = [{I, emqx_message:set_flag(dup, Msg)} || {I, Msg} <- Pubs], {ok, Msgs1, 100, Session2} = emqx_session:retry(Session1), ?assertEqual(2, emqx_session:info(inflight_cnt, Session2)). @@ -341,7 +344,7 @@ t_replay(_) -> t_expire_awaiting_rel(_) -> {ok, Session} = emqx_session:expire(awaiting_rel, session()), - Timeout = emqx_session:info(await_rel_timeout, Session) * 1000, + Timeout = emqx_session:info(await_rel_timeout, Session), Session1 = emqx_session:set_field(awaiting_rel, #{1 => Ts = ts(millisecond)}, Session), {ok, Timeout, Session2} = emqx_session:expire(awaiting_rel, Session1), ?assertEqual(#{1 => Ts}, emqx_session:info(awaiting_rel, Session2)). @@ -375,7 +378,7 @@ session(InitFields) when is_map(InitFields) -> maps:fold(fun(Field, Value, Session) -> emqx_session:set_field(Field, Value, Session) end, - emqx_session:init(#{zone => channel}, #{receive_maximum => 0}), + emqx_session:init(#{max_inflight => 0}), InitFields). diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 54a0de6d2..e3caf44a7 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -322,8 +322,8 @@ ensure_config(Strategy) -> ensure_config(Strategy, _AckEnabled = true). ensure_config(Strategy, AckEnabled) -> - application:set_env(emqx, shared_subscription_strategy, Strategy), - application:set_env(emqx, shared_dispatch_ack_enabled, AckEnabled), + emqx_config:put([broker, shared_subscription_strategy], Strategy), + emqx_config:put([broker, shared_dispatch_ack_enabled], AckEnabled), ok. subscribed(Group, Topic, Pid) -> diff --git a/apps/emqx/test/emqx_sys_SUITE.erl b/apps/emqx/test/emqx_sys_SUITE.erl index 29583524f..65f09caf6 100644 --- a/apps/emqx/test/emqx_sys_SUITE.erl +++ b/apps/emqx/test/emqx_sys_SUITE.erl @@ -25,8 +25,6 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> application:load(emqx), - ok = application:set_env(emqx, broker_sys_interval, 1), - ok = application:set_env(emqx, broker_sys_heartbeat, 1), ok = emqx_logger:set_log_level(emergency), Config. diff --git a/apps/emqx/test/emqx_trie_SUITE.erl b/apps/emqx/test/emqx_trie_SUITE.erl index 7ae23b4c6..51106f529 100644 --- a/apps/emqx/test/emqx_trie_SUITE.erl +++ b/apps/emqx/test/emqx_trie_SUITE.erl @@ -34,14 +34,14 @@ groups() -> [{compact, Cases}, {not_compact, Cases}]. init_per_group(compact, Config) -> - emqx_trie:put_compaction_flag(true), + emqx_trie:set_compact(true), Config; init_per_group(not_compact, Config) -> - emqx_trie:put_compaction_flag(false), + emqx_trie:set_compact(false), Config. end_per_group(_, _) -> - emqx_trie:put_default_compaction_flag(). + ok. init_per_suite(Config) -> application:load(emqx), diff --git a/apps/emqx/test/emqx_vm_mon_SUITE.erl b/apps/emqx/test/emqx_vm_mon_SUITE.erl index 5f9f4084c..5b39746a1 100644 --- a/apps/emqx/test/emqx_vm_mon_SUITE.erl +++ b/apps/emqx/test/emqx_vm_mon_SUITE.erl @@ -23,17 +23,16 @@ all() -> emqx_ct:all(?MODULE). -init_per_testcase(t_api, Config) -> +init_per_testcase(t_alarms, Config) -> emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([], - fun(emqx) -> - application:set_env(emqx, vm_mon, [{check_interval, 1}, - {process_high_watermark, 80}, - {process_low_watermark, 75}]), - ok; - (_) -> - ok - end), + emqx_ct_helpers:start_apps([]), + emqx_config:put([sysmon, vm], #{ + process_high_watermark => 0, + process_low_watermark => 0, + process_check_interval => 100 %% 1s + }), + ok = supervisor:terminate_child(emqx_sys_sup, emqx_vm_mon), + {ok, _} = supervisor:restart_child(emqx_sys_sup, emqx_vm_mon), Config; init_per_testcase(_, Config) -> emqx_ct_helpers:boot_modules(all), @@ -43,18 +42,12 @@ init_per_testcase(_, Config) -> end_per_testcase(_, _Config) -> emqx_ct_helpers:stop_apps([]). -t_api(_) -> - ?assertEqual(1, emqx_vm_mon:get_check_interval()), - ?assertEqual(80, emqx_vm_mon:get_process_high_watermark()), - ?assertEqual(75, emqx_vm_mon:get_process_low_watermark()), - emqx_vm_mon:set_process_high_watermark(0), - emqx_vm_mon:set_process_low_watermark(60), - ?assertEqual(0, emqx_vm_mon:get_process_high_watermark()), - ?assertEqual(60, emqx_vm_mon:get_process_low_watermark()), - timer:sleep(emqx_vm_mon:get_check_interval() * 1000 * 2), +t_alarms(_) -> + timer:sleep(500), ?assert(is_existing(too_many_processes, emqx_alarm:get_alarms(activated))), - emqx_vm_mon:set_process_high_watermark(70), - timer:sleep(emqx_vm_mon:get_check_interval() * 1000 * 2), + emqx_config:put([sysmon, vm, process_high_watermark], 70), + emqx_config:put([sysmon, vm, process_low_watermark], 60), + timer:sleep(500), ?assertNot(is_existing(too_many_processes, emqx_alarm:get_alarms(activated))). is_existing(Name, [#{name := Name} | _More]) -> diff --git a/apps/emqx/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl index 6db831972..b25f051eb 100644 --- a/apps/emqx/test/emqx_ws_connection_SUITE.erl +++ b/apps/emqx/test/emqx_ws_connection_SUITE.erl @@ -48,6 +48,7 @@ init_per_testcase(TestCase, Config) when TestCase =/= t_ws_pingreq_before_connected, TestCase =/= t_ws_non_check_origin -> + emqx_channel_SUITE:set_default_zone_conf(), %% Mock cowboy_req ok = meck:new(cowboy_req, [passthrough, no_history, no_link]), ok = meck:expect(cowboy_req, header, fun(_, _, _) -> <<>> end), @@ -55,16 +56,9 @@ init_per_testcase(TestCase, Config) when ok = meck:expect(cowboy_req, sock, fun(_) -> {{127,0,0,1}, 18083} end), ok = meck:expect(cowboy_req, cert, fun(_) -> undefined end), ok = meck:expect(cowboy_req, parse_cookies, fun(_) -> error(badarg) end), - %% Mock emqx_zone - ok = meck:new(emqx_zone, [passthrough, no_history, no_link]), - ok = meck:expect(emqx_zone, oom_policy, - fun(_) -> #{max_heap_size => 838860800, - message_queue_len => 8000 - } - end), %% Mock emqx_access_control ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), - ok = meck:expect(emqx_access_control, check_acl, fun(_, _, _) -> allow end), + ok = meck:expect(emqx_access_control, authorize, fun(_, _, _) -> allow end), %% Mock emqx_hooks ok = meck:new(emqx_hooks, [passthrough, no_history, no_link]), ok = meck:expect(emqx_hooks, run, fun(_Hook, _Args) -> ok end), @@ -85,6 +79,7 @@ init_per_testcase(TestCase, Config) when Config; init_per_testcase(_, Config) -> + ok = emqx_ct_helpers:start_apps([]), Config. end_per_testcase(TestCase, _Config) when @@ -96,7 +91,6 @@ end_per_testcase(TestCase, _Config) when -> lists:foreach(fun meck:unload/1, [cowboy_req, - emqx_zone, emqx_access_control, emqx_broker, emqx_hooks, @@ -104,6 +98,7 @@ end_per_testcase(TestCase, _Config) when ]); end_per_testcase(_, Config) -> + emqx_ct_helpers:stop_apps([]), Config. %%-------------------------------------------------------------------- @@ -118,18 +113,21 @@ t_info(_) -> end), #{sockinfo := SockInfo} = ?ws_conn:call(WsPid, info), #{socktype := ws, - active_n := 100, peername := {{127,0,0,1}, 3456}, sockname := {{127,0,0,1}, 18083}, sockstate := running } = SockInfo. +set_ws_opts(Key, Val) -> + emqx_config:put_listener_conf(default, mqtt_ws, [websocket, Key], Val). + t_header(_) -> - ok = meck:expect(cowboy_req, header, fun(<<"x-forwarded-for">>, _, _) -> <<"100.100.100.100, 99.99.99.99">>; - (<<"x-forwarded-port">>, _, _) -> <<"1000">> end), - {ok, St, _} = ?ws_conn:websocket_init([req, [{zone, external}, - {proxy_address_header, <<"x-forwarded-for">>}, - {proxy_port_header, <<"x-forwarded-port">>}]]), + ok = meck:expect(cowboy_req, header, + fun(<<"x-forwarded-for">>, _, _) -> <<"100.100.100.100, 99.99.99.99">>; + (<<"x-forwarded-port">>, _, _) -> <<"1000">> end), + set_ws_opts(proxy_address_header, <<"x-forwarded-for">>), + set_ws_opts(proxy_port_header, <<"x-forwarded-port">>), + {ok, St, _} = ?ws_conn:websocket_init([req, #{zone => default, listener => mqtt_ws}]), WsPid = spawn(fun() -> receive {call, From, info} -> gen_server:reply(From, ?ws_conn:info(St)) @@ -175,12 +173,10 @@ t_call(_) -> ?assertEqual(Info, ?ws_conn:call(WsPid, info)). t_ws_pingreq_before_connected(_) -> - ok = emqx_ct_helpers:start_apps([]), {ok, _} = application:ensure_all_started(gun), {ok, WPID} = gun:open("127.0.0.1", 8083), ws_pingreq(#{}), - gun:close(WPID), - emqx_ct_helpers:stop_apps([]). + gun:close(WPID). ws_pingreq(State) -> receive @@ -209,14 +205,11 @@ ws_pingreq(State) -> end. t_ws_sub_protocols_mqtt(_) -> - ok = emqx_ct_helpers:start_apps([]), {ok, _} = application:ensure_all_started(gun), ?assertMatch({gun_upgrade, _}, - start_ws_client(#{protocols => [<<"mqtt">>]})), - emqx_ct_helpers:stop_apps([]). + start_ws_client(#{protocols => [<<"mqtt">>]})). t_ws_sub_protocols_mqtt_equivalents(_) -> - ok = emqx_ct_helpers:start_apps([]), {ok, _} = application:ensure_all_started(gun), %% also support mqtt-v3, mqtt-v3.1.1, mqtt-v5 ?assertMatch({gun_upgrade, _}, @@ -226,58 +219,39 @@ t_ws_sub_protocols_mqtt_equivalents(_) -> ?assertMatch({gun_upgrade, _}, start_ws_client(#{protocols => [<<"mqtt-v5">>]})), ?assertMatch({gun_response, {_, 400, _}}, - start_ws_client(#{protocols => [<<"not-mqtt">>]})), - emqx_ct_helpers:stop_apps([]). + start_ws_client(#{protocols => [<<"not-mqtt">>]})). t_ws_check_origin(_) -> - emqx_ct_helpers:start_apps([], - fun(emqx) -> - {ok, Listeners} = application:get_env(emqx, listeners), - NListeners = lists:map(fun(#{listen_on := 8083, opts := Opts} = Listener) -> - NOpts = proplists:delete(check_origin_enable, Opts), - Listener#{opts => [{check_origin_enable, true} | NOpts]}; - (Listener) -> - Listener - end, Listeners), - application:set_env(emqx, listeners, NListeners), - ok; - (_) -> ok - end), + emqx_config:put_listener_conf(default, mqtt_ws, [websocket, check_origin_enable], true), + emqx_config:put_listener_conf(default, mqtt_ws, [websocket, check_origins], + [<<"http://localhost:18083">>]), {ok, _} = application:ensure_all_started(gun), ?assertMatch({gun_upgrade, _}, start_ws_client(#{protocols => [<<"mqtt">>], headers => [{<<"origin">>, <<"http://localhost:18083">>}]})), ?assertMatch({gun_response, {_, 500, _}}, start_ws_client(#{protocols => [<<"mqtt">>], - headers => [{<<"origin">>, <<"http://localhost:18080">>}]})), - emqx_ct_helpers:stop_apps([]). + headers => [{<<"origin">>, <<"http://localhost:18080">>}]})). t_ws_non_check_origin(_) -> - emqx_ct_helpers:start_apps([]), + emqx_config:put_listener_conf(default, mqtt_ws, [websocket, check_origin_enable], false), + emqx_config:put_listener_conf(default, mqtt_ws, [websocket, check_origins], []), {ok, _} = application:ensure_all_started(gun), ?assertMatch({gun_upgrade, _}, start_ws_client(#{protocols => [<<"mqtt">>], headers => [{<<"origin">>, <<"http://localhost:18083">>}]})), ?assertMatch({gun_upgrade, _}, start_ws_client(#{protocols => [<<"mqtt">>], - headers => [{<<"origin">>, <<"http://localhost:18080">>}]})), - emqx_ct_helpers:stop_apps([]). - + headers => [{<<"origin">>, <<"http://localhost:18080">>}]})). t_init(_) -> - Opts = [{idle_timeout, 300000}, - {fail_if_no_subprotocol, false}, - {supported_subprotocols, ["mqtt"]}], - WsOpts = #{compress => false, - deflate_opts => #{}, - max_frame_size => infinity, - idle_timeout => 300000 - }, + Opts = #{listener => mqtt_ws, zone => default}, ok = meck:expect(cowboy_req, parse_header, fun(_, req) -> undefined end), - {cowboy_websocket, req, [req, Opts], WsOpts} = ?ws_conn:init(req, Opts), + ok = meck:expect(cowboy_req, reply, fun(_, Req) -> Req end), + {ok, req, _} = ?ws_conn:init(req, Opts), ok = meck:expect(cowboy_req, parse_header, fun(_, req) -> [<<"mqtt">>] end), ok = meck:expect(cowboy_req, set_resp_header, fun(_, <<"mqtt">>, req) -> resp end), - {cowboy_websocket, resp, [req, Opts], WsOpts} = ?ws_conn:init(req, Opts). + {cowboy_websocket, resp, [req, Opts], _} = ?ws_conn:init(req, Opts). t_websocket_handle_binary(_) -> {ok, _} = websocket_handle({binary, <<>>}, st()), @@ -450,15 +424,6 @@ t_run_gc(_) -> WsSt = st(#{gc_state => GcSt}), ?ws_conn:run_gc(#{cnt => 100, oct => 10000}, WsSt). -t_check_oom(_) -> - %%Policy = #{max_heap_size => 10, message_queue_len => 10}, - %%meck:expect(emqx_zone, oom_policy, fun(_) -> Policy end), - _St = ?ws_conn:check_oom(st()), - ok = timer:sleep(10). - %%receive {shutdown, proc_heap_too_large} -> ok - %%after 0 -> error(expect_shutdown) - %%end. - t_enqueue(_) -> Packet = ?PUBLISH_PACKET(?QOS_0), St = ?ws_conn:enqueue(Packet, st()), @@ -473,7 +438,7 @@ t_shutdown(_) -> st() -> st(#{}). st(InitFields) when is_map(InitFields) -> - {ok, St, _} = ?ws_conn:websocket_init([req, [{zone, external}]]), + {ok, St, _} = ?ws_conn:websocket_init([req, #{zone => default, listener => mqtt_ws}]), maps:fold(fun(N, V, S) -> ?ws_conn:set_field(N, V, S) end, ?ws_conn:set_field(channel, channel(), St), InitFields @@ -493,7 +458,8 @@ channel(InitFields) -> receive_maximum => 100, expiry_interval => 0 }, - ClientInfo = #{zone => zone, + ClientInfo = #{zone => default, + listener => mqtt_ws, protocol => mqtt, peerhost => {127,0,0,1}, clientid => <<"clientid">>, @@ -502,13 +468,11 @@ channel(InitFields) -> peercert => undefined, mountpoint => undefined }, - Session = emqx_session:init(#{zone => external}, - #{receive_maximum => 0} - ), + Session = emqx_session:init(#{max_inflight => 0}), maps:fold(fun(Field, Value, Channel) -> emqx_channel:set_field(Field, Value, Channel) end, - emqx_channel:init(ConnInfo, [{zone, zone}]), + emqx_channel:init(ConnInfo, #{zone => default, listener => mqtt_ws}), maps:merge(#{clientinfo => ClientInfo, session => Session, conn_state => connected diff --git a/apps/emqx/test/emqx_zone_SUITE.erl b/apps/emqx/test/emqx_zone_SUITE.erl deleted file mode 100644 index 8294ac0da..000000000 --- a/apps/emqx/test/emqx_zone_SUITE.erl +++ /dev/null @@ -1,104 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_zone_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). - --define(ENVS, [{use_username_as_clientid, false}, - {server_keepalive, 60}, - {upgrade_qos, false}, - {session_expiry_interval, 7200}, - {retry_interval, 20}, - {mqueue_store_qos0, true}, - {mqueue_priorities, none}, - {mqueue_default_priority, highest}, - {max_subscriptions, 0}, - {max_mqueue_len, 1000}, - {max_inflight, 32}, - {max_awaiting_rel, 100}, - {keepalive_backoff, 0.75}, - {ignore_loop_deliver, false}, - {idle_timeout, 15000}, - {force_shutdown_policy, #{max_heap_size => 838860800, - message_queue_len => 8000}}, - {force_gc_policy, #{bytes => 1048576, count => 1000}}, - {enable_stats, true}, - {enable_flapping_detect, false}, - {enable_ban, true}, - {enable_acl, true}, - {await_rel_timeout, 300}, - {acl_deny_action, ignore} - ]). - -all() -> emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - _ = application:load(emqx), - application:set_env(emqx, zone_env, val), - application:set_env(emqx, zones, [{zone, ?ENVS}]), - Config. - -end_per_suite(_Config) -> - emqx_zone:unset_all_env(), - application:unset_env(emqx, zone_env), - application:unset_env(emqx, zones). - -t_zone_env_func(_) -> - lists:foreach(fun({Env, Val}) -> - case erlang:function_exported(emqx_zone, Env, 1) of - true -> - ?assertEqual(Val, erlang:apply(emqx_zone, Env, [zone])); - false -> ok - end - end, ?ENVS). - -t_get_env(_) -> - ?assertEqual(val, emqx_zone:get_env(undefined, zone_env)), - ?assertEqual(val, emqx_zone:get_env(undefined, zone_env, def)), - ?assert(emqx_zone:get_env(zone, enable_acl)), - ?assert(emqx_zone:get_env(zone, enable_ban)), - ?assertEqual(defval, emqx_zone:get_env(extenal, key, defval)), - ?assertEqual(undefined, emqx_zone:get_env(external, key)), - ?assertEqual(undefined, emqx_zone:get_env(internal, key)), - ?assertEqual(def, emqx_zone:get_env(internal, key, def)). - -t_get_set_env(_) -> - ok = emqx_zone:set_env(zone, key, val), - ?assertEqual(val, emqx_zone:get_env(zone, key)), - true = emqx_zone:unset_env(zone, key), - ?assertEqual(undefined, emqx_zone:get_env(zone, key)). - -t_force_reload(_) -> - {ok, _} = emqx_zone:start_link(), - ?assertEqual(undefined, emqx_zone:get_env(xzone, key)), - application:set_env(emqx, zones, [{xzone, [{key, val}]}]), - ok = emqx_zone:force_reload(), - ?assertEqual(val, emqx_zone:get_env(xzone, key)), - emqx_zone:stop(). - -t_uncovered_func(_) -> - {ok, Pid} = emqx_zone:start_link(), - ignored = gen_server:call(Pid, unexpected_call), - ok = gen_server:cast(Pid, unexpected_cast), - ok = Pid ! ok, - emqx_zone:stop(). - -t_frame_options(_) -> - ?assertMatch(#{strict_mode := _, max_size := _ }, emqx_zone:mqtt_frame_options(zone)). diff --git a/apps/emqx/test/props/prop_emqx_sys.erl b/apps/emqx/test/props/prop_emqx_sys.erl index 67718ec37..170611061 100644 --- a/apps/emqx/test/props/prop_emqx_sys.erl +++ b/apps/emqx/test/props/prop_emqx_sys.erl @@ -59,6 +59,8 @@ prop_sys() -> do_setup() -> ok = emqx_logger:set_log_level(emergency), + emqx_config:put([broker, sys_msg_interval], 60000), + emqx_config:put([broker, sys_heartbeat_interval], 30000), [mock(Mod) || Mod <- ?mock_modules], ok. @@ -98,8 +100,6 @@ command(_State) -> {call, emqx_sys, uptime, []}, {call, emqx_sys, datetime, []}, {call, emqx_sys, sysdescr, []}, - {call, emqx_sys, sys_interval, []}, - {call, emqx_sys, sys_heatbeat_interval, []}, %------------ unexpected message ----------------------% {call, emqx_sys, handle_call, [emqx_sys, other, state]}, {call, emqx_sys, handle_cast, [emqx_sys, other]}, diff --git a/apps/emqx_authentication/etc/emqx_authentication.conf b/apps/emqx_authentication/etc/emqx_authentication.conf deleted file mode 100644 index e69de29bb..000000000 diff --git a/apps/emqx_authentication/priv/emqx_authentication.schema b/apps/emqx_authentication/priv/emqx_authentication.schema deleted file mode 100644 index e69de29bb..000000000 diff --git a/apps/emqx_authentication/src/emqx_authentication.erl b/apps/emqx_authentication/src/emqx_authentication.erl deleted file mode 100644 index ab7b8537c..000000000 --- a/apps/emqx_authentication/src/emqx_authentication.erl +++ /dev/null @@ -1,519 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_authentication). - --include("emqx_authentication.hrl"). - --export([ enable/0 - , disable/0 - ]). - --export([authenticate/1]). - --export([register_service_types/0]). - --export([ create_chain/1 - , delete_chain/1 - , lookup_chain/1 - , list_chains/0 - , add_services/2 - , delete_services/2 - , update_service/3 - , lookup_service/2 - , list_services/1 - , move_service_to_the_front/2 - , move_service_to_the_end/2 - , move_service_to_the_nth/3 - ]). - --export([ import_users/3 - , add_user/3 - , delete_user/3 - , update_user/4 - , lookup_user/3 - , list_users/2 - ]). - --export([mnesia/1]). - --boot_mnesia({mnesia, [boot]}). --copy_mnesia({mnesia, [copy]}). - --define(CHAIN_TAB, emqx_authentication_chain). --define(SERVICE_TYPE_TAB, emqx_authentication_service_type). - -%%------------------------------------------------------------------------------ -%% Mnesia bootstrap -%%------------------------------------------------------------------------------ - -%% @doc Create or replicate tables. --spec(mnesia(boot | copy) -> ok). -mnesia(boot) -> - %% Optimize storage - StoreProps = [{ets, [{read_concurrency, true}]}], - %% Chain table - ok = ekka_mnesia:create_table(?CHAIN_TAB, [ - {disc_copies, [node()]}, - {record_name, chain}, - {attributes, record_info(fields, chain)}, - {storage_properties, StoreProps}]), - %% Service type table - ok = ekka_mnesia:create_table(?SERVICE_TYPE_TAB, [ - {ram_copies, [node()]}, - {record_name, service_type}, - {attributes, record_info(fields, service_type)}, - {storage_properties, StoreProps}]); - -mnesia(copy) -> - %% Copy chain table - ok = ekka_mnesia:copy_table(?CHAIN_TAB, disc_copies), - %% Copy service type table - ok = ekka_mnesia:copy_table(?SERVICE_TYPE_TAB, ram_copies). - -enable() -> - case emqx:hook('client.authenticate', {emqx_authentication, authenticate, []}) of - ok -> ok; - {error, already_exists} -> ok - end. - -disable() -> - emqx:unhook('client.authenticate', {emqx_authentication, authenticate}), - ok. - -authenticate(#{chain_id := ChainID} = ClientInfo) -> - case mnesia:dirty_read(?CHAIN_TAB, ChainID) of - [#chain{services = []}] -> - {error, no_services}; - [#chain{services = Services}] -> - do_authenticate(Services, ClientInfo); - [] -> - {error, todo} - end. - -do_authenticate([], _) -> - {error, user_not_found}; -do_authenticate([{_, #service{provider = Provider, state = State}} | More], ClientInfo) -> - case Provider:authenticate(ClientInfo, State) of - ignore -> do_authenticate(More, ClientInfo); - ok -> ok; - {ok, NewClientInfo} -> {ok, NewClientInfo}; - {stop, Reason} -> {error, Reason} - end. - -register_service_types() -> - Attrs = find_attrs(?APP, service_type), - register_service_types(Attrs). - -register_service_types(Attrs) -> - register_service_types(Attrs, []). - -register_service_types([], Acc) -> - do_register_service_types(Acc); -register_service_types([{_App, Mod, #{name := Name, - params_spec := ParamsSpec}} | Types], Acc) -> - %% TODO: Temporary realization - ok = emqx_rule_validator:validate_spec(ParamsSpec), - ServiceType = #service_type{name = Name, - provider = Mod, - params_spec = ParamsSpec}, - register_service_types(Types, [ServiceType | Acc]). - -create_chain(#{id := ID}) -> - trans( - fun() -> - case mnesia:read(?CHAIN_TAB, ID, write) of - [] -> - Chain = #chain{id = ID, - services = [], - created_at = erlang:system_time(millisecond)}, - mnesia:write(?CHAIN_TAB, Chain, write), - {ok, serialize_chain(Chain)}; - [_ | _] -> - {error, {already_exists, {chain, ID}}} - end - end). - -delete_chain(ID) -> - trans( - fun() -> - case mnesia:read(?CHAIN_TAB, ID, write) of - [] -> - {error, {not_found, {chain, ID}}}; - [#chain{services = Services}] -> - ok = delete_services_(Services), - mnesia:delete(?CHAIN_TAB, ID, write) - end - end). - -lookup_chain(ID) -> - case mnesia:dirty_read(?CHAIN_TAB, ID) of - [] -> - {error, {not_found, {chain, ID}}}; - [Chain] -> - {ok, serialize_chain(Chain)} - end. - -list_chains() -> - Chains = ets:tab2list(?CHAIN_TAB), - {ok, [serialize_chain(Chain) || Chain <- Chains]}. - -add_services(ChainID, ServiceParams) -> - case validate_service_params(ServiceParams) of - {ok, NServiceParams} -> - UpdateFun = fun(Chain = #chain{services = Services}) -> - Names = [Name || {Name, _} <- Services] ++ [Name || #{name := Name} <- NServiceParams], - case no_duplicate_names(Names) of - ok -> - case create_services(ChainID, NServiceParams) of - {ok, NServices} -> - NChain = Chain#chain{services = Services ++ NServices}, - ok = mnesia:write(?CHAIN_TAB, NChain, write), - {ok, serialize_services(NServices)}; - {error, Reason} -> - {error, Reason} - end; - {error, {duplicate, Name}} -> - {error, {already_exists, {service, Name}}} - end - end, - update_chain(ChainID, UpdateFun); - {error, Reason} -> - {error, Reason} - end. - -delete_services(ChainID, ServiceNames) -> - case no_duplicate_names(ServiceNames) of - ok -> - UpdateFun = fun(Chain = #chain{services = Services}) -> - case extract_services(ServiceNames, Services) of - {ok, Extracted, Rest} -> - ok = delete_services_(Extracted), - NChain = Chain#chain{services = Rest}, - mnesia:write(?CHAIN_TAB, NChain, write); - {error, Reason} -> - {error, Reason} - end - end, - update_chain(ChainID, UpdateFun); - {error, Reason} -> - {error, Reason} - end. - -update_service(ChainID, ServiceName, NewParams) -> - UpdateFun = fun(Chain = #chain{services = Services}) -> - case proplists:get_value(ServiceName, Services, undefined) of - undefined -> - {error, {not_found, {service, ServiceName}}}; - #service{type = Type, - provider = Provider, - params = OriginalParams, - state = State} = Service -> - Params = maps:merge(OriginalParams, NewParams), - {ok, #service_type{params_spec = ParamsSpec}} = find_service_type(Type), - NParams = emqx_rule_validator:validate_params(Params, ParamsSpec), - case Provider:update(ChainID, ServiceName, NParams, State) of - {ok, NState} -> - NService = Service#service{params = Params, - state = NState}, - NServices = lists:keyreplace(ServiceName, 1, Services, {ServiceName, NService}), - ok = mnesia:write(?CHAIN_TAB, Chain#chain{services = NServices}, write), - {ok, serialize_service({ServiceName, NService})}; - {error, Reason} -> - {error, Reason} - end - end - end, - update_chain(ChainID, UpdateFun). - -lookup_service(ChainID, ServiceName) -> - case mnesia:dirty_read(?CHAIN_TAB, ChainID) of - [] -> - {error, {not_found, {chain, ChainID}}}; - [#chain{services = Services}] -> - case lists:keytake(ServiceName, 1, Services) of - {value, Service, _} -> - {ok, serialize_service(Service)}; - false -> - {error, {not_found, {service, ServiceName}}} - end - end. - -list_services(ChainID) -> - case mnesia:dirty_read(?CHAIN_TAB, ChainID) of - [] -> - {error, {not_found, {chain, ChainID}}}; - [#chain{services = Services}] -> - {ok, serialize_services(Services)} - end. - -move_service_to_the_front(ChainID, ServiceName) -> - UpdateFun = fun(Chain = #chain{services = Services}) -> - case move_service_to_the_front_(ServiceName, Services) of - {ok, NServices} -> - NChain = Chain#chain{services = NServices}, - mnesia:write(?CHAIN_TAB, NChain, write); - {error, Reason} -> - {error, Reason} - end - end, - update_chain(ChainID, UpdateFun). - -move_service_to_the_end(ChainID, ServiceName) -> - UpdateFun = fun(Chain = #chain{services = Services}) -> - case move_service_to_the_end_(ServiceName, Services) of - {ok, NServices} -> - NChain = Chain#chain{services = NServices}, - mnesia:write(?CHAIN_TAB, NChain, write); - {error, Reason} -> - {error, Reason} - end - end, - update_chain(ChainID, UpdateFun). - -move_service_to_the_nth(ChainID, ServiceName, N) -> - UpdateFun = fun(Chain = #chain{services = Services}) -> - case move_service_to_the_nth_(ServiceName, Services, N) of - {ok, NServices} -> - NChain = Chain#chain{services = NServices}, - mnesia:write(?CHAIN_TAB, NChain, write); - {error, Reason} -> - {error, Reason} - end - end, - update_chain(ChainID, UpdateFun). - -import_users(ChainID, ServiceName, Filename) -> - call_service(ChainID, ServiceName, import_users, [Filename]). - -add_user(ChainID, ServiceName, UserInfo) -> - call_service(ChainID, ServiceName, add_user, [UserInfo]). - -delete_user(ChainID, ServiceName, UserID) -> - call_service(ChainID, ServiceName, delete_user, [UserID]). - -update_user(ChainID, ServiceName, UserID, NewUserInfo) -> - call_service(ChainID, ServiceName, update_user, [UserID, NewUserInfo]). - -lookup_user(ChainID, ServiceName, UserID) -> - call_service(ChainID, ServiceName, lookup_user, [UserID]). - -list_users(ChainID, ServiceName) -> - call_service(ChainID, ServiceName, list_users, []). - -%%------------------------------------------------------------------------------ -%% Internal functions -%%------------------------------------------------------------------------------ - -find_attrs(App, AttrName) -> - [{App, Mod, Attr} || {ok, Modules} <- [application:get_key(App, modules)], - Mod <- Modules, - {Name, Attrs} <- module_attributes(Mod), Name =:= AttrName, - Attr <- Attrs]. - -module_attributes(Module) -> - try Module:module_info(attributes) - catch - error:undef -> [] - end. - -do_register_service_types(ServiceTypes) -> - trans(fun lists:foreach/2, [fun insert_service_type/1, ServiceTypes]). - -insert_service_type(ServiceType) -> - mnesia:write(?SERVICE_TYPE_TAB, ServiceType, write). - -find_service_type(Name) -> - case mnesia:dirty_read(?SERVICE_TYPE_TAB, Name) of - [ServiceType] -> {ok, ServiceType}; - [] -> {error, not_found} - end. - -validate_service_params(ServiceParams) -> - case validate_service_names(ServiceParams) of - ok -> - validate_other_service_params(ServiceParams); - {error, Reason} -> - {error, Reason} - end. - -validate_service_names(ServiceParams) -> - Names = [Name || #{name := Name} <- ServiceParams], - no_duplicate_names(Names). - -validate_other_service_params(ServiceParams) -> - validate_other_service_params(ServiceParams, []). - -validate_other_service_params([], Acc) -> - {ok, lists:reverse(Acc)}; -validate_other_service_params([#{type := Type, params := Params} = ServiceParams | More], Acc) -> - case find_service_type(Type) of - {ok, #service_type{provider = Provider, params_spec = ParamsSpec}} -> - NParams = emqx_rule_validator:validate_params(Params, ParamsSpec), - validate_other_service_params(More, - [ServiceParams#{params => NParams, - original_params => Params, - provider => Provider} | Acc]); - {error, not_found} -> - {error, {not_found, {service_type, Type}}} - end. - -no_duplicate_names(Names) -> - no_duplicate_names(Names, #{}). - -no_duplicate_names([], _) -> - ok; -no_duplicate_names([Name | More], Acc) -> - case maps:is_key(Name, Acc) of - false -> no_duplicate_names(More, Acc#{Name => true}); - true -> {error, {duplicate, Name}} - end. - -create_services(ChainID, ServiceParams) -> - create_services(ChainID, ServiceParams, []). - -create_services(_ChainID, [], Acc) -> - {ok, lists:reverse(Acc)}; -create_services(ChainID, [#{name := Name, - type := Type, - provider := Provider, - params := Params, - original_params := OriginalParams} | More], Acc) -> - case Provider:create(ChainID, Name, Params) of - {ok, State} -> - Service = #service{name = Name, - type = Type, - provider = Provider, - params = OriginalParams, - state = State}, - create_services(ChainID, More, [{Name, Service} | Acc]); - {error, Reason} -> - delete_services_(Acc), - {error, Reason} - end. - -delete_services_([]) -> - ok; -delete_services_([{_, #service{provider = Provider, state = State}} | More]) -> - Provider:destroy(State), - delete_services_(More). - -extract_services(ServiceNames, Services) -> - extract_services(ServiceNames, Services, []). - -extract_services([], Rest, Extracted) -> - {ok, lists:reverse(Extracted), Rest}; -extract_services([ServiceName | More], Services, Acc) -> - case lists:keytake(ServiceName, 1, Services) of - {value, Extracted, Rest} -> - extract_services(More, Rest, [Extracted | Acc]); - false -> - {error, {not_found, {service, ServiceName}}} - end. - -move_service_to_the_front_(ServiceName, Services) -> - move_service_to_the_front_(ServiceName, Services, []). - -move_service_to_the_front_(ServiceName, [], _) -> - {error, {not_found, {service, ServiceName}}}; -move_service_to_the_front_(ServiceName, [{ServiceName, _} = Service | More], Passed) -> - {ok, [Service | (lists:reverse(Passed) ++ More)]}; -move_service_to_the_front_(ServiceName, [Service | More], Passed) -> - move_service_to_the_front_(ServiceName, More, [Service | Passed]). - -move_service_to_the_end_(ServiceName, Services) -> - move_service_to_the_end_(ServiceName, Services, []). - -move_service_to_the_end_(ServiceName, [], _) -> - {error, {not_found, {service, ServiceName}}}; -move_service_to_the_end_(ServiceName, [{ServiceName, _} = Service | More], Passed) -> - {ok, lists:reverse(Passed) ++ More ++ [Service]}; -move_service_to_the_end_(ServiceName, [Service | More], Passed) -> - move_service_to_the_end_(ServiceName, More, [Service | Passed]). - -move_service_to_the_nth_(ServiceName, Services, N) - when N =< length(Services) andalso N > 0 -> - move_service_to_the_nth_(ServiceName, Services, N, []); -move_service_to_the_nth_(_, _, _) -> - {error, out_of_range}. - -move_service_to_the_nth_(ServiceName, [], _, _) -> - {error, {not_found, {service, ServiceName}}}; -move_service_to_the_nth_(ServiceName, [{ServiceName, _} = Service | More], N, Passed) - when N =< length(Passed) -> - {L1, L2} = lists:split(N - 1, lists:reverse(Passed)), - {ok, L1 ++ [Service] ++ L2 ++ More}; -move_service_to_the_nth_(ServiceName, [{ServiceName, _} = Service | More], N, Passed) -> - {L1, L2} = lists:split(N - length(Passed) - 1, More), - {ok, lists:reverse(Passed) ++ L1 ++ [Service] ++ L2}; -move_service_to_the_nth_(ServiceName, [Service | More], N, Passed) -> - move_service_to_the_nth_(ServiceName, More, N, [Service | Passed]). - -update_chain(ChainID, UpdateFun) -> - trans( - fun() -> - case mnesia:read(?CHAIN_TAB, ChainID, write) of - [] -> - {error, {not_found, {chain, ChainID}}}; - [Chain] -> - UpdateFun(Chain) - end - end). - -call_service(ChainID, ServiceName, Func, Args) -> - case mnesia:dirty_read(?CHAIN_TAB, ChainID) of - [] -> - {error, {not_found, {chain, ChainID}}}; - [#chain{services = Services}] -> - case proplists:get_value(ServiceName, Services, undefined) of - undefined -> - {error, {not_found, {service, ServiceName}}}; - #service{provider = Provider, - state = State} -> - case erlang:function_exported(Provider, Func, length(Args) + 1) of - true -> - erlang:apply(Provider, Func, Args ++ [State]); - false -> - {error, unsupported_feature} - end - end - end. - -serialize_chain(#chain{id = ID, - services = Services, - created_at = CreatedAt}) -> - #{id => ID, - services => serialize_services(Services), - created_at => CreatedAt}. - -serialize_services(Services) -> - [serialize_service(Service) || Service <- Services]. - -serialize_service({_, #service{name = Name, - type = Type, - params = Params}}) -> - #{name => Name, - type => Type, - params => Params}. - -trans(Fun) -> - trans(Fun, []). - -trans(Fun, Args) -> - case mnesia:transaction(Fun, Args) of - {atomic, Res} -> Res; - {aborted, Reason} -> {error, Reason} - end. diff --git a/apps/emqx_authentication/src/emqx_authentication_api.erl b/apps/emqx_authentication/src/emqx_authentication_api.erl deleted file mode 100644 index 74887a0b2..000000000 --- a/apps/emqx_authentication/src/emqx_authentication_api.erl +++ /dev/null @@ -1,407 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_authentication_api). - --export([ create_chain/2 - , delete_chain/2 - , lookup_chain/2 - , list_chains/2 - , add_service/2 - , delete_service/2 - , update_service/2 - , lookup_service/2 - , list_services/2 - , move_service/2 - , import_users/2 - , add_user/2 - , delete_user/2 - , update_user/2 - , lookup_user/2 - , list_users/2 - ]). - --import(minirest, [return/1]). - --rest_api(#{name => create_chain, - method => 'POST', - path => "/authentication/chains", - func => create_chain, - descr => "Create a chain" - }). - --rest_api(#{name => delete_chain, - method => 'DELETE', - path => "/authentication/chains/:bin:id", - func => delete_chain, - descr => "Delete chain" - }). - --rest_api(#{name => lookup_chain, - method => 'GET', - path => "/authentication/chains/:bin:id", - func => lookup_chain, - descr => "Lookup chain" - }). - --rest_api(#{name => list_chains, - method => 'GET', - path => "/authentication/chains", - func => list_chains, - descr => "List all chains" - }). - --rest_api(#{name => add_service, - method => 'POST', - path => "/authentication/chains/:bin:id/services", - func => add_service, - descr => "Add service to chain" - }). - --rest_api(#{name => delete_service, - method => 'DELETE', - path => "/authentication/chains/:bin:id/services/:bin:service_name", - func => delete_service, - descr => "Delete service from chain" - }). - --rest_api(#{name => update_service, - method => 'PUT', - path => "/authentication/chains/:bin:id/services/:bin:service_name", - func => update_service, - descr => "Update service in chain" - }). - --rest_api(#{name => lookup_service, - method => 'GET', - path => "/authentication/chains/:bin:id/services/:bin:service_name", - func => lookup_service, - descr => "Lookup service in chain" - }). - --rest_api(#{name => list_services, - method => 'GET', - path => "/authentication/chains/:bin:id/services", - func => list_services, - descr => "List services in chain" - }). - --rest_api(#{name => move_service, - method => 'POST', - path => "/authentication/chains/:bin:id/services/:bin:service_name/position", - func => move_service, - descr => "Change the order of services" - }). - --rest_api(#{name => import_users, - method => 'POST', - path => "/authentication/chains/:bin:id/services/:bin:service_name/import-users", - func => import_users, - descr => "Import users" - }). - --rest_api(#{name => add_user, - method => 'POST', - path => "/authentication/chains/:bin:id/services/:bin:service_name/users", - func => add_user, - descr => "Add user" - }). - --rest_api(#{name => delete_user, - method => 'DELETE', - path => "/authentication/chains/:bin:id/services/:bin:service_name/users/:bin:user_id", - func => delete_user, - descr => "Delete user" - }). - --rest_api(#{name => update_user, - method => 'PUT', - path => "/authentication/chains/:bin:id/services/:bin:service_name/users/:bin:user_id", - func => update_user, - descr => "Update user" - }). - --rest_api(#{name => lookup_user, - method => 'GET', - path => "/authentication/chains/:bin:id/services/:bin:service_name/users/:bin:user_id", - func => lookup_user, - descr => "Lookup user" - }). - -%% TODO: Support pagination --rest_api(#{name => list_users, - method => 'GET', - path => "/authentication/chains/:bin:id/services/:bin:service_name/users", - func => list_users, - descr => "List all users" - }). - -create_chain(Binding, Params) -> - do_create_chain(uri_decode(Binding), maps:from_list(Params)). - -do_create_chain(_Binding, #{<<"id">> := ChainID}) -> - case emqx_authentication:create_chain(#{id => ChainID}) of - {ok, Chain} -> - return({ok, Chain}); - {error, Reason} -> - return(serialize_error(Reason)) - end; -do_create_chain(_Binding, _Params) -> - return(serialize_error({missing_parameter, id})). - -delete_chain(Binding, Params) -> - do_delete_chain(uri_decode(Binding), maps:from_list(Params)). - -do_delete_chain(#{id := ChainID}, _Params) -> - case emqx_authentication:delete_chain(ChainID) of - ok -> - return(ok); - {error, Reason} -> - return(serialize_error(Reason)) - end. - -lookup_chain(Binding, Params) -> - do_lookup_chain(uri_decode(Binding), maps:from_list(Params)). - -do_lookup_chain(#{id := ChainID}, _Params) -> - case emqx_authentication:lookup_chain(ChainID) of - {ok, Chain} -> - return({ok, Chain}); - {error, Reason} -> - return(serialize_error(Reason)) - end. - -list_chains(Binding, Params) -> - do_list_chains(uri_decode(Binding), maps:from_list(Params)). - -do_list_chains(_Binding, _Params) -> - {ok, Chains} = emqx_authentication:list_chains(), - return({ok, Chains}). - -add_service(Binding, Params) -> - do_add_service(uri_decode(Binding), maps:from_list(Params)). - -do_add_service(#{id := ChainID}, #{<<"name">> := Name, - <<"type">> := Type, - <<"params">> := Params}) -> - case emqx_authentication:add_services(ChainID, [#{name => Name, - type => binary_to_existing_atom(Type, utf8), - params => maps:from_list(Params)}]) of - {ok, Services} -> - return({ok, Services}); - {error, Reason} -> - return(serialize_error(Reason)) - end; -%% TODO: Check missed field in params -do_add_service(_Binding, Params) -> - Missed = get_missed_params(Params, [<<"name">>, <<"type">>, <<"params">>]), - return(serialize_error({missing_parameter, Missed})). - -delete_service(Binding, Params) -> - do_delete_service(uri_decode(Binding), maps:from_list(Params)). - -do_delete_service(#{id := ChainID, - service_name := ServiceName}, _Params) -> - case emqx_authentication:delete_services(ChainID, [ServiceName]) of - ok -> - return(ok); - {error, Reason} -> - return(serialize_error(Reason)) - end. - -update_service(Binding, Params) -> - do_update_service(uri_decode(Binding), maps:from_list(Params)). - -%% TOOD: PUT method supports creation and update -do_update_service(#{id := ChainID, - service_name := ServiceName}, Params) -> - case emqx_authentication:update_service(ChainID, ServiceName, Params) of - {ok, Service} -> - return({ok, Service}); - {error, Reason} -> - return(serialize_error(Reason)) - end. - -lookup_service(Binding, Params) -> - do_lookup_service(uri_decode(Binding), maps:from_list(Params)). - -do_lookup_service(#{id := ChainID, - service_name := ServiceName}, _Params) -> - case emqx_authentication:lookup_service(ChainID, ServiceName) of - {ok, Service} -> - return({ok, Service}); - {error, Reason} -> - return(serialize_error(Reason)) - end. - -list_services(Binding, Params) -> - do_list_services(uri_decode(Binding), maps:from_list(Params)). - -do_list_services(#{id := ChainID}, _Params) -> - case emqx_authentication:list_services(ChainID) of - {ok, Services} -> - return({ok, Services}); - {error, Reason} -> - return(serialize_error(Reason)) - end. - -move_service(Binding, Params) -> - do_move_service(uri_decode(Binding), maps:from_list(Params)). - -do_move_service(#{id := ChainID, - service_name := ServiceName}, #{<<"position">> := <<"the front">>}) -> - case emqx_authentication:move_service_to_the_front(ChainID, ServiceName) of - ok -> - return(ok); - {error, Reason} -> - return(serialize_error(Reason)) - end; -do_move_service(#{id := ChainID, - service_name := ServiceName}, #{<<"position">> := <<"the end">>}) -> - case emqx_authentication:move_service_to_the_end(ChainID, ServiceName) of - ok -> - return(ok); - {error, Reason} -> - return(serialize_error(Reason)) - end; -do_move_service(#{id := ChainID, - service_name := ServiceName}, #{<<"position">> := N}) when is_number(N) -> - case emqx_authentication:move_service_to_the_nth(ChainID, ServiceName, N) of - ok -> - return(ok); - {error, Reason} -> - return(serialize_error(Reason)) - end; -do_move_service(_Binding, _Params) -> - return(serialize_error({missing_parameter, <<"position">>})). - -import_users(Binding, Params) -> - do_import_users(uri_decode(Binding), maps:from_list(Params)). - -do_import_users(#{id := ChainID, service_name := ServiceName}, - #{<<"filename">> := Filename}) -> - case emqx_authentication:import_users(ChainID, ServiceName, Filename) of - ok -> - return(ok); - {error, Reason} -> - return(serialize_error(Reason)) - end; -do_import_users(_Binding, Params) -> - Missed = get_missed_params(Params, [<<"filename">>, <<"file_format">>]), - return(serialize_error({missing_parameter, Missed})). - -add_user(Binding, Params) -> - do_add_user(uri_decode(Binding), maps:from_list(Params)). - -do_add_user(#{id := ChainID, - service_name := ServiceName}, UserInfo) -> - case emqx_authentication:add_user(ChainID, ServiceName, UserInfo) of - {ok, User} -> - return({ok, User}); - {error, Reason} -> - return(serialize_error(Reason)) - end. - -delete_user(Binding, Params) -> - do_delete_user(uri_decode(Binding), maps:from_list(Params)). - -do_delete_user(#{id := ChainID, - service_name := ServiceName, - user_id := UserID}, _Params) -> - case emqx_authentication:delete_user(ChainID, ServiceName, UserID) of - ok -> - return(ok); - {error, Reason} -> - return(serialize_error(Reason)) - end. - -update_user(Binding, Params) -> - do_update_user(uri_decode(Binding), maps:from_list(Params)). - -do_update_user(#{id := ChainID, - service_name := ServiceName, - user_id := UserID}, NewUserInfo) -> - case emqx_authentication:update_user(ChainID, ServiceName, UserID, NewUserInfo) of - {ok, User} -> - return({ok, User}); - {error, Reason} -> - return(serialize_error(Reason)) - end. - -lookup_user(Binding, Params) -> - do_lookup_user(uri_decode(Binding), maps:from_list(Params)). - -do_lookup_user(#{id := ChainID, - service_name := ServiceName, - user_id := UserID}, _Params) -> - case emqx_authentication:lookup_user(ChainID, ServiceName, UserID) of - {ok, User} -> - return({ok, User}); - {error, Reason} -> - return(serialize_error(Reason)) - end. - -list_users(Binding, Params) -> - do_list_users(uri_decode(Binding), maps:from_list(Params)). - -do_list_users(#{id := ChainID, - service_name := ServiceName}, _Params) -> - case emqx_authentication:list_users(ChainID, ServiceName) of - {ok, Users} -> - return({ok, Users}); - {error, Reason} -> - return(serialize_error(Reason)) - end. - -%%------------------------------------------------------------------------------ -%% Internal functions -%%------------------------------------------------------------------------------ - -uri_decode(Params) -> - maps:fold(fun(K, V, Acc) -> - Acc#{K => emqx_http_lib:uri_decode(V)} - end, #{}, Params). - -serialize_error({already_exists, {Type, ID}}) -> - {error, <<"ALREADY_EXISTS">>, list_to_binary(io_lib:format("~s '~s' already exists", [serialize_type(Type), ID]))}; -serialize_error({not_found, {Type, ID}}) -> - {error, <<"NOT_FOUND">>, list_to_binary(io_lib:format("~s '~s' not found", [serialize_type(Type), ID]))}; -serialize_error({duplicate, Name}) -> - {error, <<"INVALID_PARAMETER">>, list_to_binary(io_lib:format("Service name '~s' is duplicated", [Name]))}; -serialize_error({missing_parameter, Names = [_ | Rest]}) -> - Format = ["~s," || _ <- Rest] ++ ["~s"], - NFormat = binary_to_list(iolist_to_binary(Format)), - {error, <<"MISSING_PARAMETER">>, list_to_binary(io_lib:format("The input parameters " ++ NFormat ++ " that are mandatory for processing this request are not supplied.", Names))}; -serialize_error({missing_parameter, Name}) -> - {error, <<"MISSING_PARAMETER">>, list_to_binary(io_lib:format("The input parameter '~s' that is mandatory for processing this request is not supplied.", [Name]))}; -serialize_error(_) -> - {error, <<"UNKNOWN_ERROR">>, <<"Unknown error">>}. - -serialize_type(service) -> - "Service"; -serialize_type(chain) -> - "Chain"; -serialize_type(service_type) -> - "Service type". - -get_missed_params(Actual, Expected) -> - Keys = lists:foldl(fun(Key, Acc) -> - case maps:is_key(Key, Actual) of - true -> Acc; - false -> [Key | Acc] - end - end, [], Expected), - lists:reverse(Keys). diff --git a/apps/emqx_authentication/src/emqx_authentication_app.erl b/apps/emqx_authentication/src/emqx_authentication_app.erl deleted file mode 100644 index 2d395def7..000000000 --- a/apps/emqx_authentication/src/emqx_authentication_app.erl +++ /dev/null @@ -1,34 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_authentication_app). - --behaviour(application). - --emqx_plugin(?MODULE). - -%% Application callbacks --export([ start/2 - , stop/1 - ]). - -start(_StartType, _StartArgs) -> - {ok, Sup} = emqx_authentication_sup:start_link(), - ok = emqx_authentication:register_service_types(), - {ok, Sup}. - -stop(_State) -> - ok. diff --git a/apps/emqx_authentication/src/emqx_authentication_jwt.erl b/apps/emqx_authentication/src/emqx_authentication_jwt.erl deleted file mode 100644 index 2b8024e1c..000000000 --- a/apps/emqx_authentication/src/emqx_authentication_jwt.erl +++ /dev/null @@ -1,409 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_authentication_jwt). - --export([ create/3 - , update/4 - , authenticate/2 - , destroy/1 - ]). - --service_type(#{ - name => jwt, - params_spec => #{ - use_jwks => #{ - order => 1, - type => boolean - }, - jwks_endpoint => #{ - order => 2, - type => string - }, - refresh_interval => #{ - order => 3, - type => number - }, - algorithm => #{ - order => 3, - type => string, - enum => [<<"hmac-based">>, <<"public-key">>] - }, - secret => #{ - order => 4, - type => string - }, - secret_base64_encoded => #{ - order => 5, - type => boolean - }, - jwt_certfile => #{ - order => 6, - type => file - }, - cacertfile => #{ - order => 7, - type => file - }, - keyfile => #{ - order => 8, - type => file - }, - certfile => #{ - order => 9, - type => file - }, - verify => #{ - order => 10, - type => boolean - }, - server_name_indication => #{ - order => 11, - type => string - } - } -}). - --define(RULES, - #{ - use_jwks => [], - jwks_endpoint => [use_jwks], - refresh_interval => [use_jwks], - algorithm => [use_jwks], - secret => [algorithm], - secret_base64_encoded => [algorithm], - jwt_certfile => [algorithm], - cacertfile => [jwks_endpoint], - keyfile => [jwks_endpoint], - certfile => [jwks_endpoint], - verify => [jwks_endpoint], - server_name_indication => [jwks_endpoint], - verify_claims => [] - }). - -create(_ChainID, _ServiceName, Params) -> - try handle_options(Params) of - Opts -> - do_create(Opts) - catch - {error, Reason} -> - {error, Reason} - end. - -update(_ChainID, _ServiceName, Params, State) -> - try handle_options(Params) of - Opts -> - do_update(Opts, State) - catch - {error, Reason} -> - {error, Reason} - end. - -authenticate(ClientInfo = #{password := JWT}, #{jwk := JWK, - verify_claims := VerifyClaims0}) -> - JWKs = case erlang:is_pid(JWK) of - false -> - [JWK]; - true -> - {ok, JWKs0} = emqx_authentication_jwks_connector:get_jwks(JWK), - JWKs0 - end, - VerifyClaims = replace_placeholder(VerifyClaims0, ClientInfo), - case verify(JWT, JWKs, VerifyClaims) of - ok -> ok; - {error, invalid_signature} -> ignore; - {error, {claims, _}} -> {stop, bad_passowrd} - end. - -destroy(#{jwks_connector := undefined}) -> - ok; -destroy(#{jwks_connector := Connector}) -> - _ = emqx_authentication_jwks_connector:stop(Connector), - ok. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -do_create(#{use_jwks := false, - algorithm := 'hmac-based', - secret := Secret0, - secret_base64_encoded := Base64Encoded} = Opts) -> - Secret = case Base64Encoded of - true -> - base64:decode(Secret0); - false -> - Secret0 - end, - JWK = jose_jwk:from_oct(Secret), - {ok, #{jwk => JWK, - verify_claims => maps:get(verify_claims, Opts)}}; - -do_create(#{use_jwks := false, - algorithm := 'public-key', - jwt_certfile := Certfile} = Opts) -> - JWK = jose_jwk:from_pem_file(Certfile), - {ok, #{jwk => JWK, - verify_claims => maps:get(verify_claims, Opts)}}; - -do_create(#{use_jwks := true} = Opts) -> - case emqx_authentication_jwks_connector:start_link(Opts) of - {ok, Connector} -> - {ok, #{jwk => Connector, - verify_claims => maps:get(verify_claims, Opts)}}; - {error, Reason} -> - {error, Reason} - end. - -do_update(Opts, #{jwk_connector := undefined}) -> - do_create(Opts); -do_update(#{use_jwks := false} = Opts, #{jwk_connector := Connector}) -> - _ = emqx_authentication_jwks_connector:stop(Connector), - do_create(Opts); -do_update(#{use_jwks := true} = Opts, #{jwk_connector := Connector} = State) -> - ok = emqx_authentication_jwks_connector:update(Connector, Opts), - {ok, State}. - -replace_placeholder(L, Variables) -> - replace_placeholder(L, Variables, []). - -replace_placeholder([], _Variables, Acc) -> - Acc; -replace_placeholder([{Name, {placeholder, PL}} | More], Variables, Acc) -> - Value = maps:get(PL, Variables), - replace_placeholder(More, Variables, [{Name, Value} | Acc]); -replace_placeholder([{Name, Value} | More], Variables, Acc) -> - replace_placeholder(More, Variables, [{Name, Value} | Acc]). - -verify(_JWS, [], _VerifyClaims) -> - {error, invalid_signature}; -verify(JWS, [JWK | More], VerifyClaims) -> - case jose_jws:verify(JWK, JWS) of - {true, Payload, _JWS} -> - Claims = emqx_json:decode(Payload, [return_maps]), - verify_claims(Claims, VerifyClaims); - {false, _, _} -> - verify(JWS, More, VerifyClaims) - end. - -verify_claims(Claims, VerifyClaims0) -> - Now = os:system_time(seconds), - VerifyClaims = [{<<"exp">>, fun(ExpireTime) -> - Now < ExpireTime - end}, - {<<"iat">>, fun(IssueAt) -> - IssueAt =< Now - end}, - {<<"nbf">>, fun(NotBefore) -> - NotBefore =< Now - end}] ++ VerifyClaims0, - do_verify_claims(Claims, VerifyClaims). - -do_verify_claims(_Claims, []) -> - ok; -do_verify_claims(Claims, [{Name, Fun} | More]) when is_function(Fun) -> - case maps:take(Name, Claims) of - error -> - do_verify_claims(Claims, More); - {Value, NClaims} -> - case Fun(Value) of - true -> - do_verify_claims(NClaims, More); - _ -> - {error, {claims, {Name, Value}}} - end - end; -do_verify_claims(Claims, [{Name, Value} | More]) -> - case maps:take(Name, Claims) of - error -> - do_verify_claims(Claims, More); - {Value, NClaims} -> - do_verify_claims(NClaims, More); - {Value0, _} -> - {error, {claims, {Name, Value0}}} - end. - -handle_options(Opts0) when is_map(Opts0) -> - Ks = maps:fold(fun(K, _, Acc) -> - [atom_to_binary(K, utf8) | Acc] - end, [], ?RULES), - Opts1 = maps:to_list(maps:with(Ks, Opts0)), - handle_options([{binary_to_existing_atom(K, utf8), V} || {K, V} <- Opts1]); - -handle_options(Opts0) when is_list(Opts0) -> - Opts1 = add_missing_options(Opts0), - process_options({Opts1, [], length(Opts1)}, #{}). - -add_missing_options(Opts) -> - AllOpts = maps:keys(?RULES), - Fun = fun(K, Acc) -> - case proplists:is_defined(K, Acc) of - true -> - Acc; - false -> - [{K, unbound} | Acc] - end - end, - lists:foldl(Fun, Opts, AllOpts). - -process_options({[], [], _}, OptsMap) -> - OptsMap; -process_options({[], Skipped, Counter}, OptsMap) - when length(Skipped) < Counter -> - process_options({Skipped, [], length(Skipped)}, OptsMap); -process_options({[], _Skipped, _Counter}, _OptsMap) -> - throw({error, faulty_configuration}); -process_options({[{K, V} = Opt | More], Skipped, Counter}, OptsMap0) -> - case check_dependencies(K, OptsMap0) of - true -> - OptsMap1 = handle_option(K, V, OptsMap0), - process_options({More, Skipped, Counter}, OptsMap1); - false -> - process_options({More, [Opt | Skipped], Counter}, OptsMap0) - end. - -%% TODO: This is not a particularly good implementation(K => needless), it needs to be improved -handle_option(use_jwks, true, OptsMap) -> - OptsMap#{use_jwks => true, - algorithm => needless}; -handle_option(use_jwks, false, OptsMap) -> - OptsMap#{use_jwks => false, - jwks_endpoint => needless}; -handle_option(jwks_endpoint = Opt, unbound, #{use_jwks := true}) -> - throw({error, {options, {Opt, unbound}}}); -handle_option(jwks_endpoint, Value, #{use_jwks := true} = OptsMap) - when Value =/= unbound -> - case emqx_http_lib:uri_parse(Value) of - {ok, #{scheme := http}} -> - OptsMap#{enable_ssl => false, - jwks_endpoint => Value}; - {ok, #{scheme := https}} -> - OptsMap#{enable_ssl => true, - jwks_endpoint => Value}; - {error, _Reason} -> - throw({error, {options, {jwks_endpoint, Value}}}) - end; -handle_option(refresh_interval = Opt, Value0, #{use_jwks := true} = OptsMap) -> - Value = validate_option(Opt, Value0), - OptsMap#{Opt => Value}; -handle_option(algorithm = Opt, Value0, #{use_jwks := false} = OptsMap) -> - Value = validate_option(Opt, Value0), - OptsMap#{Opt => Value}; -handle_option(secret = Opt, unbound, #{algorithm := 'hmac-based'}) -> - throw({error, {options, {Opt, unbound}}}); -handle_option(secret = Opt, Value, #{algorithm := 'hmac-based'} = OptsMap) -> - OptsMap#{Opt => Value}; -handle_option(secret_base64_encoded = Opt, Value0, #{algorithm := 'hmac-based'} = OptsMap) -> - Value = validate_option(Opt, Value0), - OptsMap#{Opt => Value}; -handle_option(jwt_certfile = Opt, unbound, #{algorithm := 'public-key'}) -> - throw({error, {options, {Opt, unbound}}}); -handle_option(jwt_certfile = Opt, Value, #{algorithm := 'public-key'} = OptsMap) -> - OptsMap#{Opt => Value}; -handle_option(verify = Opt, Value0, #{enable_ssl := true} = OptsMap) -> - Value = validate_option(Opt, Value0), - OptsMap#{Opt => Value}; -handle_option(cacertfile = Opt, Value, #{enable_ssl := true} = OptsMap) - when Value =/= unbound -> - OptsMap#{Opt => Value}; -handle_option(certfile, unbound, #{enable_ssl := true} = OptsMap) -> - OptsMap; -handle_option(certfile = Opt, Value, #{enable_ssl := true} = OptsMap) -> - OptsMap#{Opt => Value}; -handle_option(keyfile, unbound, #{enable_ssl := true} = OptsMap) -> - OptsMap; -handle_option(keyfile = Opt, Value, #{enable_ssl := true} = OptsMap) -> - OptsMap#{Opt => Value}; -handle_option(server_name_indication = Opt, Value0, #{enable_ssl := true} = OptsMap) -> - Value = validate_option(Opt, Value0), - OptsMap#{Opt => Value}; -handle_option(verify_claims = Opt, Value0, OptsMap) -> - Value = handle_verify_claims(Value0), - OptsMap#{Opt => Value}; -handle_option(_Opt, _Value, OptsMap) -> - OptsMap. - -validate_option(refresh_interval, unbound) -> - 300; -validate_option(refresh_interval, Value) when is_integer(Value) -> - Value; -validate_option(algorithm, <<"hmac-based">>) -> - 'hmac-based'; -validate_option(algorithm, <<"public-key">>) -> - 'public-key'; -validate_option(secret_base64_encoded, unbound) -> - false; -validate_option(secret_base64_encoded, Value) when is_boolean(Value) -> - Value; -validate_option(verify, unbound) -> - verify_none; -validate_option(verify, true) -> - verify_peer; -validate_option(verify, false) -> - verify_none; -validate_option(server_name_indication, unbound) -> - disable; -validate_option(server_name_indication, <<"disable">>) -> - disable; -validate_option(server_name_indication, Value) when is_list(Value) -> - Value; -validate_option(Opt, Value) -> - throw({error, {options, {Opt, Value}}}). - -handle_verify_claims(Opts0) -> - try handle_verify_claims(Opts0, []) - catch - error:_ -> - throw({error, {options, {verify_claims, Opts0}}}) - end. - -handle_verify_claims([], Acc) -> - Acc; -handle_verify_claims([{Name, Expected0} | More], Acc) - when is_binary(Name) andalso is_binary(Expected0) -> - Expected = handle_placeholder(Expected0), - handle_verify_claims(More, [{Name, Expected} | Acc]). - -handle_placeholder(Placeholder0) -> - case re:run(Placeholder0, "^\\$\\{[a-z0-9\\_]+\\}$", [{capture, all}]) of - {match, [{Offset, Length}]} -> - Placeholder1 = binary:part(Placeholder0, Offset + 2, Length - 3), - Placeholder2 = validate_placeholder(Placeholder1), - {placeholder, Placeholder2}; - nomatch -> - Placeholder0 - end. - -validate_placeholder(<<"clientid">>) -> - clientid; -validate_placeholder(<<"username">>) -> - username. - -check_dependencies(Opt, OptsMap) -> - case maps:get(Opt, ?RULES) of - [] -> - true; - Deps -> - option_already_defined(Opt, OptsMap) orelse - dependecies_already_defined(Deps, OptsMap) - end. - -option_already_defined(Opt, OptsMap) -> - maps:get(Opt, OptsMap, unbound) =/= unbound. - -dependecies_already_defined(Deps, OptsMap) -> - Fun = fun(Opt) -> option_already_defined(Opt, OptsMap) end, - lists:all(Fun, Deps). diff --git a/apps/emqx_authentication/test/emqx_authentication_SUITE.erl b/apps/emqx_authentication/test/emqx_authentication_SUITE.erl deleted file mode 100644 index d110d940a..000000000 --- a/apps/emqx_authentication/test/emqx_authentication_SUITE.erl +++ /dev/null @@ -1,191 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_authentication_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("common_test/include/ct.hrl"). --include_lib("eunit/include/eunit.hrl"). - --define(AUTH, emqx_authentication). - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_authentication]), - Config. - -end_per_suite(_) -> - emqx_ct_helpers:stop_apps([emqx_authentication]), - ok. - -t_chain(_) -> - ChainID = <<"mychain">>, - ?assertMatch({ok, #{id := ChainID, services := []}}, ?AUTH:create_chain(#{id => ChainID})), - ?assertEqual({error, {already_exists, {chain, ChainID}}}, ?AUTH:create_chain(#{id => ChainID})), - ?assertMatch({ok, #{id := ChainID, services := []}}, ?AUTH:lookup_chain(ChainID)), - ?assertEqual(ok, ?AUTH:delete_chain(ChainID)), - ?assertMatch({error, {not_found, {chain, ChainID}}}, ?AUTH:lookup_chain(ChainID)), - ok. - -t_service(_) -> - ChainID = <<"mychain">>, - ?assertMatch({ok, #{id := ChainID, services := []}}, ?AUTH:create_chain(#{id => ChainID})), - ?assertMatch({ok, #{id := ChainID, services := []}}, ?AUTH:lookup_chain(ChainID)), - - ServiceName1 = <<"myservice1">>, - ServiceParams1 = #{name => ServiceName1, - type => mnesia, - params => #{ - user_id_type => <<"username">>, - password_hash_algorithm => <<"sha256">>}}, - ?assertEqual({ok, [ServiceParams1]}, ?AUTH:add_services(ChainID, [ServiceParams1])), - ?assertEqual({ok, ServiceParams1}, ?AUTH:lookup_service(ChainID, ServiceName1)), - ?assertEqual({ok, [ServiceParams1]}, ?AUTH:list_services(ChainID)), - ?assertEqual({error, {already_exists, {service, ServiceName1}}}, ?AUTH:add_services(ChainID, [ServiceParams1])), - - ServiceName2 = <<"myservice2">>, - ServiceParams2 = ServiceParams1#{name => ServiceName2}, - ?assertEqual({ok, [ServiceParams2]}, ?AUTH:add_services(ChainID, [ServiceParams2])), - ?assertMatch({ok, #{id := ChainID, services := [ServiceParams1, ServiceParams2]}}, ?AUTH:lookup_chain(ChainID)), - ?assertEqual({ok, ServiceParams2}, ?AUTH:lookup_service(ChainID, ServiceName2)), - ?assertEqual({ok, [ServiceParams1, ServiceParams2]}, ?AUTH:list_services(ChainID)), - - ?assertEqual(ok, ?AUTH:move_service_to_the_front(ChainID, ServiceName2)), - ?assertEqual({ok, [ServiceParams2, ServiceParams1]}, ?AUTH:list_services(ChainID)), - ?assertEqual(ok, ?AUTH:move_service_to_the_end(ChainID, ServiceName2)), - ?assertEqual({ok, [ServiceParams1, ServiceParams2]}, ?AUTH:list_services(ChainID)), - ?assertEqual(ok, ?AUTH:move_service_to_the_nth(ChainID, ServiceName2, 1)), - ?assertEqual({ok, [ServiceParams2, ServiceParams1]}, ?AUTH:list_services(ChainID)), - ?assertEqual({error, out_of_range}, ?AUTH:move_service_to_the_nth(ChainID, ServiceName2, 3)), - ?assertEqual({error, out_of_range}, ?AUTH:move_service_to_the_nth(ChainID, ServiceName2, 0)), - ?assertEqual(ok, ?AUTH:delete_services(ChainID, [ServiceName1, ServiceName2])), - ?assertEqual({ok, []}, ?AUTH:list_services(ChainID)), - ?assertEqual(ok, ?AUTH:delete_chain(ChainID)), - ok. - -t_mnesia_service(_) -> - ChainID = <<"mychain">>, - ?assertMatch({ok, #{id := ChainID, services := []}}, ?AUTH:create_chain(#{id => ChainID})), - - ServiceName = <<"myservice">>, - ServiceParams = #{name => ServiceName, - type => mnesia, - params => #{ - user_id_type => <<"username">>, - password_hash_algorithm => <<"sha256">>}}, - ?assertEqual({ok, [ServiceParams]}, ?AUTH:add_services(ChainID, [ServiceParams])), - - UserInfo = #{<<"user_id">> => <<"myuser">>, - <<"password">> => <<"mypass">>}, - ?assertEqual({ok, #{user_id => <<"myuser">>}}, ?AUTH:add_user(ChainID, ServiceName, UserInfo)), - ?assertEqual({ok, #{user_id => <<"myuser">>}}, ?AUTH:lookup_user(ChainID, ServiceName, <<"myuser">>)), - ClientInfo = #{chain_id => ChainID, - username => <<"myuser">>, - password => <<"mypass">>}, - ?assertEqual(ok, ?AUTH:authenticate(ClientInfo)), - ClientInfo2 = ClientInfo#{username => <<"baduser">>}, - ?assertEqual({error, user_not_found}, ?AUTH:authenticate(ClientInfo2)), - ClientInfo3 = ClientInfo#{password => <<"badpass">>}, - ?assertEqual({error, bad_password}, ?AUTH:authenticate(ClientInfo3)), - UserInfo2 = UserInfo#{<<"password">> => <<"mypass2">>}, - ?assertEqual({ok, #{user_id => <<"myuser">>}}, ?AUTH:update_user(ChainID, ServiceName, <<"myuser">>, UserInfo2)), - ClientInfo4 = ClientInfo#{password => <<"mypass2">>}, - ?assertEqual(ok, ?AUTH:authenticate(ClientInfo4)), - ?assertEqual(ok, ?AUTH:delete_user(ChainID, ServiceName, <<"myuser">>)), - ?assertEqual({error, not_found}, ?AUTH:lookup_user(ChainID, ServiceName, <<"myuser">>)), - - ?assertEqual({ok, #{user_id => <<"myuser">>}}, ?AUTH:add_user(ChainID, ServiceName, UserInfo)), - ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:lookup_user(ChainID, ServiceName, <<"myuser">>)), - ?assertEqual(ok, ?AUTH:delete_services(ChainID, [ServiceName])), - ?assertEqual({ok, [ServiceParams]}, ?AUTH:add_services(ChainID, [ServiceParams])), - ?assertMatch({error, not_found}, ?AUTH:lookup_user(ChainID, ServiceName, <<"myuser">>)), - - ?assertEqual(ok, ?AUTH:delete_chain(ChainID)), - ?assertEqual([], ets:tab2list(mnesia_basic_auth)), - ok. - -t_import(_) -> - ChainID = <<"mychain">>, - ?assertMatch({ok, #{id := ChainID, services := []}}, ?AUTH:create_chain(#{id => ChainID})), - - ServiceName = <<"myservice">>, - ServiceParams = #{name => ServiceName, - type => mnesia, - params => #{ - user_id_type => <<"username">>, - password_hash_algorithm => <<"sha256">>}}, - ?assertEqual({ok, [ServiceParams]}, ?AUTH:add_services(ChainID, [ServiceParams])), - - Dir = code:lib_dir(emqx_authentication, test), - ?assertEqual(ok, ?AUTH:import_users(ChainID, ServiceName, filename:join([Dir, "data/user-credentials.json"]))), - ?assertEqual(ok, ?AUTH:import_users(ChainID, ServiceName, filename:join([Dir, "data/user-credentials.csv"]))), - ?assertMatch({ok, #{user_id := <<"myuser1">>}}, ?AUTH:lookup_user(ChainID, ServiceName, <<"myuser1">>)), - ?assertMatch({ok, #{user_id := <<"myuser3">>}}, ?AUTH:lookup_user(ChainID, ServiceName, <<"myuser3">>)), - ClientInfo1 = #{chain_id => ChainID, - username => <<"myuser1">>, - password => <<"mypassword1">>}, - ?assertEqual(ok, ?AUTH:authenticate(ClientInfo1)), - ClientInfo2 = ClientInfo1#{username => <<"myuser3">>, - password => <<"mypassword3">>}, - ?assertEqual(ok, ?AUTH:authenticate(ClientInfo2)), - ?assertEqual(ok, ?AUTH:delete_chain(ChainID)), - ok. - -t_multi_mnesia_service(_) -> - ChainID = <<"mychain">>, - ?assertMatch({ok, #{id := ChainID, services := []}}, ?AUTH:create_chain(#{id => ChainID})), - - ServiceName1 = <<"myservice1">>, - ServiceParams1 = #{name => ServiceName1, - type => mnesia, - params => #{ - user_id_type => <<"username">>, - password_hash_algorithm => <<"sha256">>}}, - ServiceName2 = <<"myservice2">>, - ServiceParams2 = #{name => ServiceName2, - type => mnesia, - params => #{ - user_id_type => <<"clientid">>, - password_hash_algorithm => <<"sha256">>}}, - ?assertEqual({ok, [ServiceParams1]}, ?AUTH:add_services(ChainID, [ServiceParams1])), - ?assertEqual({ok, [ServiceParams2]}, ?AUTH:add_services(ChainID, [ServiceParams2])), - - ?assertEqual({ok, #{user_id => <<"myuser">>}}, - ?AUTH:add_user(ChainID, ServiceName1, - #{<<"user_id">> => <<"myuser">>, - <<"password">> => <<"mypass1">>})), - ?assertEqual({ok, #{user_id => <<"myclient">>}}, - ?AUTH:add_user(ChainID, ServiceName2, - #{<<"user_id">> => <<"myclient">>, - <<"password">> => <<"mypass2">>})), - ClientInfo1 = #{chain_id => ChainID, - username => <<"myuser">>, - clientid => <<"myclient">>, - password => <<"mypass1">>}, - ?assertEqual(ok, ?AUTH:authenticate(ClientInfo1)), - ?assertEqual(ok, ?AUTH:move_service_to_the_front(ChainID, ServiceName2)), - ?assertEqual({error, bad_password}, ?AUTH:authenticate(ClientInfo1)), - ClientInfo2 = ClientInfo1#{password => <<"mypass2">>}, - ?assertEqual(ok, ?AUTH:authenticate(ClientInfo2)), - ?assertEqual(ok, ?AUTH:delete_chain(ChainID)), - ok. - - - diff --git a/apps/emqx_authentication/.formatter.exs b/apps/emqx_authn/.formatter.exs similarity index 100% rename from apps/emqx_authentication/.formatter.exs rename to apps/emqx_authn/.formatter.exs diff --git a/apps/emqx_authentication/data/user-credentials.csv b/apps/emqx_authn/data/user-credentials.csv similarity index 100% rename from apps/emqx_authentication/data/user-credentials.csv rename to apps/emqx_authn/data/user-credentials.csv diff --git a/apps/emqx_authentication/data/user-credentials.json b/apps/emqx_authn/data/user-credentials.json similarity index 100% rename from apps/emqx_authentication/data/user-credentials.json rename to apps/emqx_authn/data/user-credentials.json diff --git a/apps/emqx_authn/etc/emqx_authn.conf b/apps/emqx_authn/etc/emqx_authn.conf new file mode 100644 index 000000000..8af40fe8f --- /dev/null +++ b/apps/emqx_authn/etc/emqx_authn.conf @@ -0,0 +1,26 @@ +emqx_authn: { + enable: false + authenticators: [ + # { + # name: "authenticator1" + # mechanism: password-based + # server_type: built-in-database + # user_id_type: clientid + # }, + # { + # name: "authenticator2" + # mechanism: password-based + # server_type: mongodb + # server: "127.0.0.1:27017" + # database: mqtt + # collection: users + # selector: { + # username: "${mqtt-username}" + # } + # password_hash_field: password_hash + # salt_field: salt + # password_hash_algorithm: sha256 + # salt_position: prefix + # } + ] +} diff --git a/apps/emqx_authentication/include/emqx_authentication.hrl b/apps/emqx_authn/include/emqx_authn.hrl similarity index 65% rename from apps/emqx_authentication/include/emqx_authentication.hrl rename to apps/emqx_authn/include/emqx_authn.hrl index 09d3c5fc4..f9ba7c3b5 100644 --- a/apps/emqx_authentication/include/emqx_authentication.hrl +++ b/apps/emqx_authn/include/emqx_authn.hrl @@ -14,28 +14,26 @@ %% limitations under the License. %%-------------------------------------------------------------------- --define(APP, emqx_authentication). +-define(APP, emqx_authn). +-define(CHAIN, <<"mqtt">>). --type(service_type_name() :: atom()). --type(service_name() :: binary()). --type(chain_id() :: binary()). +-define(VER_1, <<"1">>). +-define(VER_2, <<"2">>). --record(service_type, - { name :: service_type_name() +-define(RE_PLACEHOLDER, "\\$\\{[a-z0-9\\-]+\\}"). + +-record(authenticator, + { id :: binary() + , name :: binary() , provider :: module() - , params_spec :: #{atom() => term()} - }). - --record(service, - { name :: service_name() - , type :: service_type_name() - , provider :: module() - , params :: map() + , config :: map() , state :: map() }). -record(chain, - { id :: chain_id() - , services :: [{service_name(), #service{}}] + { id :: binary() + , authenticators :: [{binary(), binary(), #authenticator{}}] , created_at :: integer() }). + +-define(AUTH_SHARD, emqx_authn_shard). diff --git a/apps/emqx_authentication/mix.exs b/apps/emqx_authn/mix.exs similarity index 95% rename from apps/emqx_authentication/mix.exs rename to apps/emqx_authn/mix.exs index 65bc1a0bb..75a7735b7 100644 --- a/apps/emqx_authentication/mix.exs +++ b/apps/emqx_authn/mix.exs @@ -3,7 +3,7 @@ defmodule EMQXAuthentication.MixProject do def project do [ - app: :emqx_authentication, + app: :emqx_authn, version: "0.1.0", build_path: "../../_build", config_path: "../../config/config.exs", diff --git a/apps/emqx_authentication/rebar.config b/apps/emqx_authn/rebar.config similarity index 84% rename from apps/emqx_authentication/rebar.config rename to apps/emqx_authn/rebar.config index 73696b033..32b5a43e0 100644 --- a/apps/emqx_authentication/rebar.config +++ b/apps/emqx_authn/rebar.config @@ -1,4 +1,6 @@ -{deps, []}. +{deps, [ + {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}} +]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, diff --git a/apps/emqx_authentication/src/emqx_authentication.app.src b/apps/emqx_authn/src/emqx_authn.app.src similarity index 54% rename from apps/emqx_authentication/src/emqx_authentication.app.src rename to apps/emqx_authn/src/emqx_authn.app.src index 4f55ca0a7..3b89d2a99 100644 --- a/apps/emqx_authentication/src/emqx_authentication.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,10 +1,10 @@ -{application, emqx_authentication, +{application, emqx_authn, [{description, "EMQ X Authentication"}, {vsn, "0.1.0"}, {modules, []}, - {registered, [emqx_authentication_sup, emqx_authentication_registry]}, - {applications, [kernel,stdlib]}, - {mod, {emqx_authentication_app,[]}}, + {registered, [emqx_authn_sup, emqx_authn_registry]}, + {applications, [kernel,stdlib,emqx_resource,ehttpc,epgsql,mysql,jose]}, + {mod, {emqx_authn_app,[]}}, {env, []}, {licenses, ["Apache-2.0"]}, {maintainers, ["EMQ X Team "]}, diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl new file mode 100644 index 000000000..27f3eab10 --- /dev/null +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -0,0 +1,430 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn). + +-include("emqx_authn.hrl"). + +-export([ enable/0 + , disable/0 + ]). + +-export([authenticate/2]). + +-export([ create_chain/1 + , delete_chain/1 + , lookup_chain/1 + , list_chains/0 + , create_authenticator/2 + , delete_authenticator/2 + , update_authenticator/3 + , update_or_create_authenticator/3 + , lookup_authenticator/2 + , list_authenticators/1 + , move_authenticator_to_the_nth/3 + ]). + +-export([ import_users/3 + , add_user/3 + , delete_user/3 + , update_user/4 + , lookup_user/3 + , list_users/2 + ]). + +-export([mnesia/1]). + +-boot_mnesia({mnesia, [boot]}). +-copy_mnesia({mnesia, [copy]}). + +-define(CHAIN_TAB, emqx_authn_chain). + +-rlog_shard({?AUTH_SHARD, ?CHAIN_TAB}). + +%%------------------------------------------------------------------------------ +%% Mnesia bootstrap +%%------------------------------------------------------------------------------ + +%% @doc Create or replicate tables. +-spec(mnesia(boot) -> ok). +mnesia(boot) -> + %% Optimize storage + StoreProps = [{ets, [{read_concurrency, true}]}], + %% Chain table + ok = ekka_mnesia:create_table(?CHAIN_TAB, [ + {ram_copies, [node()]}, + {record_name, chain}, + {local_content, true}, + {attributes, record_info(fields, chain)}, + {storage_properties, StoreProps}]); + +mnesia(copy) -> + ok = ekka_mnesia:copy_table(?CHAIN_TAB, ram_copies). + +enable() -> + case emqx:hook('client.authenticate', {?MODULE, authenticate, []}) of + ok -> ok; + {error, already_exists} -> ok + end. + +disable() -> + emqx:unhook('client.authenticate', {?MODULE, authenticate, []}), + ok. + +authenticate(Credential, _AuthResult) -> + case mnesia:dirty_read(?CHAIN_TAB, ?CHAIN) of + [#chain{authenticators = Authenticators}] -> + do_authenticate(Authenticators, Credential); + [] -> + {stop, {error, not_authorized}} + end. + +do_authenticate([], _) -> + {stop, {error, not_authorized}}; +do_authenticate([{_, _, #authenticator{provider = Provider, state = State}} | More], Credential) -> + case Provider:authenticate(Credential, State) of + ignore -> + do_authenticate(More, Credential); + Result -> + %% ok + %% {ok, AuthData} + %% {continue, AuthCache} + %% {continue, AuthData, AuthCache} + %% {error, Reason} + {stop, Result} + end. + +create_chain(#{id := ID}) -> + trans( + fun() -> + case mnesia:read(?CHAIN_TAB, ID, write) of + [] -> + Chain = #chain{id = ID, + authenticators = [], + created_at = erlang:system_time(millisecond)}, + mnesia:write(?CHAIN_TAB, Chain, write), + {ok, serialize_chain(Chain)}; + [_ | _] -> + {error, {already_exists, {chain, ID}}} + end + end). + +delete_chain(ID) -> + trans( + fun() -> + case mnesia:read(?CHAIN_TAB, ID, write) of + [] -> + {error, {not_found, {chain, ID}}}; + [#chain{authenticators = Authenticators}] -> + _ = [do_delete_authenticator(Authenticator) || {_, _, Authenticator} <- Authenticators], + mnesia:delete(?CHAIN_TAB, ID, write) + end + end). + +lookup_chain(ID) -> + case mnesia:dirty_read(?CHAIN_TAB, ID) of + [] -> + {error, {not_found, {chain, ID}}}; + [Chain] -> + {ok, serialize_chain(Chain)} + end. + +list_chains() -> + Chains = ets:tab2list(?CHAIN_TAB), + {ok, [serialize_chain(Chain) || Chain <- Chains]}. + +create_authenticator(ChainID, #{name := Name} = Config) -> + UpdateFun = + fun(Chain = #chain{authenticators = Authenticators}) -> + case lists:keymember(Name, 2, Authenticators) of + true -> + {error, name_has_be_used}; + false -> + AlreadyExist = fun(ID) -> + lists:keymember(ID, 1, Authenticators) + end, + AuthenticatorID = gen_id(AlreadyExist), + case do_create_authenticator(ChainID, AuthenticatorID, Config) of + {ok, Authenticator} -> + NAuthenticators = Authenticators ++ [{AuthenticatorID, Name, Authenticator}], + ok = mnesia:write(?CHAIN_TAB, Chain#chain{authenticators = NAuthenticators}, write), + {ok, serialize_authenticator(Authenticator)}; + {error, Reason} -> + {error, Reason} + end + end + end, + update_chain(ChainID, UpdateFun). + +delete_authenticator(ChainID, AuthenticatorID) -> + UpdateFun = fun(Chain = #chain{authenticators = Authenticators}) -> + case lists:keytake(AuthenticatorID, 1, Authenticators) of + false -> + {error, {not_found, {authenticator, AuthenticatorID}}}; + {value, {_, _, Authenticator}, NAuthenticators} -> + _ = do_delete_authenticator(Authenticator), + NChain = Chain#chain{authenticators = NAuthenticators}, + mnesia:write(?CHAIN_TAB, NChain, write) + end + end, + update_chain(ChainID, UpdateFun). + +update_authenticator(ChainID, AuthenticatorID, Config) -> + do_update_authenticator(ChainID, AuthenticatorID, Config, false). + +update_or_create_authenticator(ChainID, AuthenticatorID, Config) -> + do_update_authenticator(ChainID, AuthenticatorID, Config, true). + +do_update_authenticator(ChainID, AuthenticatorID, #{name := NewName} = Config, CreateWhenNotFound) -> + UpdateFun = fun(Chain = #chain{authenticators = Authenticators}) -> + case lists:keytake(AuthenticatorID, 1, Authenticators) of + false -> + case CreateWhenNotFound of + true -> + case lists:keymember(NewName, 2, Authenticators) of + true -> + {error, name_has_be_used}; + false -> + case do_create_authenticator(ChainID, AuthenticatorID, Config) of + {ok, Authenticator} -> + NAuthenticators = Authenticators ++ [{AuthenticatorID, NewName, Authenticator}], + ok = mnesia:write(?CHAIN_TAB, Chain#chain{authenticators = NAuthenticators}, write), + {ok, serialize_authenticator(Authenticator)}; + {error, Reason} -> + {error, Reason} + end + end; + false -> + {error, {not_found, {authenticator, AuthenticatorID}}} + end; + {value, + {_, _, #authenticator{provider = Provider, + state = #{version := Version} = State} = Authenticator}, + Others} -> + case lists:keymember(NewName, 2, Others) of + true -> + {error, name_has_be_used}; + false -> + case (NewProvider = authenticator_provider(Config)) =:= Provider of + true -> + Unique = <>, + case Provider:update(Config#{'_unique' => Unique}, State) of + {ok, NewState} -> + NewAuthenticator = Authenticator#authenticator{name = NewName, + config = Config, + state = switch_version(NewState)}, + NewAuthenticators = replace_authenticator(AuthenticatorID, NewAuthenticator, Authenticators), + ok = mnesia:write(?CHAIN_TAB, Chain#chain{authenticators = NewAuthenticators}, write), + {ok, serialize_authenticator(NewAuthenticator)}; + {error, Reason} -> + {error, Reason} + end; + false -> + Unique = <>, + case NewProvider:create(Config#{'_unique' => Unique}) of + {ok, NewState} -> + NewAuthenticator = Authenticator#authenticator{name = NewName, + provider = NewProvider, + config = Config, + state = switch_version(NewState)}, + NewAuthenticators = replace_authenticator(AuthenticatorID, NewAuthenticator, Authenticators), + ok = mnesia:write(?CHAIN_TAB, Chain#chain{authenticators = NewAuthenticators}, write), + _ = Provider:destroy(State), + {ok, serialize_authenticator(NewAuthenticator)}; + {error, Reason} -> + {error, Reason} + end + end + end + end + end, + update_chain(ChainID, UpdateFun). + +lookup_authenticator(ChainID, AuthenticatorID) -> + case mnesia:dirty_read(?CHAIN_TAB, ChainID) of + [] -> + {error, {not_found, {chain, ChainID}}}; + [#chain{authenticators = Authenticators}] -> + case lists:keyfind(AuthenticatorID, 1, Authenticators) of + false -> + {error, {not_found, {authenticator, AuthenticatorID}}}; + {_, _, Authenticator} -> + {ok, serialize_authenticator(Authenticator)} + end + end. + +list_authenticators(ChainID) -> + case mnesia:dirty_read(?CHAIN_TAB, ChainID) of + [] -> + {error, {not_found, {chain, ChainID}}}; + [#chain{authenticators = Authenticators}] -> + {ok, serialize_authenticators(Authenticators)} + end. + +move_authenticator_to_the_nth(ChainID, AuthenticatorID, N) -> + UpdateFun = fun(Chain = #chain{authenticators = Authenticators}) -> + case move_authenticator_to_the_nth_(AuthenticatorID, Authenticators, N) of + {ok, NAuthenticators} -> + NChain = Chain#chain{authenticators = NAuthenticators}, + mnesia:write(?CHAIN_TAB, NChain, write); + {error, Reason} -> + {error, Reason} + end + end, + update_chain(ChainID, UpdateFun). + +import_users(ChainID, AuthenticatorID, Filename) -> + call_authenticator(ChainID, AuthenticatorID, import_users, [Filename]). + +add_user(ChainID, AuthenticatorID, UserInfo) -> + call_authenticator(ChainID, AuthenticatorID, add_user, [UserInfo]). + +delete_user(ChainID, AuthenticatorID, UserID) -> + call_authenticator(ChainID, AuthenticatorID, delete_user, [UserID]). + +update_user(ChainID, AuthenticatorID, UserID, NewUserInfo) -> + call_authenticator(ChainID, AuthenticatorID, update_user, [UserID, NewUserInfo]). + +lookup_user(ChainID, AuthenticatorID, UserID) -> + call_authenticator(ChainID, AuthenticatorID, lookup_user, [UserID]). + +list_users(ChainID, AuthenticatorID) -> + call_authenticator(ChainID, AuthenticatorID, list_users, []). + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +authenticator_provider(#{mechanism := 'password-based', server_type := 'built-in-database'}) -> + emqx_authn_mnesia; +authenticator_provider(#{mechanism := 'password-based', server_type := 'mysql'}) -> + emqx_authn_mysql; +authenticator_provider(#{mechanism := 'password-based', server_type := 'pgsql'}) -> + emqx_authn_pgsql; +authenticator_provider(#{mechanism := 'password-based', server_type := 'mongodb'}) -> + emqx_authn_mongodb; +authenticator_provider(#{mechanism := 'password-based', server_type := 'http-server'}) -> + emqx_authn_http; +authenticator_provider(#{mechanism := jwt}) -> + emqx_authn_jwt; +authenticator_provider(#{mechanism := scram, server_type := 'built-in-database'}) -> + emqx_enhanced_authn_scram_mnesia. + +gen_id(AlreadyExist) -> + ID = list_to_binary(emqx_rule_id:gen()), + case AlreadyExist(ID) of + true -> gen_id(AlreadyExist); + false -> ID + end. + +switch_version(State = #{version := ?VER_1}) -> + State#{version := ?VER_2}; +switch_version(State = #{version := ?VER_2}) -> + State#{version := ?VER_1}; +switch_version(State) -> + State#{version => ?VER_1}. + +do_create_authenticator(ChainID, AuthenticatorID, #{name := Name} = Config) -> + Provider = authenticator_provider(Config), + Unique = <>, + case Provider:create(Config#{'_unique' => Unique}) of + {ok, State} -> + Authenticator = #authenticator{id = AuthenticatorID, + name = Name, + provider = Provider, + config = Config, + state = switch_version(State)}, + {ok, Authenticator}; + {error, Reason} -> + {error, Reason} + end. + +do_delete_authenticator(#authenticator{provider = Provider, state = State}) -> + _ = Provider:destroy(State), + ok. + +replace_authenticator(ID, #authenticator{name = Name} = Authenticator, Authenticators) -> + lists:keyreplace(ID, 1, Authenticators, {ID, Name, Authenticator}). + +move_authenticator_to_the_nth_(AuthenticatorID, Authenticators, N) + when N =< length(Authenticators) andalso N > 0 -> + move_authenticator_to_the_nth_(AuthenticatorID, Authenticators, N, []); +move_authenticator_to_the_nth_(_, _, _) -> + {error, out_of_range}. + +move_authenticator_to_the_nth_(AuthenticatorID, [], _, _) -> + {error, {not_found, {authenticator, AuthenticatorID}}}; +move_authenticator_to_the_nth_(AuthenticatorID, [{AuthenticatorID, _, _} = Authenticator | More], N, Passed) + when N =< length(Passed) -> + {L1, L2} = lists:split(N - 1, lists:reverse(Passed)), + {ok, L1 ++ [Authenticator] ++ L2 ++ More}; +move_authenticator_to_the_nth_(AuthenticatorID, [{AuthenticatorID, _, _} = Authenticator | More], N, Passed) -> + {L1, L2} = lists:split(N - length(Passed) - 1, More), + {ok, lists:reverse(Passed) ++ L1 ++ [Authenticator] ++ L2}; +move_authenticator_to_the_nth_(AuthenticatorID, [Authenticator | More], N, Passed) -> + move_authenticator_to_the_nth_(AuthenticatorID, More, N, [Authenticator | Passed]). + +update_chain(ChainID, UpdateFun) -> + trans( + fun() -> + case mnesia:read(?CHAIN_TAB, ChainID, write) of + [] -> + {error, {not_found, {chain, ChainID}}}; + [Chain] -> + UpdateFun(Chain) + end + end). + +call_authenticator(ChainID, AuthenticatorID, Func, Args) -> + case mnesia:dirty_read(?CHAIN_TAB, ChainID) of + [] -> + {error, {not_found, {chain, ChainID}}}; + [#chain{authenticators = Authenticators}] -> + case lists:keyfind(AuthenticatorID, 1, Authenticators) of + false -> + {error, {not_found, {authenticator, AuthenticatorID}}}; + {_, _, #authenticator{provider = Provider, state = State}} -> + case erlang:function_exported(Provider, Func, length(Args) + 1) of + true -> + erlang:apply(Provider, Func, Args ++ [State]); + false -> + {error, unsupported_feature} + end + end + end. + +serialize_chain(#chain{id = ID, + authenticators = Authenticators, + created_at = CreatedAt}) -> + #{id => ID, + authenticators => serialize_authenticators(Authenticators), + created_at => CreatedAt}. + +serialize_authenticators(Authenticators) -> + [serialize_authenticator(Authenticator) || {_, _, Authenticator} <- Authenticators]. + +serialize_authenticator(#authenticator{id = ID, + config = Config}) -> + Config#{id => ID}. + +trans(Fun) -> + trans(Fun, []). + +trans(Fun, Args) -> + case ekka_mnesia:transaction(?AUTH_SHARD, Fun, Args) of + {atomic, Res} -> Res; + {aborted, Reason} -> {error, Reason} + end. diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl new file mode 100644 index 000000000..c0071114a --- /dev/null +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -0,0 +1,1314 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_api). + +-behavior(minirest_api). + +-include("emqx_authn.hrl"). + +-export([ api_spec/0 + , authentication/2 + , authenticators/2 + , authenticators2/2 + , position/2 + , import_users/2 + , users/2 + , users2/2 + ]). + +-define(EXAMPLE_1, #{name => <<"example 1">>, + mechanism => <<"password-based">>, + server_type => <<"built-in-example">>, + user_id_type => <<"username">>, + password_hash_algorithm => #{ + name => <<"sha256">> + }}). + +-define(EXAMPLE_2, #{name => <<"example 2">>, + mechanism => <<"password-based">>, + server_type => <<"http-server">>, + method => <<"post">>, + url => <<"http://localhost:80/login">>, + headers => #{ + <<"content-type">> => <<"application/json">> + }, + form_data => #{ + <<"username">> => <<"${mqtt-username}">>, + <<"password">> => <<"${mqtt-password}">> + }}). + +-define(EXAMPLE_3, #{name => <<"example 3">>, + mechanism => <<"jwt">>, + use_jwks => false, + algorithm => <<"hmac-based">>, + secret => <<"mysecret">>, + secret_base64_encoded => false, + verify_claims => #{ + <<"username">> => <<"${mqtt-username}">> + }}). + +-define(EXAMPLE_4, #{name => <<"example 4">>, + mechanism => <<"password-based">>, + server_type => <<"mongodb">>, + server => <<"127.0.0.1:27017">>, + database => example, + collection => users, + selector => #{ + username => <<"${mqtt-username}">> + }, + password_hash_field => <<"password_hash">>, + salt_field => <<"salt">>, + password_hash_algorithm => <<"sha256">>, + salt_position => <<"prefix">> + }). + +-define(ERR_RESPONSE(Desc), #{description => Desc, + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"error">>), + examples => #{ + example1 => #{ + summary => <<"Not Found">>, + value => #{code => <<"NOT_FOUND">>, message => <<"Authenticator '67e4c9d3' does not exist">>} + }, + example2 => #{ + summary => <<"Conflict">>, + value => #{code => <<"ALREADY_EXISTS">>, message => <<"Name has be used">>} + }, + example3 => #{ + summary => <<"Bad Request 1">>, + value => #{code => <<"OUT_OF_RANGE">>, message => <<"Out of range">>} + } + }}}}). + +api_spec() -> + {[ authentication_api() + , authenticators_api() + , authenticators_api2() + , position_api() + , import_users_api() + , users_api() + , users2_api() + ], definitions()}. + +authentication_api() -> + Metadata = #{ + post => #{ + description => "Enable or disbale authentication", + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => #{ + type => object, + required => [enable], + properties => #{ + enable => #{ + type => boolean, + example => true + } + } + } + } + } + }, + responses => #{ + <<"204">> => #{ + description => <<"No Content">> + }, + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>) + } + } + }, + {"/authentication", Metadata, authentication}. + +authenticators_api() -> + Metadata = #{ + post => #{ + description => "Create authenticator", + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"authenticator">>), + examples => #{ + default => #{ + summary => <<"Default">>, + value => emqx_json:encode(?EXAMPLE_1) + }, + http => #{ + summary => <<"Authentication provided by HTTP Server">>, + value => emqx_json:encode(?EXAMPLE_2) + }, + jwt => #{ + summary => <<"JWT Authentication">>, + value => emqx_json:encode(?EXAMPLE_3) + }, + mongodb => #{ + summary => <<"Authentication with MongoDB">>, + value => emqx_json:encode(?EXAMPLE_4) + } + } + } + } + }, + responses => #{ + <<"201">> => #{ + description => <<"Created">>, + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"returned_authenticator">>), + examples => #{ + example1 => #{ + summary => <<"Example 1">>, + value => emqx_json:encode(maps:put(id, <<"example 1">>, ?EXAMPLE_1)) + }, + example2 => #{ + summary => <<"Example 2">>, + value => emqx_json:encode(maps:put(id, <<"example 2">>, ?EXAMPLE_2)) + }, + example3 => #{ + summary => <<"Example 3">>, + value => emqx_json:encode(maps:put(id, <<"example 3">>, ?EXAMPLE_3)) + }, + example4 => #{ + summary => <<"Example 4">>, + value => emqx_json:encode(maps:put(id, <<"example 4">>, ?EXAMPLE_4)) + } + } + } + } + }, + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), + <<"409">> => ?ERR_RESPONSE(<<"Conflict">>) + } + }, + get => #{ + description => "List authenticators", + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => #{ + type => array, + items => minirest:ref(<<"returned_authenticator">>) + }, + examples => #{ + example1 => #{ + summary => <<"Example 1">>, + value => emqx_json:encode([ maps:put(id, <<"example 1">>, ?EXAMPLE_1) + , maps:put(id, <<"example 2">>, ?EXAMPLE_2) + , maps:put(id, <<"example 3">>, ?EXAMPLE_3) + , maps:put(id, <<"example 4">>, ?EXAMPLE_4) + ]) + } + } + } + } + } + } + } + }, + {"/authentication/authenticators", Metadata, authenticators}. + +authenticators_api2() -> + Metadata = #{ + get => #{ + description => "Get authenicator by id", + parameters => [ + #{ + name => id, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"returned_authenticator">>), + examples => #{ + example1 => #{ + summary => <<"Example 1">>, + value => emqx_json:encode(maps:put(id, <<"example 1">>, ?EXAMPLE_1)) + }, + example2 => #{ + summary => <<"Example 2">>, + value => emqx_json:encode(maps:put(id, <<"example 2">>, ?EXAMPLE_2)) + }, + example3 => #{ + summary => <<"Example 3">>, + value => emqx_json:encode(maps:put(id, <<"example 3">>, ?EXAMPLE_3)) + }, + example4 => #{ + summary => <<"Example 4">>, + value => emqx_json:encode(maps:put(id, <<"example 4">>, ?EXAMPLE_4)) + } + } + } + } + }, + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }, + put => #{ + description => "Update authenticator", + parameters => [ + #{ + name => id, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => #{ + oneOf => [ minirest:ref(<<"password_based">>) + , minirest:ref(<<"jwt">>) + , minirest:ref(<<"scram">>) + ] + }, + examples => #{ + example1 => #{ + summary => <<"Example 1">>, + value => emqx_json:encode(?EXAMPLE_1) + }, + example2 => #{ + summary => <<"Example 2">>, + value => emqx_json:encode(?EXAMPLE_2) + } + } + } + } + }, + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"returned_authenticator">>), + examples => #{ + example1 => #{ + summary => <<"Example 1">>, + value => emqx_json:encode(maps:put(id, <<"example 1">>, ?EXAMPLE_1)) + }, + example2 => #{ + summary => <<"Example 2">>, + value => emqx_json:encode(maps:put(id, <<"example 2">>, ?EXAMPLE_2)) + }, + example3 => #{ + summary => <<"Example 3">>, + value => emqx_json:encode(maps:put(id, <<"example 3">>, ?EXAMPLE_3)) + }, + example4 => #{ + summary => <<"Example 4">>, + value => emqx_json:encode(maps:put(id, <<"example 4">>, ?EXAMPLE_4)) + } + } + } + } + }, + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>), + <<"409">> => ?ERR_RESPONSE(<<"Conflict">>) + } + }, + delete => #{ + description => "Delete authenticator", + parameters => [ + #{ + name => id, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"204">> => #{ + description => <<"No Content">> + }, + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + } + }, + {"/authentication/authenticators/:id", Metadata, authenticators2}. + +position_api() -> + Metadata = #{ + post => #{ + description => "Change the order of authenticators", + parameters => [ + #{ + name => id, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => #{ + type => object, + required => [position], + properties => #{ + position => #{ + type => integer, + example => 1 + } + } + } + } + } + }, + responses => #{ + <<"204">> => #{ + description => <<"No Content">> + }, + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + } + }, + {"/authentication/authenticators/:id/position", Metadata, position}. + +import_users_api() -> + Metadata = #{ + post => #{ + description => "Import users from json/csv file", + parameters => [ + #{ + name => id, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => #{ + type => object, + required => [filename], + properties => #{ + filename => #{ + type => string + } + } + } + } + } + }, + responses => #{ + <<"204">> => #{ + description => <<"No Content">> + }, + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + } + }, + {"/authentication/authenticators/:id/import-users", Metadata, import_users}. + +users_api() -> + Metadata = #{ + post => #{ + description => "Add user", + parameters => [ + #{ + name => id, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => #{ + type => object, + required => [user_id, password], + properties => #{ + user_id => #{ + type => string + }, + password => #{ + type => string + } + } + } + } + } + }, + responses => #{ + <<"201">> => #{ + description => <<"Created">>, + content => #{ + 'application/json' => #{ + schema => #{ + type => object, + required => [user_id], + properties => #{ + user_id => #{ + type => string + } + } + } + } + } + }, + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }, + get => #{ + description => "List users", + parameters => [ + #{ + name => id, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => #{ + type => array, + items => #{ + type => object, + required => [user_id], + properties => #{ + user_id => #{ + type => string + } + } + } + } + } + } + }, + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + } + }, + {"/authentication/authenticators/:id/users", Metadata, users}. + +users2_api() -> + Metadata = #{ + patch => #{ + description => "Update user", + parameters => [ + #{ + name => id, + in => path, + schema => #{ + type => string + }, + required => true + }, + #{ + name => user_id, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => #{ + type => object, + required => [password], + properties => #{ + password => #{ + type => string + } + } + } + } + } + }, + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => #{ + type => array, + items => #{ + type => object, + properties => #{ + user_id => #{ + type => string + } + } + } + } + } + } + }, + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }, + get => #{ + description => "Get user info", + parameters => [ + #{ + name => id, + in => path, + schema => #{ + type => string + }, + required => true + }, + #{ + name => user_id, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => #{ + type => array, + items => #{ + type => object, + properties => #{ + user_id => #{ + type => string + } + } + } + } + } + } + }, + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }, + delete => #{ + description => "Delete user", + parameters => [ + #{ + name => id, + in => path, + schema => #{ + type => string + }, + required => true + }, + #{ + name => user_id, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"204">> => #{ + description => <<"No Content">> + }, + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + } + }, + {"/authentication/authenticators/:id/users/:user_id", Metadata, users2}. + +definitions() -> + AuthenticatorDef = #{ + oneOf => [ minirest:ref(<<"password_based">>) + , minirest:ref(<<"jwt">>) + , minirest:ref(<<"scram">>) + ] + }, + + ReturnedAuthenticatorDef = #{ + allOf => [ + #{ + type => object, + properties => #{ + id => #{ + type => string + } + } + }, + #{ + oneOf => [ minirest:ref(<<"password_based">>) + , minirest:ref(<<"jwt">>) + , minirest:ref(<<"scram">>) + ] + } + ] + }, + + PasswordBasedDef = #{ + allOf => [ + #{ + type => object, + required => [name, mechanism], + properties => #{ + name => #{ + type => string, + example => "exmaple" + }, + mechanism => #{ + type => string, + enum => [<<"password-based">>], + example => <<"password-based">> + } + } + }, + #{ + oneOf => [ minirest:ref(<<"password_based_built_in_database">>) + , minirest:ref(<<"password_based_mysql">>) + , minirest:ref(<<"password_based_pgsql">>) + , minirest:ref(<<"password_based_mongodb">>) + , minirest:ref(<<"password_based_http_server">>) + ] + } + ] + }, + + JWTDef = #{ + type => object, + required => [name, mechanism], + properties => #{ + name => #{ + type => string, + example => "exmaple" + }, + mechanism => #{ + type => string, + enum => [<<"jwt">>], + example => <<"jwt">> + }, + use_jwks => #{ + type => boolean, + default => false, + example => false + }, + algorithm => #{ + type => string, + enum => [<<"hmac-based">>, <<"public-key">>], + default => <<"hmac-based">>, + example => <<"hmac-based">> + }, + secret => #{ + type => string + }, + secret_base64_encoded => #{ + type => boolean, + default => false + }, + certificate => #{ + type => string + }, + verify_claims => #{ + type => object, + additionalProperties => #{ + type => string + } + }, + ssl => minirest:ref(<<"ssl">>) + } + }, + + SCRAMDef = #{ + type => object, + required => [name, mechanism, server_type], + properties => #{ + name => #{ + type => string, + example => "exmaple" + }, + mechanism => #{ + type => string, + enum => [<<"scram">>], + example => <<"scram">> + }, + server_type => #{ + type => string, + enum => [<<"built-in-database">>], + default => <<"built-in-database">> + }, + algorithm => #{ + type => string, + enum => [<<"sha256">>, <<"sha512">>], + default => <<"sha256">> + }, + iteration_count => #{ + type => integer, + default => 4096 + } + } + }, + + PasswordBasedBuiltInDatabaseDef = #{ + type => object, + required => [server_type], + properties => #{ + server_type => #{ + type => string, + enum => [<<"built-in-database">>], + example => <<"built-in-database">> + }, + user_id_type => #{ + type => string, + enum => [<<"username">>, <<"clientid">>], + default => <<"username">>, + example => <<"username">> + }, + password_hash_algorithm => minirest:ref(<<"password_hash_algorithm">>) + } + }, + + PasswordBasedMySQLDef = #{ + type => object, + required => [ server_type + , server + , database + , username + , password + , query], + properties => #{ + server_type => #{ + type => string, + enum => [<<"mysql">>], + example => <<"mysql">> + }, + server => #{ + type => string, + example => <<"localhost:3306">> + }, + database => #{ + type => string + }, + pool_size => #{ + type => integer, + default => 8 + }, + username => #{ + type => string + }, + password => #{ + type => string + }, + auto_reconnect => #{ + type => boolean, + default => true + }, + ssl => minirest:ref(<<"ssl">>), + password_hash_algorithm => #{ + type => string, + enum => [<<"plain">>, <<"md5">>, <<"sha">>, <<"sha256">>, <<"sha512">>, <<"bcrypt">>], + default => <<"sha256">> + }, + salt_position => #{ + type => string, + enum => [<<"prefix">>, <<"suffix">>], + default => <<"prefix">> + }, + query => #{ + type => string, + example => <<"SELECT password_hash FROM mqtt_user WHERE username = ${mqtt-username}">> + }, + query_timeout => #{ + type => integer, + description => <<"Query timeout, Unit: Milliseconds">>, + default => 5000 + } + } + }, + + PasswordBasedPgSQLDef = #{ + type => object, + required => [ server_type + , server + , database + , username + , password + , query], + properties => #{ + server_type => #{ + type => string, + enum => [<<"pgsql">>], + example => <<"pgsql">> + }, + server => #{ + type => string, + example => <<"localhost:5432">> + }, + database => #{ + type => string + }, + pool_size => #{ + type => integer, + default => 8 + }, + username => #{ + type => string + }, + password => #{ + type => string + }, + auto_reconnect => #{ + type => boolean, + default => true + }, + password_hash_algorithm => #{ + type => string, + enum => [<<"plain">>, <<"md5">>, <<"sha">>, <<"sha256">>, <<"sha512">>, <<"bcrypt">>], + default => <<"sha256">> + }, + salt_position => #{ + type => string, + enum => [<<"prefix">>, <<"suffix">>], + default => <<"prefix">> + }, + query => #{ + type => string, + example => <<"SELECT password_hash FROM mqtt_user WHERE username = ${mqtt-username}">> + } + } + }, + + PasswordBasedMongoDBDef = #{ + type => object, + required => [ server_type + , server + , servers + , replica_set_name + , database + , username + , password + , collection + , selector + , password_hash_field + ], + properties => #{ + server_type => #{ + type => string, + enum => [<<"mongodb">>], + example => [<<"mongodb">>] + }, + server => #{ + description => <<"Mutually exclusive with the 'servers' field, only valid in standalone mode">>, + type => string, + example => <<"127.0.0.1:27017">> + }, + servers => #{ + description => <<"Mutually exclusive with the 'server' field, only valid in replica set and sharded mode">>, + type => array, + items => #{ + type => string + }, + example => [<<"127.0.0.1:27017">>] + }, + replica_set_name => #{ + description => <<"Only valid in replica set mode">>, + type => string + }, + database => #{ + type => string + }, + username => #{ + type => string + }, + password => #{ + type => string + }, + auth_source => #{ + type => string, + default => <<"admin">> + }, + pool_size => #{ + type => integer, + default => 8 + }, + collection => #{ + type => string + }, + selector => #{ + type => object, + additionalProperties => true, + example => <<"{\"username\":\"${mqtt-username}\"}">> + }, + password_hash_field => #{ + type => string, + example => <<"password_hash">> + }, + salt_field => #{ + type => string, + example => <<"salt">> + }, + password_hash_algorithm => #{ + type => string, + enum => [<<"plain">>, <<"md5">>, <<"sha">>, <<"sha256">>, <<"sha512">>, <<"bcrypt">>], + default => <<"sha256">>, + example => <<"sha256">> + }, + salt_position => #{ + description => <<"Only valid when the 'salt_field' field is specified">>, + type => string, + enum => [<<"prefix">>, <<"suffix">>], + default => <<"prefix">>, + example => <<"prefix">> + } + } + }, + + PasswordBasedHTTPServerDef = #{ + type => object, + required => [ server_type + , url + , form_data + ], + properties => #{ + server_type => #{ + type => string, + enum => [<<"http-server">>], + example => <<"http-server">> + }, + method => #{ + type => string, + enum => [<<"get">>, <<"post">>], + default => <<"post">> + }, + url => #{ + type => string, + example => <<"http://localhost:80/login">> + }, + headers => #{ + type => object, + additionalProperties => #{ + type => string + } + }, + form_data => #{ + type => string + }, + connect_timeout => #{ + type => integer, + default => 5000 + }, + max_retries => #{ + type => integer, + default => 5 + }, + retry_interval => #{ + type => integer, + default => 1000 + }, + request_timout => #{ + type => integer, + default => 5000 + }, + pool_size => #{ + type => integer, + default => 8 + }, + enable_pipelining => #{ + type => boolean, + default => true + } + } + }, + + PasswordHashAlgorithmDef = #{ + type => object, + required => [name], + properties => #{ + name => #{ + type => string, + enum => [<<"plain">>, <<"md5">>, <<"sha">>, <<"sha256">>, <<"sha512">>, <<"bcrypt">>], + default => <<"sha256">> + }, + salt_rounds => #{ + type => integer, + default => 10 + } + } + }, + + SSLDef = #{ + type => object, + properties => #{ + enable => #{ + type => boolean, + default => false + }, + certfile => #{ + type => string + }, + keyfile => #{ + type => string + }, + cacertfile => #{ + type => string + }, + verify => #{ + type => boolean, + default => true + }, + server_name_indication => #{ + type => object, + properties => #{ + enable => #{ + type => boolean, + default => false + }, + hostname => #{ + type => string + } + } + } + } + }, + + ErrorDef = #{ + type => object, + properties => #{ + code => #{ + type => string, + enum => [<<"NOT_FOUND">>], + example => <<"NOT_FOUND">> + }, + message => #{ + type => string + } + } + }, + + [ #{<<"authenticator">> => AuthenticatorDef} + , #{<<"returned_authenticator">> => ReturnedAuthenticatorDef} + , #{<<"password_based">> => PasswordBasedDef} + , #{<<"jwt">> => JWTDef} + , #{<<"scram">> => SCRAMDef} + , #{<<"password_based_built_in_database">> => PasswordBasedBuiltInDatabaseDef} + , #{<<"password_based_mysql">> => PasswordBasedMySQLDef} + , #{<<"password_based_pgsql">> => PasswordBasedPgSQLDef} + , #{<<"password_based_mongodb">> => PasswordBasedMongoDBDef} + , #{<<"password_based_http_server">> => PasswordBasedHTTPServerDef} + , #{<<"password_hash_algorithm">> => PasswordHashAlgorithmDef} + , #{<<"ssl">> => SSLDef} + , #{<<"error">> => ErrorDef} + ]. + +authentication(post, Request) -> + {ok, Body, _} = cowboy_req:read_body(Request), + case emqx_json:decode(Body, [return_maps]) of + #{<<"enable">> := true} -> + ok = emqx_authn:enable(), + {204}; + #{<<"enable">> := false} -> + ok = emqx_authn:disable(), + {204}; + #{<<"enable">> := _} -> + serialize_error({invalid_parameter, enable}); + _ -> + serialize_error({missing_parameter, enable}) + end. + +authenticators(post, Request) -> + {ok, Body, _} = cowboy_req:read_body(Request), + AuthenticatorConfig = emqx_json:decode(Body, [return_maps]), + Config = #{<<"emqx_authn">> => #{ + <<"authenticators">> => [AuthenticatorConfig] + }}, + NConfig = hocon_schema:check_plain(emqx_authn_schema, Config, + #{nullable => true}), + #{emqx_authn := #{authenticators := [NAuthenticatorConfig]}} = emqx_map_lib:unsafe_atom_key_map(NConfig), + case emqx_authn:create_authenticator(?CHAIN, NAuthenticatorConfig) of + {ok, Authenticator2} -> + {201, Authenticator2}; + {error, Reason} -> + serialize_error(Reason) + end; +authenticators(get, _Request) -> + {ok, Authenticators} = emqx_authn:list_authenticators(?CHAIN), + {200, Authenticators}. + +authenticators2(get, Request) -> + AuthenticatorID = cowboy_req:binding(id, Request), + case emqx_authn:lookup_authenticator(?CHAIN, AuthenticatorID) of + {ok, Authenticator} -> + {200, Authenticator}; + {error, Reason} -> + serialize_error(Reason) + end; +authenticators2(put, Request) -> + AuthenticatorID = cowboy_req:binding(id, Request), + {ok, Body, _} = cowboy_req:read_body(Request), + AuthenticatorConfig = emqx_json:decode(Body, [return_maps]), + Config = #{<<"emqx_authn">> => #{ + <<"authenticators">> => [AuthenticatorConfig] + }}, + NConfig = hocon_schema:check_plain(emqx_authn_schema, Config, + #{nullable => true}), + #{emqx_authn := #{authenticators := [NAuthenticatorConfig]}} = emqx_map_lib:unsafe_atom_key_map(NConfig), + case emqx_authn:update_or_create_authenticator(?CHAIN, AuthenticatorID, NAuthenticatorConfig) of + {ok, Authenticator} -> + {200, Authenticator}; + {error, Reason} -> + serialize_error(Reason) + end; +authenticators2(delete, Request) -> + AuthenticatorID = cowboy_req:binding(id, Request), + case emqx_authn:delete_authenticator(?CHAIN, AuthenticatorID) of + ok -> + {204}; + {error, Reason} -> + serialize_error(Reason) + end. + +position(post, Request) -> + AuthenticatorID = cowboy_req:binding(id, Request), + {ok, Body, _} = cowboy_req:read_body(Request), + NBody = emqx_json:decode(Body, [return_maps]), + Config = hocon_schema:check_plain(emqx_authn_other_schema, #{<<"position">> => NBody}, + #{nullable => true}, ["position"]), + #{position := #{position := Position}} = emqx_map_lib:unsafe_atom_key_map(Config), + case emqx_authn:move_authenticator_to_the_nth(?CHAIN, AuthenticatorID, Position) of + ok -> + {204}; + {error, Reason} -> + serialize_error(Reason) + end. + +import_users(post, Request) -> + AuthenticatorID = cowboy_req:binding(id, Request), + {ok, Body, _} = cowboy_req:read_body(Request), + NBody = emqx_json:decode(Body, [return_maps]), + Config = hocon_schema:check_plain(emqx_authn_other_schema, #{<<"filename">> => NBody}, + #{nullable => true}, ["filename"]), + #{filename := #{filename := Filename}} = emqx_map_lib:unsafe_atom_key_map(Config), + case emqx_authn:import_users(?CHAIN, AuthenticatorID, Filename) of + ok -> + {204}; + {error, Reason} -> + serialize_error(Reason) + end. + +users(post, Request) -> + AuthenticatorID = cowboy_req:binding(id, Request), + {ok, Body, _} = cowboy_req:read_body(Request), + NBody = emqx_json:decode(Body, [return_maps]), + Config = hocon_schema:check_plain(emqx_authn_other_schema, #{<<"user_info">> => NBody}, + #{nullable => true}, ["user_info"]), + #{user_info := UserInfo} = emqx_map_lib:unsafe_atom_key_map(Config), + case emqx_authn:add_user(?CHAIN, AuthenticatorID, UserInfo) of + {ok, User} -> + {201, User}; + {error, Reason} -> + serialize_error(Reason) + end; +users(get, Request) -> + AuthenticatorID = cowboy_req:binding(id, Request), + case emqx_authn:list_users(?CHAIN, AuthenticatorID) of + {ok, Users} -> + {200, Users}; + {error, Reason} -> + serialize_error(Reason) + end. + +users2(patch, Request) -> + AuthenticatorID = cowboy_req:binding(id, Request), + UserID = cowboy_req:binding(user_id, Request), + {ok, Body, _} = cowboy_req:read_body(Request), + NBody = emqx_json:decode(Body, [return_maps]), + Config = hocon_schema:check_plain(emqx_authn_other_schema, #{<<"new_user_info">> => NBody}, + #{nullable => true}, ["new_user_info"]), + #{new_user_info := NewUserInfo} = emqx_map_lib:unsafe_atom_key_map(Config), + case emqx_authn:update_user(?CHAIN, AuthenticatorID, UserID, NewUserInfo) of + {ok, User} -> + {200, User}; + {error, Reason} -> + serialize_error(Reason) + end; +users2(get, Request) -> + AuthenticatorID = cowboy_req:binding(id, Request), + UserID = cowboy_req:binding(user_id, Request), + case emqx_authn:lookup_user(?CHAIN, AuthenticatorID, UserID) of + {ok, User} -> + {200, User}; + {error, Reason} -> + serialize_error(Reason) + end; +users2(delete, Request) -> + AuthenticatorID = cowboy_req:binding(id, Request), + UserID = cowboy_req:binding(user_id, Request), + case emqx_authn:delete_user(?CHAIN, AuthenticatorID, UserID) of + ok -> + {204}; + {error, Reason} -> + serialize_error(Reason) + end. + +serialize_error({not_found, {authenticator, ID}}) -> + {404, #{code => <<"NOT_FOUND">>, + message => list_to_binary(io_lib:format("Authenticator '~s' does not exist", [ID]))}}; +serialize_error(name_has_be_used) -> + {409, #{code => <<"ALREADY_EXISTS">>, + message => <<"Name has be used">>}}; +serialize_error(out_of_range) -> + {400, #{code => <<"OUT_OF_RANGE">>, + message => <<"Out of range">>}}; +serialize_error({missing_parameter, Name}) -> + {400, #{code => <<"MISSING_PARAMETER">>, + message => list_to_binary( + io_lib:format("The input parameter '~p' that is mandatory for processing this request is not supplied", [Name]) + )}}; +serialize_error({invalid_parameter, Name}) -> + {400, #{code => <<"INVALID_PARAMETER">>, + message => list_to_binary( + io_lib:format("The value of input parameter '~p' is invalid", [Name]) + )}}; +serialize_error(Reason) -> + {400, #{code => <<"BAD_REQUEST">>, + message => list_to_binary(io_lib:format("Todo: ~p", [Reason]))}}. \ No newline at end of file diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl new file mode 100644 index 000000000..b55727a00 --- /dev/null +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -0,0 +1,57 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_app). + +-include("emqx_authn.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-behaviour(application). + +%% Application callbacks +-export([ start/2 + , stop/1 + ]). + +start(_StartType, _StartArgs) -> + {ok, Sup} = emqx_authn_sup:start_link(), + ok = ekka_rlog:wait_for_shards([?AUTH_SHARD], infinity), + initialize(), + {ok, Sup}. + +stop(_State) -> + ok. + +initialize() -> + AuthNConfig = emqx_config:get([emqx_authn], #{enable => false, + authenticators => []}), + initialize(AuthNConfig). + +initialize(#{enable := Enable, authenticators := AuthenticatorsConfig}) -> + {ok, _} = emqx_authn:create_chain(#{id => ?CHAIN}), + initialize_authenticators(AuthenticatorsConfig), + Enable =:= true andalso emqx_authn:enable(), + ok. + +initialize_authenticators([]) -> + ok; +initialize_authenticators([#{name := Name} = AuthenticatorConfig | More]) -> + case emqx_authn:create_authenticator(?CHAIN, AuthenticatorConfig) of + {ok, _} -> + initialize_authenticators(More); + {error, Reason} -> + ?LOG(error, "Failed to create authenticator '~s': ~p", [Name, Reason]) + end. diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl new file mode 100644 index 000000000..694d5c7ca --- /dev/null +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -0,0 +1,61 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_schema). + +-include("emqx_authn.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-behaviour(hocon_schema). + +-export([ structs/0 + , fields/1 + ]). + +-export([ authenticator_name/1 + ]). + +structs() -> [ "emqx_authn" ]. + +fields("emqx_authn") -> + [ {enable, fun enable/1} + , {authenticators, fun authenticators/1} + ]. + +authenticator_name(type) -> binary(); +authenticator_name(nullable) -> false; +authenticator_name(_) -> undefined. + +enable(type) -> boolean(); +enable(default) -> false; +enable(_) -> undefined. + +authenticators(type) -> + hoconsc:array({union, [ hoconsc:ref(emqx_authn_mnesia, config) + , hoconsc:ref(emqx_authn_mysql, config) + , hoconsc:ref(emqx_authn_pgsql, config) + , hoconsc:ref(emqx_authn_mongodb, standalone) + , hoconsc:ref(emqx_authn_mongodb, 'replica-set') + , hoconsc:ref(emqx_authn_mongodb, sharded) + , hoconsc:ref(emqx_authn_http, get) + , hoconsc:ref(emqx_authn_http, post) + , hoconsc:ref(emqx_authn_jwt, 'hmac-based') + , hoconsc:ref(emqx_authn_jwt, 'public-key') + , hoconsc:ref(emqx_authn_jwt, 'jwks') + , hoconsc:ref(emqx_enhanced_authn_scram_mnesia, config) + ]}); +authenticators(default) -> []; +authenticators(_) -> undefined. diff --git a/apps/emqx_authentication/src/emqx_authentication_sup.erl b/apps/emqx_authn/src/emqx_authn_sup.erl similarity index 96% rename from apps/emqx_authentication/src/emqx_authentication_sup.erl rename to apps/emqx_authn/src/emqx_authn_sup.erl index 06e12ce6c..bb26af0ad 100644 --- a/apps/emqx_authentication/src/emqx_authentication_sup.erl +++ b/apps/emqx_authn/src/emqx_authn_sup.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_authentication_sup). +-module(emqx_authn_sup). -behaviour(supervisor). diff --git a/apps/emqx_authn/src/emqx_authn_utils.erl b/apps/emqx_authn/src/emqx_authn_utils.erl new file mode 100644 index 000000000..c035278cc --- /dev/null +++ b/apps/emqx_authn/src/emqx_authn_utils.erl @@ -0,0 +1,73 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_utils). + +-export([ replace_placeholders/2 + , replace_placeholder/2 + , gen_salt/0 + , bin/1 + ]). + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +replace_placeholders(PlaceHolders, Data) -> + replace_placeholders(PlaceHolders, Data, []). + +replace_placeholders([], _Credential, Acc) -> + lists:reverse(Acc); +replace_placeholders([Placeholder | More], Credential, Acc) -> + case replace_placeholder(Placeholder, Credential) of + undefined -> + error({cannot_get_variable, Placeholder}); + V -> + replace_placeholders(More, Credential, [convert_to_sql_param(V) | Acc]) + end. + +replace_placeholder(<<"${mqtt-username}">>, Credential) -> + maps:get(username, Credential, undefined); +replace_placeholder(<<"${mqtt-clientid}">>, Credential) -> + maps:get(clientid, Credential, undefined); +replace_placeholder(<<"${mqtt-password}">>, Credential) -> + maps:get(password, Credential, undefined); +replace_placeholder(<<"${ip-address}">>, Credential) -> + maps:get(peerhost, Credential, undefined); +replace_placeholder(<<"${cert-subject}">>, Credential) -> + maps:get(dn, Credential, undefined); +replace_placeholder(<<"${cert-common-name}">>, Credential) -> + maps:get(cn, Credential, undefined); +replace_placeholder(Constant, _) -> + Constant. + + +gen_salt() -> + <> = crypto:strong_rand_bytes(16), + iolist_to_binary(io_lib:format("~32.16.0b", [X])). + +bin(A) when is_atom(A) -> atom_to_binary(A, utf8); +bin(L) when is_list(L) -> list_to_binary(L); +bin(X) -> X. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +convert_to_sql_param(undefined) -> + null; +convert_to_sql_param(V) -> + bin(V). \ No newline at end of file diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl new file mode 100644 index 000000000..56629c568 --- /dev/null +++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl @@ -0,0 +1,243 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_enhanced_authn_scram_mnesia). + +-include("emqx_authn.hrl"). +-include_lib("esasl/include/esasl_scram.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-behaviour(hocon_schema). + +-export([ structs/0 + , fields/1 + ]). + +-export([ create/1 + , update/2 + , authenticate/2 + , destroy/1 + ]). + +-export([ add_user/2 + , delete_user/2 + , update_user/3 + , lookup_user/2 + , list_users/1 + ]). + +-define(TAB, ?MODULE). + +-export([mnesia/1]). + +-boot_mnesia({mnesia, [boot]}). +-copy_mnesia({mnesia, [copy]}). + +-rlog_shard({?AUTH_SHARD, ?TAB}). + +%%------------------------------------------------------------------------------ +%% Mnesia bootstrap +%%------------------------------------------------------------------------------ + +%% @doc Create or replicate tables. +-spec(mnesia(boot | copy) -> ok). +mnesia(boot) -> + ok = ekka_mnesia:create_table(?TAB, [ + {disc_copies, [node()]}, + {record_name, scram_user_credentail}, + {attributes, record_info(fields, scram_user_credentail)}, + {storage_properties, [{ets, [{read_concurrency, true}]}]}]); + +mnesia(copy) -> + ok = ekka_mnesia:copy_table(?TAB, disc_copies). + +%%------------------------------------------------------------------------------ +%% Hocon Schema +%%------------------------------------------------------------------------------ + +structs() -> [config]. + +fields(config) -> + [ {name, fun emqx_authn_schema:authenticator_name/1} + , {mechanism, {enum, [scram]}} + , {server_type, fun server_type/1} + , {algorithm, fun algorithm/1} + , {iteration_count, fun iteration_count/1} + ]. + +server_type(type) -> hoconsc:enum(['built-in-database']); +server_type(default) -> 'built-in-database'; +server_type(_) -> undefined. + +algorithm(type) -> hoconsc:enum([sha256, sha512]); +algorithm(default) -> sha256; +algorithm(_) -> undefined. + +iteration_count(type) -> non_neg_integer(); +iteration_count(default) -> 4096; +iteration_count(_) -> undefined. + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +create(#{ algorithm := Algorithm + , iteration_count := IterationCount + , '_unique' := Unique + }) -> + State = #{user_group => Unique, + algorithm => Algorithm, + iteration_count => IterationCount}, + {ok, State}. + +update(Config, #{user_group := Unique}) -> + create(Config#{'_unique' => Unique}). + +authenticate(#{auth_method := AuthMethod, + auth_data := AuthData, + auth_cache := AuthCache}, State) -> + case ensure_auth_method(AuthMethod, State) of + true -> + case AuthCache of + #{next_step := client_final} -> + check_client_final_message(AuthData, AuthCache, State); + _ -> + check_client_first_message(AuthData, AuthCache, State) + end; + false -> + ignore + end; +authenticate(_Credential, _State) -> + ignore. + +destroy(#{user_group := UserGroup}) -> + trans( + fun() -> + MatchSpec = [{{scram_user_credentail, {UserGroup, '_'}, '_', '_', '_'}, [], ['$_']}], + ok = lists:foreach(fun(UserCredential) -> + mnesia:delete_object(?TAB, UserCredential, write) + end, mnesia:select(?TAB, MatchSpec, write)) + end). + +add_user(#{user_id := UserID, + password := Password}, #{user_group := UserGroup} = State) -> + trans( + fun() -> + case mnesia:read(?TAB, {UserGroup, UserID}, write) of + [] -> + add_user(UserID, Password, State), + {ok, #{user_id => UserID}}; + [_] -> + {error, already_exist} + end + end). + +delete_user(UserID, #{user_group := UserGroup}) -> + trans( + fun() -> + case mnesia:read(?TAB, {UserGroup, UserID}, write) of + [] -> + {error, not_found}; + [_] -> + mnesia:delete(?TAB, {UserGroup, UserID}, write) + end + end). + +update_user(UserID, #{password := Password}, + #{user_group := UserGroup} = State) -> + trans( + fun() -> + case mnesia:read(?TAB, {UserGroup, UserID}, write) of + [] -> + {error, not_found}; + [_] -> + add_user(UserID, Password, State), + {ok, #{user_id => UserID}} + end + end). + +lookup_user(UserID, #{user_group := UserGroup}) -> + case mnesia:dirty_read(?TAB, {UserGroup, UserID}) of + [#scram_user_credentail{user_id = {_, UserID}}] -> + {ok, #{user_id => UserID}}; + [] -> + {error, not_found} + end. + +%% TODO: Support Pagination +list_users(#{user_group := UserGroup}) -> + Users = [#{user_id => UserID} || + #scram_user_credentail{user_id = {UserGroup0, UserID}} <- ets:tab2list(?TAB), UserGroup0 =:= UserGroup], + {ok, Users}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +ensure_auth_method('SCRAM-SHA-256', #{algorithm := sha256}) -> + true; +ensure_auth_method('SCRAM-SHA-512', #{algorithm := sha512}) -> + true; +ensure_auth_method(_, _) -> + false. + +check_client_first_message(Bin, _Cache, #{iteration_count := IterationCount} = State) -> + LookupFun = fun(Username) -> + lookup_user2(Username, State) + end, + case esasl_scram:check_client_first_message( + Bin, + #{iteration_count => IterationCount, + lookup => LookupFun} + ) of + {cotinue, ServerFirstMessage, Cache} -> + {cotinue, ServerFirstMessage, Cache}; + {error, _Reason} -> + {error, not_authorized} + end. + +check_client_final_message(Bin, Cache, #{algorithm := Alg}) -> + case esasl_scram:check_client_final_message( + Bin, + Cache#{algorithm => Alg} + ) of + {ok, ServerFinalMessage} -> + {ok, ServerFinalMessage}; + {error, _Reason} -> + {error, not_authorized} + end. + +add_user(UserID, Password, State) -> + UserCredential = esasl_scram:generate_user_credential(UserID, Password, State), + mnesia:write(?TAB, UserCredential, write). + +lookup_user2(UserID, #{user_group := UserGroup}) -> + case mnesia:dirty_read(?TAB, {UserGroup, UserID}) of + [#scram_user_credentail{} = UserCredential] -> + {ok, UserCredential}; + [] -> + {error, not_found} + end. + +%% TODO: Move to emqx_authn_utils.erl +trans(Fun) -> + trans(Fun, []). + +trans(Fun, Args) -> + case ekka_mnesia:transaction(?AUTH_SHARD, Fun, Args) of + {atomic, Res} -> Res; + {aborted, Reason} -> {error, Reason} + end. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl new file mode 100644 index 000000000..aa10a3b98 --- /dev/null +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -0,0 +1,298 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_http). + +-include("emqx_authn.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-behaviour(hocon_schema). + +-export([ structs/0 + , fields/1 + , validations/0 + ]). + +-export([ create/1 + , update/2 + , authenticate/2 + , destroy/1 + ]). + +%%------------------------------------------------------------------------------ +%% Hocon Schema +%%------------------------------------------------------------------------------ + +structs() -> [""]. + +fields("") -> + [ {config, {union, [ hoconsc:t(get) + , hoconsc:t(post) + ]}} + ]; + +fields(get) -> + [ {method, #{type => get, + default => post}} + , {headers, fun headers_no_content_type/1} + ] ++ common_fields(); + +fields(post) -> + [ {method, #{type => post, + default => post}} + , {headers, fun headers/1} + ] ++ common_fields(). + +common_fields() -> + [ {name, fun emqx_authn_schema:authenticator_name/1} + , {mechanism, {enum, ['password-based']}} + , {server_type, {enum, ['http-server']}} + , {url, fun url/1} + , {form_data, fun form_data/1} + , {request_timeout, fun request_timeout/1} + ] ++ maps:to_list(maps:without([ base_url + , pool_type], + maps:from_list(emqx_connector_http:fields(config)))). + +validations() -> + [ {check_ssl_opts, fun check_ssl_opts/1} + , {check_headers, fun check_headers/1} + ]. + +url(type) -> binary(); +url(nullable) -> false; +url(validate) -> [fun check_url/1]; +url(_) -> undefined. + +headers(type) -> map(); +headers(converter) -> + fun(Headers) -> + maps:merge(default_headers(), transform_header_name(Headers)) + end; +headers(default) -> default_headers(); +headers(_) -> undefined. + +headers_no_content_type(type) -> map(); +headers_no_content_type(converter) -> + fun(Headers) -> + maps:merge(default_headers_no_content_type(), transform_header_name(Headers)) + end; +headers_no_content_type(default) -> default_headers_no_content_type(); +headers_no_content_type(_) -> undefined. + +%% TODO: Using map() +form_data(type) -> map(); +form_data(nullable) -> false; +form_data(validate) -> [fun check_form_data/1]; +form_data(_) -> undefined. + +request_timeout(type) -> non_neg_integer(); +request_timeout(default) -> 5000; +request_timeout(_) -> undefined. + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +create(#{ method := Method + , url := URL + , headers := Headers + , form_data := FormData + , request_timeout := RequestTimeout + , '_unique' := Unique + } = Config) -> + #{path := Path, + query := Query} = URIMap = parse_url(URL), + State = #{ method => Method + , path => Path + , base_query => cow_qs:parse_qs(list_to_binary(Query)) + , headers => normalize_headers(Headers) + , form_data => maps:to_list(FormData) + , request_timeout => RequestTimeout + , '_unique' => Unique + }, + case emqx_resource:create_local(Unique, + emqx_connector_http, + Config#{base_url => maps:remove(query, URIMap), + pool_type => random}) of + {ok, _} -> + {ok, State}; + {error, already_created} -> + {ok, State}; + {error, Reason} -> + {error, Reason} + end. + +update(Config, State) -> + case create(Config) of + {ok, NewState} -> + ok = destroy(State), + {ok, NewState}; + {error, Reason} -> + {error, Reason} + end. + +authenticate(#{auth_method := _}, _) -> + ignore; +authenticate(Credential, #{'_unique' := Unique, + method := Method, + request_timeout := RequestTimeout} = State) -> + try + Request = generate_request(Credential, State), + case emqx_resource:query(Unique, {Method, Request, RequestTimeout}) of + {ok, 204, _Headers} -> ok; + {ok, 200, Headers, Body} -> + ContentType = proplists:get_value(<<"content-type">>, Headers, <<"application/json">>), + case safely_parse_body(ContentType, Body) of + {ok, _NBody} -> + %% TODO: Return by user property + ok; + {error, _Reason} -> + ok + end; + {error, _Reason} -> + ignore + end + catch + error:Reason -> + ?LOG(warning, "The following error occurred in '~s' during authentication: ~p", [Unique, Reason]), + ignore + end. + +destroy(#{'_unique' := Unique}) -> + _ = emqx_resource:remove_local(Unique), + ok. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +check_url(URL) -> + case emqx_http_lib:uri_parse(URL) of + {ok, _} -> true; + {error, _} -> false + end. + +check_form_data(FormData) -> + lists:any(fun({_, V}) -> + not is_binary(V) + end, maps:to_list(FormData)). + +default_headers() -> + maps:put(<<"content-type">>, + <<"application/json">>, + default_headers_no_content_type()). + +default_headers_no_content_type() -> + #{ <<"accept">> => <<"application/json">> + , <<"cache-control">> => <<"no-cache">> + , <<"connection">> => <<"keep-alive">> + , <<"keep-alive">> => <<"timeout=5">> + }. + +transform_header_name(Headers) -> + maps:fold(fun(K0, V, Acc) -> + K = list_to_binary(string:to_lower(binary_to_list(K0))), + maps:put(K, V, Acc) + end, #{}, Headers). + +check_ssl_opts(Conf) -> + emqx_connector_http:check_ssl_opts("url", Conf). + +check_headers(Conf) -> + Method = hocon_schema:get_value("method", Conf), + Headers = hocon_schema:get_value("headers", Conf), + case Method =:= get andalso maps:get(<<"content-type">>, Headers, undefined) =/= undefined of + true -> false; + false -> true + end. + +parse_url(URL) -> + {ok, URIMap} = emqx_http_lib:uri_parse(URL), + case maps:get(query, URIMap, undefined) of + undefined -> + URIMap#{query => ""}; + _ -> + URIMap + end. + +normalize_headers(Headers) -> + [{atom_to_binary(K), V} || {K, V} <- maps:to_list(Headers)]. + +generate_request(Credential, #{method := Method, + path := Path, + base_query := BaseQuery, + headers := Headers, + form_data := FormData0}) -> + FormData = replace_placeholders(FormData0, Credential), + case Method of + get -> + NPath = append_query(Path, BaseQuery ++ FormData), + {NPath, Headers}; + post -> + NPath = append_query(Path, BaseQuery), + ContentType = proplists:get_value(<<"content-type">>, Headers), + Body = serialize_body(ContentType, FormData), + {NPath, Headers, Body} + end. + +replace_placeholders(KVs, Credential) -> + replace_placeholders(KVs, Credential, []). + +replace_placeholders([], _Credential, Acc) -> + lists:reverse(Acc); +replace_placeholders([{K, V0} | More], Credential, Acc) -> + case emqx_authn_utils:replace_placeholder(V0, Credential) of + undefined -> + error({cannot_get_variable, V0}); + V -> + replace_placeholders(More, Credential, [{K, emqx_authn_utils:bin(V)} | Acc]) + end. + +append_query(Path, []) -> + Path; +append_query(Path, Query) -> + Path ++ "?" ++ binary_to_list(qs(Query)). + +qs(KVs) -> + qs(KVs, []). + +qs([], Acc) -> + <<$&, Qs/binary>> = iolist_to_binary(lists:reverse(Acc)), + Qs; +qs([{K, V} | More], Acc) -> + qs(More, [["&", emqx_http_lib:uri_encode(K), "=", emqx_http_lib:uri_encode(V)] | Acc]). + +serialize_body(<<"application/json">>, FormData) -> + emqx_json:encode(FormData); +serialize_body(<<"application/x-www-form-urlencoded">>, FormData) -> + qs(FormData). + +safely_parse_body(ContentType, Body) -> + try parse_body(ContentType, Body) of + Result -> Result + catch + _Class:_Reason -> + {error, invalid_body} + end. + +parse_body(<<"application/json">>, Body) -> + {ok, emqx_json:decode(Body)}; +parse_body(<<"application/x-www-form-urlencoded">>, Body) -> + {ok, cow_qs:parse_qs(Body)}; +parse_body(ContentType, _) -> + {error, {unsupported_content_type, ContentType}}. \ No newline at end of file diff --git a/apps/emqx_authentication/src/emqx_authentication_jwks_connector.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl similarity index 88% rename from apps/emqx_authentication/src/emqx_authentication_jwks_connector.erl rename to apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl index 9dafc9f5e..d6e977be6 100644 --- a/apps/emqx_authentication/src/emqx_authentication_jwks_connector.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_authentication_jwks_connector). +-module(emqx_authn_jwks_connector). -behaviour(gen_server). @@ -125,24 +125,15 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- -handle_options(Opts) -> - #{endpoint => proplists:get_value(jwks_endpoint, Opts), - refresh_interval => limit_refresh_interval(proplists:get_value(refresh_interval, Opts)), - ssl_opts => get_ssl_opts(Opts), +handle_options(#{endpoint := Endpoint, + refresh_interval := RefreshInterval0, + ssl_opts := SSLOpts}) -> + #{endpoint => Endpoint, + refresh_interval => limit_refresh_interval(RefreshInterval0), + ssl_opts => maps:to_list(SSLOpts), jwks => [], request_id => undefined}. -get_ssl_opts(Opts) -> - case proplists:get_value(enable_ssl, Opts) of - false -> []; - true -> - maps:to_list(maps:with([cacertfile, - keyfile, - certfile, - verify, - server_name_indication], maps:from_list(Opts))) - end. - refresh_jwks(#{endpoint := Endpoint, ssl_opts := SSLOpts} = State) -> HTTPOpts = [{timeout, 5000}, {connect_timeout, 5000}, {ssl, SSLOpts}], diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl new file mode 100644 index 000000000..fe034994e --- /dev/null +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -0,0 +1,338 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_jwt). + +-include_lib("typerefl/include/types.hrl"). + +-behaviour(hocon_schema). + +-export([ structs/0 + , fields/1 + ]). + +-export([ create/1 + , update/2 + , authenticate/2 + , destroy/1 + ]). + +%%------------------------------------------------------------------------------ +%% Hocon Schema +%%------------------------------------------------------------------------------ + +structs() -> [""]. + +fields("") -> + [ {config, {union, [ hoconsc:t('hmac-based') + , hoconsc:t('public-key') + , hoconsc:t('jwks') + ]}} + ]; + +fields('hmac-based') -> + [ {use_jwks, {enum, [false]}} + , {algorithm, {enum, ['hmac-based']}} + , {secret, fun secret/1} + , {secret_base64_encoded, fun secret_base64_encoded/1} + ] ++ common_fields(); + +fields('public-key') -> + [ {use_jwks, {enum, [false]}} + , {algorithm, {enum, ['public-key']}} + , {certificate, fun certificate/1} + ] ++ common_fields(); + +fields('jwks') -> + [ {use_jwks, {enum, [true]}} + , {endpoint, fun endpoint/1} + , {refresh_interval, fun refresh_interval/1} + , {ssl, #{type => hoconsc:union( + [ hoconsc:ref(?MODULE, ssl_enable) + , hoconsc:ref(?MODULE, ssl_disable) + ]), + default => #{<<"enable">> => false}}} + ] ++ common_fields(); + +fields(ssl_enable) -> + [ {enable, #{type => true}} + , {cacertfile, fun cacertfile/1} + , {certfile, fun certfile/1} + , {keyfile, fun keyfile/1} + , {verify, fun verify/1} + , {server_name_indication, fun server_name_indication/1} + ]; + +fields(ssl_disable) -> + [ {enable, #{type => false}} ]. + +common_fields() -> + [ {name, fun emqx_authn_schema:authenticator_name/1} + , {mechanism, {enum, [jwt]}} + , {verify_claims, fun verify_claims/1} + ]. + +secret(type) -> string(); +secret(_) -> undefined. + +secret_base64_encoded(type) -> boolean(); +secret_base64_encoded(default) -> false; +secret_base64_encoded(_) -> undefined. + +certificate(type) -> string(); +certificate(_) -> undefined. + +endpoint(type) -> string(); +endpoint(_) -> undefined. + +refresh_interval(type) -> integer(); +refresh_interval(default) -> 300; +refresh_interval(validator) -> [fun(I) -> I > 0 end]; +refresh_interval(_) -> undefined. + +cacertfile(type) -> string(); +cacertfile(_) -> undefined. + +certfile(type) -> string(); +certfile(_) -> undefined. + +keyfile(type) -> string(); +keyfile(_) -> undefined. + +verify(type) -> boolean(); +verify(default) -> false; +verify(_) -> undefined. + +server_name_indication(type) -> string(); +server_name_indication(_) -> undefined. + +verify_claims(type) -> list(); +verify_claims(default) -> #{}; +verify_claims(validate) -> [fun check_verify_claims/1]; +verify_claims(converter) -> + fun(VerifyClaims) -> + maps:to_list(VerifyClaims) + end; +verify_claims(_) -> undefined. + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +create(#{verify_claims := VerifyClaims} = Config) -> + create2(Config#{verify_claims => handle_verify_claims(VerifyClaims)}). + +update(#{use_jwks := false} = Config, #{jwk := Connector}) + when is_pid(Connector) -> + _ = emqx_authn_jwks_connector:stop(Connector), + create(Config); + +update(#{use_jwks := false} = Config, _) -> + create(Config); + +update(#{use_jwks := true} = Config, #{jwk := Connector} = State) + when is_pid(Connector) -> + ok = emqx_authn_jwks_connector:update(Connector, Config), + case maps:get(verify_cliams, Config, undefined) of + undefined -> + {ok, State}; + VerifyClaims -> + {ok, State#{verify_claims => handle_verify_claims(VerifyClaims)}} + end; + +update(#{use_jwks := true} = Config, _) -> + create(Config). + +authenticate(#{auth_method := _}, _) -> + ignore; +authenticate(Credential = #{password := JWT}, #{jwk := JWK, + verify_claims := VerifyClaims0}) -> + JWKs = case erlang:is_pid(JWK) of + false -> + [JWK]; + true -> + {ok, JWKs0} = emqx_authn_jwks_connector:get_jwks(JWK), + JWKs0 + end, + VerifyClaims = replace_placeholder(VerifyClaims0, Credential), + case verify(JWT, JWKs, VerifyClaims) of + ok -> ok; + {error, invalid_signature} -> ignore; + {error, {claims, _}} -> {error, bad_username_or_password} + end. + +destroy(#{jwk := Connector}) when is_pid(Connector) -> + _ = emqx_authn_jwks_connector:stop(Connector), + ok; +destroy(_) -> + ok. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +create2(#{use_jwks := false, + algorithm := 'hmac-based', + secret := Secret0, + secret_base64_encoded := Base64Encoded, + verify_claims := VerifyClaims}) -> + Secret = case Base64Encoded of + true -> + base64:decode(Secret0); + false -> + Secret0 + end, + JWK = jose_jwk:from_oct(Secret), + {ok, #{jwk => JWK, + verify_claims => VerifyClaims}}; + +create2(#{use_jwks := false, + algorithm := 'public-key', + certificate := Certificate, + verify_claims := VerifyClaims}) -> + JWK = jose_jwk:from_pem_file(Certificate), + {ok, #{jwk => JWK, + verify_claims => VerifyClaims}}; + +create2(#{use_jwks := true, + verify_claims := VerifyClaims, + ssl := #{enable := Enable} = SSL} = Config) -> + SSLOpts = case Enable of + true -> maps:without([enable], SSL); + false -> #{} + end, + case emqx_authn_jwks_connector:start_link(Config#{ssl_opts => SSLOpts}) of + {ok, Connector} -> + {ok, #{jwk => Connector, + verify_claims => VerifyClaims}}; + {error, Reason} -> + {error, Reason} + end. + +replace_placeholder(L, Variables) -> + replace_placeholder(L, Variables, []). + +replace_placeholder([], _Variables, Acc) -> + Acc; +replace_placeholder([{Name, {placeholder, PL}} | More], Variables, Acc) -> + Value = maps:get(PL, Variables), + replace_placeholder(More, Variables, [{Name, Value} | Acc]); +replace_placeholder([{Name, Value} | More], Variables, Acc) -> + replace_placeholder(More, Variables, [{Name, Value} | Acc]). + +verify(_JWS, [], _VerifyClaims) -> + {error, invalid_signature}; +verify(JWS, [JWK | More], VerifyClaims) -> + try jose_jws:verify(JWK, JWS) of + {true, Payload, _JWS} -> + Claims = emqx_json:decode(Payload, [return_maps]), + verify_claims(Claims, VerifyClaims); + {false, _, _} -> + verify(JWS, More, VerifyClaims) + catch + _:_Reason:_Stacktrace -> + %% TODO: Add log + {error, invalid_signature} + end. + +verify_claims(Claims, VerifyClaims0) -> + Now = os:system_time(seconds), + VerifyClaims = [{<<"exp">>, fun(ExpireTime) -> + Now < ExpireTime + end}, + {<<"iat">>, fun(IssueAt) -> + IssueAt =< Now + end}, + {<<"nbf">>, fun(NotBefore) -> + NotBefore =< Now + end}] ++ VerifyClaims0, + do_verify_claims(Claims, VerifyClaims). + +do_verify_claims(_Claims, []) -> + ok; +do_verify_claims(Claims, [{Name, Fun} | More]) when is_function(Fun) -> + case maps:take(Name, Claims) of + error -> + do_verify_claims(Claims, More); + {Value, NClaims} -> + case Fun(Value) of + true -> + do_verify_claims(NClaims, More); + _ -> + {error, {claims, {Name, Value}}} + end + end; +do_verify_claims(Claims, [{Name, Value} | More]) -> + case maps:take(Name, Claims) of + error -> + {error, {missing_claim, Name}}; + {Value, NClaims} -> + do_verify_claims(NClaims, More); + {Value0, _} -> + {error, {claims, {Name, Value0}}} + end. + +check_verify_claims(Conf) -> + Claims = hocon_schema:get_value("verify_claims", Conf), + do_check_verify_claims(Claims). + +do_check_verify_claims([]) -> + false; +do_check_verify_claims([{Name, Expected} | More]) -> + check_claim_name(Name) andalso + check_claim_expected(Expected) andalso + do_check_verify_claims(More). + +check_claim_name(exp) -> + false; +check_claim_name(iat) -> + false; +check_claim_name(nbf) -> + false; +check_claim_name(_) -> + true. + +check_claim_expected(Expected) -> + try handle_placeholder(Expected) of + _ -> true + catch + _:_ -> + false + end. + +handle_verify_claims(VerifyClaims) -> + handle_verify_claims(VerifyClaims, []). + +handle_verify_claims([], Acc) -> + Acc; +handle_verify_claims([{Name, Expected0} | More], Acc) -> + Expected = handle_placeholder(Expected0), + handle_verify_claims(More, [{Name, Expected} | Acc]). + +handle_placeholder(Placeholder0) -> + case re:run(Placeholder0, "^\\$\\{[a-z0-9\\-]+\\}$", [{capture, all}]) of + {match, [{Offset, Length}]} -> + Placeholder1 = binary:part(Placeholder0, Offset + 2, Length - 3), + Placeholder2 = validate_placeholder(Placeholder1), + {placeholder, Placeholder2}; + nomatch -> + Placeholder0 + end. + +validate_placeholder(<<"mqtt-clientid">>) -> + clientid; +validate_placeholder(<<"mqtt-username">>) -> + username. diff --git a/apps/emqx_authentication/src/emqx_authentication_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl similarity index 73% rename from apps/emqx_authentication/src/emqx_authentication_mnesia.erl rename to apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index 53dc4dd73..ce845d4e3 100644 --- a/apps/emqx_authentication/src/emqx_authentication_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -14,12 +14,17 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_authentication_mnesia). +-module(emqx_authn_mnesia). --include("emqx_authentication.hrl"). +-include("emqx_authn.hrl"). +-include_lib("typerefl/include/types.hrl"). --export([ create/3 - , update/4 +-behaviour(hocon_schema). + +-export([ structs/0, fields/1 ]). + +-export([ create/1 + , update/2 , authenticate/2 , destroy/1 ]). @@ -32,29 +37,10 @@ , list_users/1 ]). -%% TODO: support bcrypt --service_type(#{ - name => mnesia, - params_spec => #{ - user_id_type => #{ - order => 1, - type => string, - enum => [<<"username">>, <<"clientid">>, <<"ip">>, <<"common name">>, <<"issuer">>], - default => <<"username">> - }, - password_hash_algorithm => #{ - order => 2, - type => string, - enum => [<<"plain">>, <<"md5">>, <<"sha">>, <<"sha256">>, <<"sha512">>, <<"bcrypt">>], - default => <<"sha256">> - }, - salt_rounds => #{ - order => 3, - type => number, - default => 10 - } - } -}). +-type user_id_type() :: clientid | username. + +-type user_group() :: {binary(), binary()}. +-type user_id() :: binary(). -record(user_info, { user_id :: {user_group(), user_id()} @@ -62,16 +48,16 @@ , salt :: binary() }). --type(user_group() :: {chain_id(), service_name()}). --type(user_id() :: binary()). +-reflect_type([ user_id_type/0 ]). -export([mnesia/1]). -boot_mnesia({mnesia, [boot]}). -copy_mnesia({mnesia, [copy]}). --define(TAB, mnesia_basic_auth). +-define(TAB, ?MODULE). +-rlog_shard({?AUTH_SHARD, ?TAB}). %%------------------------------------------------------------------------------ %% Mnesia bootstrap %%------------------------------------------------------------------------------ @@ -88,31 +74,87 @@ mnesia(boot) -> mnesia(copy) -> ok = ekka_mnesia:copy_table(?TAB, disc_copies). -create(ChainID, ServiceName, #{<<"user_id_type">> := Type, - <<"password_hash_algorithm">> := Algorithm, - <<"salt_rounds">> := SaltRounds}) -> - Algorithm =:= <<"bcrypt">> andalso ({ok, _} = application:ensure_all_started(bcrypt)), - State = #{user_group => {ChainID, ServiceName}, - user_id_type => binary_to_atom(Type, utf8), - password_hash_algorithm => binary_to_atom(Algorithm, utf8), +%%------------------------------------------------------------------------------ +%% Hocon Schema +%%------------------------------------------------------------------------------ + +structs() -> [config]. + +fields(config) -> + [ {name, fun emqx_authn_schema:authenticator_name/1} + , {mechanism, {enum, ['password-based']}} + , {server_type, {enum, ['built-in-database']}} + , {user_id_type, fun user_id_type/1} + , {password_hash_algorithm, fun password_hash_algorithm/1} + ]; + +fields(bcrypt) -> + [ {name, {enum, [bcrypt]}} + , {salt_rounds, fun salt_rounds/1} + ]; + +fields(other_algorithms) -> + [ {name, {enum, [plain, md5, sha, sha256, sha512]}} + ]. + +user_id_type(type) -> user_id_type(); +user_id_type(default) -> username; +user_id_type(_) -> undefined. + +password_hash_algorithm(type) -> {union, [hoconsc:ref(bcrypt), hoconsc:ref(other_algorithms)]}; +password_hash_algorithm(default) -> #{<<"name">> => sha256}; +password_hash_algorithm(_) -> undefined. + +salt_rounds(type) -> integer(); +salt_rounds(default) -> 10; +salt_rounds(_) -> undefined. + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +create(#{ user_id_type := Type + , password_hash_algorithm := #{name := bcrypt, + salt_rounds := SaltRounds} + , '_unique' := Unique + }) -> + {ok, _} = application:ensure_all_started(bcrypt), + State = #{user_group => Unique, + user_id_type => Type, + password_hash_algorithm => bcrypt, salt_rounds => SaltRounds}, + {ok, State}; + +create(#{ user_id_type := Type + , password_hash_algorithm := #{name := Name} + , '_unique' := Unique + }) -> + State = #{user_group => Unique, + user_id_type => Type, + password_hash_algorithm => Name}, {ok, State}. -update(ChainID, ServiceName, Params, _State) -> - create(ChainID, ServiceName, Params). +update(Config, #{user_group := Unique}) -> + create(Config#{'_unique' => Unique}). -authenticate(ClientInfo = #{password := Password}, +authenticate(#{auth_method := _}, _) -> + ignore; +authenticate(#{password := Password} = Credential, #{user_group := UserGroup, user_id_type := Type, password_hash_algorithm := Algorithm}) -> - UserID = get_user_identity(ClientInfo, Type), + UserID = get_user_identity(Credential, Type), case mnesia:dirty_read(?TAB, {UserGroup, UserID}) of [] -> ignore; - [#user_info{password_hash = PasswordHash, salt = Salt}] -> + [#user_info{password_hash = PasswordHash, salt = Salt0}] -> + Salt = case Algorithm of + bcrypt -> PasswordHash; + _ -> Salt0 + end, case PasswordHash =:= hash(Algorithm, Password, Salt) of true -> ok; - false -> {stop, bad_password} + false -> {error, bad_username_or_password} end end. @@ -136,8 +178,8 @@ import_users(Filename0, State) -> {error, {unsupported_file_format, Extension}} end. -add_user(#{<<"user_id">> := UserID, - <<"password">> := Password}, +add_user(#{user_id := UserID, + password := Password}, #{user_group := UserGroup} = State) -> trans( fun() -> @@ -161,7 +203,7 @@ delete_user(UserID, #{user_group := UserGroup}) -> end end). -update_user(UserID, #{<<"password">> := Password}, +update_user(UserID, #{password := Password}, #{user_group := UserGroup} = State) -> trans( fun() -> @@ -231,7 +273,7 @@ import(UserGroup, [#{<<"user_id">> := UserID, import(_UserGroup, [_ | _More]) -> {error, bad_format}. -%% Importing 5w users needs 1.7 seconds +%% Importing 5w users needs 1.7 seconds import(UserGroup, File, Seq) -> case file:read_line(File) of {ok, Line} -> @@ -297,8 +339,7 @@ gen_salt(#{password_hash_algorithm := bcrypt, {ok, Salt} = bcrypt:gen_salt(Rounds), Salt; gen_salt(_) -> - <> = crypto:strong_rand_bytes(16), - iolist_to_binary(io_lib:format("~32.16.0b", [X])). + emqx_authn_utils:gen_salt(). hash(bcrypt, Password, Salt) -> {ok, Hash} = bcrypt:hashpw(Password, Salt), @@ -310,10 +351,10 @@ insert_user(UserGroup, UserID, PasswordHash) -> insert_user(UserGroup, UserID, PasswordHash, <<>>). insert_user(UserGroup, UserID, PasswordHash, Salt) -> - Credential = #user_info{user_id = {UserGroup, UserID}, - password_hash = PasswordHash, - salt = Salt}, - mnesia:write(?TAB, Credential, write). + UserInfo = #user_info{user_id = {UserGroup, UserID}, + password_hash = PasswordHash, + salt = Salt}, + mnesia:write(?TAB, UserInfo, write). delete_user2(UserInfo) -> mnesia:delete_object(?TAB, UserInfo, write). @@ -330,7 +371,7 @@ trans(Fun) -> trans(Fun, []). trans(Fun, Args) -> - case mnesia:transaction(Fun, Args) of + case ekka_mnesia:transaction(?AUTH_SHARD, Fun, Args) of {atomic, Res} -> Res; {aborted, Reason} -> {error, Reason} end. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl new file mode 100644 index 000000000..4b9fab2be --- /dev/null +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -0,0 +1,227 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_mongodb). + +-include("emqx_authn.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-behaviour(hocon_schema). + +-export([ structs/0 + , fields/1 + ]). + +-export([ create/1 + , update/2 + , authenticate/2 + , destroy/1 + ]). + +%%------------------------------------------------------------------------------ +%% Hocon Schema +%%------------------------------------------------------------------------------ + +structs() -> [""]. + +fields("") -> + [ {config, {union, [ hoconsc:t(standalone) + , hoconsc:t('replica-set') + , hoconsc:t(sharded) + ]}} + ]; + +fields(standalone) -> + common_fields() ++ emqx_connector_mongo:fields(single); + +fields('replica-set') -> + common_fields() ++ emqx_connector_mongo:fields(rs); + +fields(sharded) -> + common_fields() ++ emqx_connector_mongo:fields(sharded). + +common_fields() -> + [ {name, fun emqx_authn_schema:authenticator_name/1} + , {mechanism, {enum, ['password-based']}} + , {server_type, {enum, [mongodb]}} + , {collection, fun collection/1} + , {selector, fun selector/1} + , {password_hash_field, fun password_hash_field/1} + , {salt_field, fun salt_field/1} + , {password_hash_algorithm, fun password_hash_algorithm/1} + , {salt_position, fun salt_position/1} + ]. + +collection(type) -> binary(); +collection(nullable) -> false; +collection(_) -> undefined. + +selector(type) -> map(); +selector(nullable) -> false; +selector(_) -> undefined. + +password_hash_field(type) -> binary(); +password_hash_field(nullable) -> false; +password_hash_field(_) -> undefined. + +salt_field(type) -> binary(); +salt_field(nullable) -> true; +salt_field(_) -> undefined. + +password_hash_algorithm(type) -> {enum, [plain, md5, sha, sha256, sha512, bcrypt]}; +password_hash_algorithm(default) -> sha256; +password_hash_algorithm(_) -> undefined. + +salt_position(type) -> {enum, [prefix, suffix]}; +salt_position(default) -> prefix; +salt_position(_) -> undefined. + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +create(#{ selector := Selector + , '_unique' := Unique + } = Config) -> + NSelector = parse_selector(Selector), + State = maps:with([ collection + , password_hash_field + , salt_field + , password_hash_algorithm + , salt_position + , '_unique'], Config), + NState = State#{selector => NSelector}, + case emqx_resource:create_local(Unique, emqx_connector_mongo, Config) of + {ok, _} -> + {ok, NState}; + {error, already_created} -> + {ok, NState}; + {error, Reason} -> + {error, Reason} + end. + +update(Config, State) -> + case create(Config) of + {ok, NewState} -> + ok = destroy(State), + {ok, NewState}; + {error, Reason} -> + {error, Reason} + end. + +authenticate(#{auth_method := _}, _) -> + ignore; +authenticate(#{password := Password} = Credential, + #{ collection := Collection + , selector := Selector0 + , '_unique' := Unique + } = State) -> + try + Selector1 = replace_placeholders(Selector0, Credential), + Selector2 = normalize_selector(Selector1), + case emqx_resource:query(Unique, {find_one, Collection, Selector2, #{}}) of + undefined -> ignore; + {error, Reason} -> + ?LOG(error, "['~s'] Query failed: ~p", [Unique, Reason]), + ignore; + Doc -> + case check_password(Password, Doc, State) of + ok -> ok; + {error, {cannot_find_password_hash_field, PasswordHashField}} -> + ?LOG(error, "['~s'] Can't find password hash field: ~s", [Unique, PasswordHashField]), + {error, bad_username_or_password}; + {error, Reason} -> + {error, Reason} + end + end + catch + error:Error -> + ?LOG(warning, "The following error occurred in '~s' during authentication: ~p", [Unique, Error]), + ignore + end. + +destroy(#{'_unique' := Unique}) -> + _ = emqx_resource:remove_local(Unique), + ok. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +parse_selector(Selector) -> + NSelector = emqx_json:encode(Selector), + Tokens = re:split(NSelector, "(" ++ ?RE_PLACEHOLDER ++ ")", [{return, binary}, group, trim]), + parse_selector(Tokens, []). + +parse_selector([], Acc) -> + lists:reverse(Acc); +parse_selector([[Constant, Placeholder] | Tokens], Acc) -> + parse_selector(Tokens, [{placeholder, Placeholder}, {constant, Constant} | Acc]); +parse_selector([[Constant] | Tokens], Acc) -> + parse_selector(Tokens, [{constant, Constant} | Acc]). + +replace_placeholders(Selector, Credential) -> + lists:map(fun({constant, Constant}) -> + Constant; + ({placeholder, Placeholder}) -> + case emqx_authn_utils:replace_placeholder(Placeholder, Credential) of + undefined -> error({cannot_get_variable, Placeholder}); + Value -> Value + end + end, Selector). + +normalize_selector(Selector) -> + emqx_json:decode(iolist_to_binary(Selector), [return_maps]). + +check_password(undefined, _Selected, _State) -> + {error, bad_username_or_password}; +check_password(Password, + Doc, + #{password_hash_algorithm := bcrypt, + password_hash_field := PasswordHashField}) -> + case maps:get(PasswordHashField, Doc, undefined) of + undefined -> + {error, {cannot_find_password_hash_field, PasswordHashField}}; + Hash -> + case {ok, Hash} =:= bcrypt:hashpw(Password, Hash) of + true -> ok; + false -> {error, bad_username_or_password} + end + end; +check_password(Password, + Doc, + #{password_hash_algorithm := Algorithm, + password_hash_field := PasswordHashField, + salt_position := SaltPosition} = State) -> + case maps:get(PasswordHashField, Doc, undefined) of + undefined -> + {error, {cannot_find_password_hash_field, PasswordHashField}}; + Hash -> + Salt = case maps:get(salt_field, State, undefined) of + undefined -> <<>>; + SaltField -> maps:get(SaltField, Doc, <<>>) + end, + case Hash =:= hash(Algorithm, Password, Salt, SaltPosition) of + true -> ok; + false -> {error, bad_username_or_password} + end + end. + +hash(Algorithm, Password, Salt, prefix) -> + emqx_passwd:hash(Algorithm, <>); +hash(Algorithm, Password, Salt, suffix) -> + emqx_passwd:hash(Algorithm, <>). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl new file mode 100644 index 000000000..62c5c49e7 --- /dev/null +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -0,0 +1,167 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_mysql). + +-include("emqx_authn.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-behaviour(hocon_schema). + +-export([ structs/0 + , fields/1 + ]). + +-export([ create/1 + , update/2 + , authenticate/2 + , destroy/1 + ]). + +%%------------------------------------------------------------------------------ +%% Hocon Schema +%%------------------------------------------------------------------------------ + +structs() -> [config]. + +fields(config) -> + [ {name, fun emqx_authn_schema:authenticator_name/1} + , {mechanism, {enum, ['password-based']}} + , {server_type, {enum, [mysql]}} + , {password_hash_algorithm, fun password_hash_algorithm/1} + , {salt_position, fun salt_position/1} + , {query, fun query/1} + , {query_timeout, fun query_timeout/1} + ] ++ emqx_connector_schema_lib:relational_db_fields() + ++ emqx_connector_schema_lib:ssl_fields(). + +password_hash_algorithm(type) -> {enum, [plain, md5, sha, sha256, sha512, bcrypt]}; +password_hash_algorithm(default) -> sha256; +password_hash_algorithm(_) -> undefined. + +salt_position(type) -> {enum, [prefix, suffix]}; +salt_position(default) -> prefix; +salt_position(_) -> undefined. + +query(type) -> string(); +query(nullable) -> false; +query(_) -> undefined. + +query_timeout(type) -> integer(); +query_timeout(default) -> 5000; +query_timeout(_) -> undefined. + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +create(#{ password_hash_algorithm := Algorithm + , salt_position := SaltPosition + , query := Query0 + , query_timeout := QueryTimeout + , '_unique' := Unique + } = Config) -> + {Query, PlaceHolders} = parse_query(Query0), + State = #{password_hash_algorithm => Algorithm, + salt_position => SaltPosition, + query => Query, + placeholders => PlaceHolders, + query_timeout => QueryTimeout, + '_unique' => Unique}, + case emqx_resource:create_local(Unique, emqx_connector_mysql, Config) of + {ok, _} -> + {ok, State}; + {error, already_created} -> + {ok, State}; + {error, Reason} -> + {error, Reason} + end. + +update(Config, State) -> + case create(Config) of + {ok, NewState} -> + ok = destroy(State), + {ok, NewState}; + {error, Reason} -> + {error, Reason} + end. + +authenticate(#{auth_method := _}, _) -> + ignore; +authenticate(#{password := Password} = Credential, + #{placeholders := PlaceHolders, + query := Query, + query_timeout := Timeout, + '_unique' := Unique} = State) -> + try + Params = emqx_authn_utils:replace_placeholders(PlaceHolders, Credential), + case emqx_resource:query(Unique, {sql, Query, Params, Timeout}) of + {ok, _Columns, []} -> ignore; + {ok, Columns, Rows} -> + %% TODO: Support superuser + Selected = maps:from_list(lists:zip(Columns, Rows)), + check_password(Password, Selected, State); + {error, _Reason} -> + ignore + end + catch + error:Reason -> + ?LOG(warning, "The following error occurred in '~s' during authentication: ~p", [Unique, Reason]), + ignore + end. + +destroy(#{'_unique' := Unique}) -> + _ = emqx_resource:remove_local(Unique), + ok. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +check_password(undefined, _Selected, _State) -> + {error, bad_username_or_password}; +check_password(Password, + #{password_hash := Hash}, + #{password_hash_algorithm := bcrypt}) -> + case {ok, Hash} =:= bcrypt:hashpw(Password, Hash) of + true -> ok; + false -> {error, bad_username_or_password} + end; +check_password(Password, + #{password_hash := Hash} = Selected, + #{password_hash_algorithm := Algorithm, + salt_position := SaltPosition}) -> + Salt = maps:get(salt, Selected, <<>>), + Hash0 = case SaltPosition of + prefix -> emqx_passwd:hash(Algorithm, <>); + suffix -> emqx_passwd:hash(Algorithm, <>) + end, + case Hash0 =:= Hash of + true -> ok; + false -> {error, bad_username_or_password} + end. + +%% TODO: Support prepare +parse_query(Query) -> + case re:run(Query, ?RE_PLACEHOLDER, [global, {capture, all, binary}]) of + {match, Captured} -> + PlaceHolders = [PlaceHolder || PlaceHolder <- Captured], + NQuery = re:replace(Query, "'\\$\\{[a-z0-9\\_]+\\}'", "?", [global, {return, binary}]), + {NQuery, PlaceHolders}; + nomatch -> + {Query, []} + end. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_other_schema.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_other_schema.erl new file mode 100644 index 000000000..0f5c8abb8 --- /dev/null +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_other_schema.erl @@ -0,0 +1,58 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_other_schema). + +-include("emqx_authn.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-behaviour(hocon_schema). + +-export([ structs/0 + , fields/1 + ]). + +structs() -> [ "filename", "position", "user_info", "new_user_info"]. + +fields("filename") -> + [ {filename, fun filename/1} ]; +fields("position") -> + [ {position, fun position/1} ]; +fields("user_info") -> + [ {user_id, fun user_id/1} + , {password, fun password/1} + ]; +fields("new_user_info") -> + [ {password, fun password/1} + ]. + +filename(type) -> string(); +filename(nullable) -> false; +filename(_) -> undefined. + +position(type) -> integer(); +position(validate) -> [fun (Position) -> Position > 0 end]; +position(nullable) -> false; +position(_) -> undefined. + +user_id(type) -> binary(); +user_id(nullable) -> false; +user_id(_) -> undefined. + +password(type) -> binary(); +password(nullable) -> false; +password(_) -> undefined. + diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl new file mode 100644 index 000000000..a4d00be29 --- /dev/null +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -0,0 +1,156 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_pgsql). + +-include("emqx_authn.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-behaviour(hocon_schema). + +-export([ structs/0, fields/1 ]). + +-export([ create/1 + , update/2 + , authenticate/2 + , destroy/1 + ]). + +%%------------------------------------------------------------------------------ +%% Hocon Schema +%%------------------------------------------------------------------------------ + +structs() -> [config]. + +fields(config) -> + [ {name, fun emqx_authn_schema:authenticator_name/1} + , {mechanism, {enum, ['password-based']}} + , {server_type, {enum, [pgsql]}} + , {password_hash_algorithm, fun password_hash_algorithm/1} + , {salt_position, {enum, [prefix, suffix]}} + , {query, fun query/1} + ] ++ emqx_connector_schema_lib:relational_db_fields() + ++ emqx_connector_schema_lib:ssl_fields(). + +password_hash_algorithm(type) -> {enum, [plain, md5, sha, sha256, sha512, bcrypt]}; +password_hash_algorithm(default) -> sha256; +password_hash_algorithm(_) -> undefined. + +query(type) -> string(); +query(nullable) -> false; +query(_) -> undefined. + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +create(#{ query := Query0 + , password_hash_algorithm := Algorithm + , salt_position := SaltPosition + , '_unique' := Unique + } = Config) -> + {Query, PlaceHolders} = parse_query(Query0), + State = #{query => Query, + placeholders => PlaceHolders, + password_hash_algorithm => Algorithm, + salt_position => SaltPosition, + '_unique' => Unique}, + case emqx_resource:create_local(Unique, emqx_connector_pgsql, Config) of + {ok, _} -> + {ok, State}; + {error, already_created} -> + {ok, State}; + {error, Reason} -> + {error, Reason} + end. + +update(Config, State) -> + case create(Config) of + {ok, NewState} -> + ok = destroy(State), + {ok, NewState}; + {error, Reason} -> + {error, Reason} + end. + +authenticate(#{auth_method := _}, _) -> + ignore; +authenticate(#{password := Password} = Credential, + #{query := Query, + placeholders := PlaceHolders, + '_unique' := Unique} = State) -> + try + Params = emqx_authn_utils:replace_placeholders(PlaceHolders, Credential), + case emqx_resource:query(Unique, {sql, Query, Params}) of + {ok, _Columns, []} -> ignore; + {ok, Columns, Rows} -> + %% TODO: Support superuser + Selected = maps:from_list(lists:zip(Columns, Rows)), + check_password(Password, Selected, State); + {error, _Reason} -> + ignore + end + catch + error:Reason -> + ?LOG(warning, "The following error occurred in '~s' during authentication: ~p", [Unique, Reason]), + ignore + end. + +destroy(#{'_unique' := Unique}) -> + _ = emqx_resource:remove_local(Unique), + ok. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +check_password(undefined, _Selected, _State) -> + {error, bad_username_or_password}; +check_password(Password, + #{password_hash := Hash}, + #{password_hash_algorithm := bcrypt}) -> + case {ok, Hash} =:= bcrypt:hashpw(Password, Hash) of + true -> ok; + false -> {error, bad_username_or_password} + end; +check_password(Password, + #{password_hash := Hash} = Selected, + #{password_hash_algorithm := Algorithm, + salt_position := SaltPosition}) -> + Salt = maps:get(salt, Selected, <<>>), + Hash0 = case SaltPosition of + prefix -> emqx_passwd:hash(Algorithm, <>); + suffix -> emqx_passwd:hash(Algorithm, <>) + end, + case Hash0 =:= Hash of + true -> ok; + false -> {error, bad_username_or_password} + end. + +%% TODO: Support prepare +parse_query(Query) -> + case re:run(Query, ?RE_PLACEHOLDER, [global, {capture, all, binary}]) of + {match, Captured} -> + PlaceHolders = [PlaceHolder || PlaceHolder <- Captured], + Replacements = ["$" ++ integer_to_list(I) || I <- lists:seq(1, length(Captured))], + NQuery = lists:foldl(fun({PlaceHolder, Replacement}, Query0) -> + re:replace(Query0, <<"'\\", PlaceHolder/binary, "'">>, Replacement, [{return, binary}]) + end, Query, lists:zip(PlaceHolders, Replacements)), + {NQuery, PlaceHolders}; + nomatch -> + {Query, []} + end. diff --git a/apps/emqx_authn/test/data/private_key.pem b/apps/emqx_authn/test/data/private_key.pem new file mode 100644 index 000000000..318eefe27 --- /dev/null +++ b/apps/emqx_authn/test/data/private_key.pem @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICWwIBAAKBgHA+aGk+D/0J9ZAj3tQIAvDTnRvZNF0IeaTmJcBooxsY6Ze8PGFS +QJ/+EJf1i1ffExeaIH99d3nwyBspNlihooGHvTVDeYsu15Htxpqig1L/+MphbZlF +ClDXtzV0+GeZGoqeloHAXku3Qzk+hMxROalFtH+8GNp+/j2yir1Z9E2xAgMBAAEC +gYBX7HsLncMWexux6nddbl0nWwyhyPZcvgvT4TjHTPAfhNdOtfQyZCUdbv5+mqip +j6O8BE7ar2TMz5FgvVrF+O97LkYHNmZk0q3xtZlCYXp4BQqD6Wq65H5U4fAomalK +xm7HsTCSVXx5CvnZK/JbkPw18QsgwrSHEFs+4Pf2noH+FQJBAL/bpPrkDOB476Iy +RGnuCckUN1pdCU+UINC8oOWGNwsG6EE5ywlIWRXHtp4VMksG6mCLNJwGUAv2zWIs +iEjZVfsCQQCVxOciTajTtYO5bPjkXoZoe4VKKXWMYv9AXXVCjq0ff/LjrnKJjbRm +aoKQGhzjKHk5rgd9+Ydl6FnJw5K4B9dDAkEAtaHfQpZ7ildzpf4ovpBYO0EkViwW +EHyPxI2PVTwHCC1126o3CYawr+nufSJcBqN5aAThvYRMa8cvEW5PZ4g52QJALF5L +tt7Yz/crEciVp1nVaaiGISVNHIzLX28QaOpJoVZPR2ILrnJbaifNjBEgU69O0maa +85fzo54E03/rvDcebwJAfTMgIyzFQK/ESnM43bUCI/Y5XAeKFBiN1YhCioNR4Hj7 +Lkw2RdrrPC9LV+gVJK0b7VUqR5odjdj7PN6SipuXNw== +-----END RSA PRIVATE KEY----- diff --git a/apps/emqx_authn/test/data/public_key.pem b/apps/emqx_authn/test/data/public_key.pem new file mode 100644 index 000000000..b0b151981 --- /dev/null +++ b/apps/emqx_authn/test/data/public_key.pem @@ -0,0 +1,6 @@ +-----BEGIN PUBLIC KEY----- +MIGeMA0GCSqGSIb3DQEBAQUAA4GMADCBiAKBgHA+aGk+D/0J9ZAj3tQIAvDTnRvZ +NF0IeaTmJcBooxsY6Ze8PGFSQJ/+EJf1i1ffExeaIH99d3nwyBspNlihooGHvTVD +eYsu15Htxpqig1L/+MphbZlFClDXtzV0+GeZGoqeloHAXku3Qzk+hMxROalFtH+8 +GNp+/j2yir1Z9E2xAgMBAAE= +-----END PUBLIC KEY----- diff --git a/apps/emqx_authentication/test/data/user-credentials.csv b/apps/emqx_authn/test/data/user-credentials.csv similarity index 100% rename from apps/emqx_authentication/test/data/user-credentials.csv rename to apps/emqx_authn/test/data/user-credentials.csv diff --git a/apps/emqx_authentication/test/data/user-credentials.json b/apps/emqx_authn/test/data/user-credentials.json similarity index 100% rename from apps/emqx_authentication/test/data/user-credentials.json rename to apps/emqx_authn/test/data/user-credentials.json diff --git a/apps/emqx_authn/test/emqx_authn_SUITE.erl b/apps/emqx_authn/test/emqx_authn_SUITE.erl new file mode 100644 index 000000000..36b0eabf3 --- /dev/null +++ b/apps/emqx_authn/test/emqx_authn_SUITE.erl @@ -0,0 +1,105 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-include("emqx_authn.hrl"). + +-define(AUTH, emqx_authn). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + application:set_env(ekka, strict_mode, true), + emqx_ct_helpers:start_apps([emqx_authn]), + Config. + +end_per_suite(_) -> + emqx_ct_helpers:stop_apps([emqx_authn]), + ok. + +t_chain(_) -> + ?assertMatch({ok, #{id := ?CHAIN, authenticators := []}}, ?AUTH:lookup_chain(?CHAIN)), + + ChainID = <<"mychain">>, + Chain = #{id => ChainID}, + ?assertMatch({ok, #{id := ChainID, authenticators := []}}, ?AUTH:create_chain(Chain)), + ?assertEqual({error, {already_exists, {chain, ChainID}}}, ?AUTH:create_chain(Chain)), + ?assertMatch({ok, #{id := ChainID, authenticators := []}}, ?AUTH:lookup_chain(ChainID)), + ?assertEqual(ok, ?AUTH:delete_chain(ChainID)), + ?assertMatch({error, {not_found, {chain, ChainID}}}, ?AUTH:lookup_chain(ChainID)), + ok. + +t_authenticator(_) -> + AuthenticatorName1 = <<"myauthenticator1">>, + AuthenticatorConfig1 = #{name => AuthenticatorName1, + mechanism => 'password-based', + server_type => 'built-in-database', + user_id_type => username, + password_hash_algorithm => #{ + name => sha256 + }}, + {ok, #{name := AuthenticatorName1, id := ID1}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig1), + ?assertMatch({ok, #{name := AuthenticatorName1}}, ?AUTH:lookup_authenticator(?CHAIN, ID1)), + ?assertMatch({ok, [#{name := AuthenticatorName1}]}, ?AUTH:list_authenticators(?CHAIN)), + ?assertEqual({error, name_has_be_used}, ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig1)), + + AuthenticatorConfig2 = #{name => AuthenticatorName1, + mechanism => jwt, + use_jwks => false, + algorithm => 'hmac-based', + secret => <<"abcdef">>, + secret_base64_encoded => false, + verify_claims => []}, + {ok, #{name := AuthenticatorName1, id := ID1, mechanism := jwt}} = ?AUTH:update_authenticator(?CHAIN, ID1, AuthenticatorConfig2), + + ID2 = <<"random">>, + ?assertEqual({error, {not_found, {authenticator, ID2}}}, ?AUTH:update_authenticator(?CHAIN, ID2, AuthenticatorConfig2)), + ?assertEqual({error, name_has_be_used}, ?AUTH:update_or_create_authenticator(?CHAIN, ID2, AuthenticatorConfig2)), + + AuthenticatorName2 = <<"myauthenticator2">>, + AuthenticatorConfig3 = AuthenticatorConfig2#{name => AuthenticatorName2}, + {ok, #{name := AuthenticatorName2, id := ID2, secret := <<"abcdef">>}} = ?AUTH:update_or_create_authenticator(?CHAIN, ID2, AuthenticatorConfig3), + ?assertMatch({ok, #{name := AuthenticatorName2}}, ?AUTH:lookup_authenticator(?CHAIN, ID2)), + {ok, #{name := AuthenticatorName2, id := ID2, secret := <<"fedcba">>}} = ?AUTH:update_or_create_authenticator(?CHAIN, ID2, AuthenticatorConfig3#{secret := <<"fedcba">>}), + + ?assertMatch({ok, #{id := ?CHAIN, authenticators := [#{name := AuthenticatorName1}, #{name := AuthenticatorName2}]}}, ?AUTH:lookup_chain(?CHAIN)), + ?assertMatch({ok, [#{name := AuthenticatorName1}, #{name := AuthenticatorName2}]}, ?AUTH:list_authenticators(?CHAIN)), + + ?assertEqual(ok, ?AUTH:move_authenticator_to_the_nth(?CHAIN, ID2, 1)), + ?assertMatch({ok, [#{name := AuthenticatorName2}, #{name := AuthenticatorName1}]}, ?AUTH:list_authenticators(?CHAIN)), + ?assertEqual({error, out_of_range}, ?AUTH:move_authenticator_to_the_nth(?CHAIN, ID2, 3)), + ?assertEqual({error, out_of_range}, ?AUTH:move_authenticator_to_the_nth(?CHAIN, ID2, 0)), + ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID1)), + ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID2)), + ?assertEqual({ok, []}, ?AUTH:list_authenticators(?CHAIN)), + ok. + +t_authenticate(_) -> + ClientInfo = #{zone => default, + listener => mqtt_tcp, + username => <<"myuser">>, + password => <<"mypass">>}, + ?assertEqual(ok, emqx_access_control:authenticate(ClientInfo)), + emqx_authn:enable(), + ?assertEqual({error, not_authorized}, emqx_access_control:authenticate(ClientInfo)). diff --git a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl new file mode 100644 index 000000000..7435deaa0 --- /dev/null +++ b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl @@ -0,0 +1,152 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_jwt_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-include("emqx_authn.hrl"). + +-define(AUTH, emqx_authn). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_authn]), + Config. + +end_per_suite(_) -> + emqx_ct_helpers:stop_apps([emqx_authn]), + ok. + +t_jwt_authenticator(_) -> + AuthenticatorName = <<"myauthenticator">>, + Config = #{name => AuthenticatorName, + mechanism => jwt, + use_jwks => false, + algorithm => 'hmac-based', + secret => <<"abcdef">>, + secret_base64_encoded => false, + verify_claims => []}, + {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, Config), + + Payload = #{<<"username">> => <<"myuser">>}, + JWS = generate_jws('hmac-based', Payload, <<"abcdef">>), + ClientInfo = #{username => <<"myuser">>, + password => JWS}, + ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo, ok)), + + BadJWS = generate_jws('hmac-based', Payload, <<"bad_secret">>), + ClientInfo2 = ClientInfo#{password => BadJWS}, + ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo2, ok)), + + %% secret_base64_encoded + Config2 = Config#{secret => base64:encode(<<"abcdef">>), + secret_base64_encoded => true}, + ?assertMatch({ok, _}, ?AUTH:update_authenticator(?CHAIN, ID, Config2)), + ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo, ok)), + + Config3 = Config#{verify_claims => [{<<"username">>, <<"${mqtt-username}">>}]}, + ?assertMatch({ok, _}, ?AUTH:update_authenticator(?CHAIN, ID, Config3)), + ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo, ok)), + ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo#{username => <<"otheruser">>}, ok)), + + %% Expiration + Payload3 = #{ <<"username">> => <<"myuser">> + , <<"exp">> => erlang:system_time(second) - 60}, + JWS3 = generate_jws('hmac-based', Payload3, <<"abcdef">>), + ClientInfo3 = ClientInfo#{password => JWS3}, + ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo3, ok)), + + Payload4 = #{ <<"username">> => <<"myuser">> + , <<"exp">> => erlang:system_time(second) + 60}, + JWS4 = generate_jws('hmac-based', Payload4, <<"abcdef">>), + ClientInfo4 = ClientInfo#{password => JWS4}, + ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo4, ok)), + + %% Issued At + Payload5 = #{ <<"username">> => <<"myuser">> + , <<"iat">> => erlang:system_time(second) - 60}, + JWS5 = generate_jws('hmac-based', Payload5, <<"abcdef">>), + ClientInfo5 = ClientInfo#{password => JWS5}, + ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo5, ok)), + + Payload6 = #{ <<"username">> => <<"myuser">> + , <<"iat">> => erlang:system_time(second) + 60}, + JWS6 = generate_jws('hmac-based', Payload6, <<"abcdef">>), + ClientInfo6 = ClientInfo#{password => JWS6}, + ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo6, ok)), + + %% Not Before + Payload7 = #{ <<"username">> => <<"myuser">> + , <<"nbf">> => erlang:system_time(second) - 60}, + JWS7 = generate_jws('hmac-based', Payload7, <<"abcdef">>), + ClientInfo7 = ClientInfo#{password => JWS7}, + ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo7, ok)), + + Payload8 = #{ <<"username">> => <<"myuser">> + , <<"nbf">> => erlang:system_time(second) + 60}, + JWS8 = generate_jws('hmac-based', Payload8, <<"abcdef">>), + ClientInfo8 = ClientInfo#{password => JWS8}, + ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo8, ok)), + + ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), + ok. + +t_jwt_authenticator2(_) -> + Dir = code:lib_dir(emqx_authn, test), + PublicKey = list_to_binary(filename:join([Dir, "data/public_key.pem"])), + PrivateKey = list_to_binary(filename:join([Dir, "data/private_key.pem"])), + AuthenticatorName = <<"myauthenticator">>, + Config = #{name => AuthenticatorName, + mechanism => jwt, + use_jwks => false, + algorithm => 'public-key', + certificate => PublicKey, + verify_claims => []}, + {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, Config), + + Payload = #{<<"username">> => <<"myuser">>}, + JWS = generate_jws('public-key', Payload, PrivateKey), + ClientInfo = #{username => <<"myuser">>, + password => JWS}, + ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo, ok)), + ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo#{password => <<"badpassword">>}, ok)), + + ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), + ok. + +generate_jws('hmac-based', Payload, Secret) -> + JWK = jose_jwk:from_oct(Secret), + Header = #{ <<"alg">> => <<"HS256">> + , <<"typ">> => <<"JWT">> + }, + Signed = jose_jwt:sign(JWK, Header, Payload), + {_, JWS} = jose_jws:compact(Signed), + JWS; +generate_jws('public-key', Payload, PrivateKey) -> + JWK = jose_jwk:from_pem_file(PrivateKey), + Header = #{ <<"alg">> => <<"RS256">> + , <<"typ">> => <<"JWT">> + }, + Signed = jose_jwt:sign(JWK, Header, Payload), + {_, JWS} = jose_jws:compact(Signed), + JWS. diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl new file mode 100644 index 000000000..4a5a24844 --- /dev/null +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -0,0 +1,155 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_mnesia_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-include("emqx_authn.hrl"). + +-define(AUTH, emqx_authn). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_authn]), + Config. + +end_per_suite(_) -> + emqx_ct_helpers:stop_apps([emqx_authn]), + ok. + +t_mnesia_authenticator(_) -> + AuthenticatorName = <<"myauthenticator">>, + AuthenticatorConfig = #{name => AuthenticatorName, + mechanism => 'password-based', + server_type => 'built-in-database', + user_id_type => username, + password_hash_algorithm => #{ + name => sha256 + }}, + {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig), + + UserInfo = #{user_id => <<"myuser">>, + password => <<"mypass">>}, + ?assertEqual({ok, #{user_id => <<"myuser">>}}, ?AUTH:add_user(?CHAIN, ID, UserInfo)), + ?assertEqual({ok, #{user_id => <<"myuser">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), + + ClientInfo = #{zone => external, + username => <<"myuser">>, + password => <<"mypass">>}, + ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo, ok)), + ?AUTH:enable(), + ?assertEqual(ok, emqx_access_control:authenticate(ClientInfo)), + + ClientInfo2 = ClientInfo#{username => <<"baduser">>}, + ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo2, ok)), + ?assertEqual({error, not_authorized}, emqx_access_control:authenticate(ClientInfo2)), + + ClientInfo3 = ClientInfo#{password => <<"badpass">>}, + ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo3, ok)), + ?assertEqual({error, bad_username_or_password}, emqx_access_control:authenticate(ClientInfo3)), + + UserInfo2 = UserInfo#{password => <<"mypass2">>}, + ?assertEqual({ok, #{user_id => <<"myuser">>}}, ?AUTH:update_user(?CHAIN, ID, <<"myuser">>, UserInfo2)), + ClientInfo4 = ClientInfo#{password => <<"mypass2">>}, + ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo4, ok)), + + ?assertEqual(ok, ?AUTH:delete_user(?CHAIN, ID, <<"myuser">>)), + ?assertEqual({error, not_found}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), + + ?assertEqual({ok, #{user_id => <<"myuser">>}}, ?AUTH:add_user(?CHAIN, ID, UserInfo)), + ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), + ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), + + {ok, #{name := AuthenticatorName, id := ID1}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig), + ?assertMatch({error, not_found}, ?AUTH:lookup_user(?CHAIN, ID1, <<"myuser">>)), + ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID1)), + ok. + +t_import(_) -> + AuthenticatorName = <<"myauthenticator">>, + AuthenticatorConfig = #{name => AuthenticatorName, + mechanism => 'password-based', + server_type => 'built-in-database', + user_id_type => username, + password_hash_algorithm => #{ + name => sha256 + }}, + {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig), + + Dir = code:lib_dir(emqx_authn, test), + ?assertEqual(ok, ?AUTH:import_users(?CHAIN, ID, filename:join([Dir, "data/user-credentials.json"]))), + ?assertEqual(ok, ?AUTH:import_users(?CHAIN, ID, filename:join([Dir, "data/user-credentials.csv"]))), + ?assertMatch({ok, #{user_id := <<"myuser1">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser1">>)), + ?assertMatch({ok, #{user_id := <<"myuser3">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser3">>)), + + ClientInfo1 = #{username => <<"myuser1">>, + password => <<"mypassword1">>}, + ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo1, ok)), + ClientInfo2 = ClientInfo1#{username => <<"myuser3">>, + password => <<"mypassword3">>}, + ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo2, ok)), + ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), + ok. + +t_multi_mnesia_authenticator(_) -> + AuthenticatorName1 = <<"myauthenticator1">>, + AuthenticatorConfig1 = #{name => AuthenticatorName1, + mechanism => 'password-based', + server_type => 'built-in-database', + user_id_type => username, + password_hash_algorithm => #{ + name => sha256 + }}, + AuthenticatorName2 = <<"myauthenticator2">>, + AuthenticatorConfig2 = #{name => AuthenticatorName2, + mechanism => 'password-based', + server_type => 'built-in-database', + user_id_type => clientid, + password_hash_algorithm => #{ + name => sha256 + }}, + {ok, #{name := AuthenticatorName1, id := ID1}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig1), + {ok, #{name := AuthenticatorName2, id := ID2}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig2), + + ?assertEqual({ok, #{user_id => <<"myuser">>}}, + ?AUTH:add_user(?CHAIN, ID1, + #{user_id => <<"myuser">>, + password => <<"mypass1">>})), + ?assertEqual({ok, #{user_id => <<"myclient">>}}, + ?AUTH:add_user(?CHAIN, ID2, + #{user_id => <<"myclient">>, + password => <<"mypass2">>})), + + ClientInfo1 = #{username => <<"myuser">>, + clientid => <<"myclient">>, + password => <<"mypass1">>}, + ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo1, ok)), + ?assertEqual(ok, ?AUTH:move_authenticator_to_the_nth(?CHAIN, ID2, 1)), + + ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo1, ok)), + ClientInfo2 = ClientInfo1#{password => <<"mypass2">>}, + ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo2, ok)), + + ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID1)), + ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID2)), + ok. diff --git a/apps/emqx_authz/README.md b/apps/emqx_authz/README.md index 0fddac9b0..420898c95 100644 --- a/apps/emqx_authz/README.md +++ b/apps/emqx_authz/README.md @@ -16,9 +16,14 @@ authz:{ username: root password: public auto_reconnect: true - ssl: false + ssl: { + enable: true + cacertfile: "etc/certs/cacert.pem" + certfile: "etc/certs/client-cert.pem" + keyfile: "etc/certs/client-key.pem" + } } - sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_acl where ipaddr = '%a' or username = '%u' or clientid = '%c'" + sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or clientid = '%c'" }, { type: pgsql @@ -29,9 +34,9 @@ authz:{ username: root password: public auto_reconnect: true - ssl: false + ssl: {enable: false} } - sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_acl where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'" + sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'" }, { type: redis @@ -41,9 +46,9 @@ authz:{ pool_size: 1 password: public auto_reconnect: true - ssl: false + ssl: {enable: false} } - cmd: "HGETALL mqtt_acl:%u" + cmd: "HGETALL mqtt_authz:%u" }, { principal: {username: "^admin?"} @@ -72,7 +77,7 @@ authz:{ Create Example Table ```sql -CREATE TABLE `mqtt_acl` ( +CREATE TABLE `mqtt_authz` ( `id` int(11) unsigned NOT NULL AUTO_INCREMENT, `ipaddress` VARCHAR(60) NOT NULL DEFAULT '', `username` VARCHAR(100) NOT NULL DEFAULT '', @@ -88,7 +93,7 @@ Sample data in the default configuration: ```sql -- Only 127.0.0.1 users can subscribe to system topics -INSERT INTO mqtt_acl (ipaddress, username, clientid, action, permission, topic) VALUES ('127.0.0.1', '', '', 'subscribe', 'allow', '$SYS/#'); +INSERT INTO mqtt_authz (ipaddress, username, clientid, action, permission, topic) VALUES ('127.0.0.1', '', '', 'subscribe', 'allow', '$SYS/#'); ``` #### Pgsql @@ -99,7 +104,7 @@ Create Example Table CREATE TYPE ACTION AS ENUM('publish','subscribe','all'); CREATE TYPE PERMISSION AS ENUM('allow','deny'); -CREATE TABLE mqtt_acl ( +CREATE TABLE mqtt_authz ( id SERIAL PRIMARY KEY, ipaddress CHARACTER VARYING(60) NOT NULL DEFAULT '', username CHARACTER VARYING(100) NOT NULL DEFAULT '', @@ -115,7 +120,7 @@ Sample data in the default configuration: ```sql -- Only 127.0.0.1 users can subscribe to system topics -INSERT INTO mqtt_acl (ipaddress, username, clientid, action, permission, topic) VALUES ('127.0.0.1', '', '', 'subscribe', 'allow', '$SYS/#'); +INSERT INTO mqtt_authz (ipaddress, username, clientid, action, permission, topic) VALUES ('127.0.0.1', '', '', 'subscribe', 'allow', '$SYS/#'); ``` #### Redis @@ -123,8 +128,21 @@ INSERT INTO mqtt_acl (ipaddress, username, clientid, action, permission, topic) Sample data in the default configuration: ``` -HSET mqtt_acl:emqx '$SYS/#' subscribe +HSET mqtt_authz:emqx '$SYS/#' subscribe ``` -A rule of Redis ACL defines `publish`, `subscribe`, or `all `information. All lists in the rule are **allow** lists. +A rule of Redis AuthZ defines `publish`, `subscribe`, or `all `information. All lists in the rule are **allow** lists. +#### Mongo + +Create Example BSON documents +```sql +db.inventory.insertOne( + {username: "emqx", + clientid: "emqx", + ipaddress: "127.0.0.1", + permission: "allow", + action: "all", + topics: ["#"] + }) +``` diff --git a/apps/emqx_authz/etc/emqx_authz.conf b/apps/emqx_authz/etc/emqx_authz.conf index ec8f642b1..8826a94f7 100644 --- a/apps/emqx_authz/etc/emqx_authz.conf +++ b/apps/emqx_authz/etc/emqx_authz.conf @@ -1,5 +1,15 @@ -authz:{ +authorization:{ rules: [ + # { + # type: http + # config: { + # url: "https://emqx.com" + # headers: { + # Accept: "application/json" + # Content-Type: "application/json" + # } + # } + # }, # { # type: mysql # config: { @@ -9,9 +19,14 @@ authz:{ # username: root # password: public # auto_reconnect: true - # ssl: false + # ssl: { + # enable: true + # cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem" + # certfile: "{{ platform_etc_dir }}/certs/client-cert.pem" + # keyfile: "{{ platform_etc_dir }}/certs/client-key.pem" + # } # } - # sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_acl where ipaddr = '%a' or username = '%u' or clientid = '%c'" + # sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or clientid = '%c'" # }, # { # type: pgsql @@ -22,9 +37,9 @@ authz:{ # username: root # password: public # auto_reconnect: true - # ssl: false + # ssl: {enable: false} # } - # sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_acl where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'" + # sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'" # }, # { # type: redis @@ -34,9 +49,21 @@ authz:{ # pool_size: 1 # password: public # auto_reconnect: true - # ssl: false + # ssl: {enable: false} # } - # cmd: "HGETALL mqtt_acl:%u" + # cmd: "HGETALL mqtt_authz:%u" + # }, + # { + # type: mongo + # config: { + # mongo_type: single + # server: "127.0.0.1:27017" + # pool_size: 1 + # database: mqtt + # ssl: {enable: false} + # } + # collection: mqtt_authz + # find: { "$or": [ { "username": "%u" }, { "clientid": "%c" } ] } # }, { permission: allow diff --git a/apps/emqx_authz/include/emqx_authz.hrl b/apps/emqx_authz/include/emqx_authz.hrl index ca595525d..76aa20688 100644 --- a/apps/emqx_authz/include/emqx_authz.hrl +++ b/apps/emqx_authz/include/emqx_authz.hrl @@ -1,4 +1,4 @@ --type(rule() :: #{binary() => any()}). +-type(rule() :: #{atom() => any()}). -type(rules() :: [rule()]). -define(APP, emqx_authz). @@ -6,14 +6,14 @@ -define(ALLOW_DENY(A), ((A =:= allow) orelse (A =:= deny))). -define(PUBSUB(A), ((A =:= subscribe) orelse (A =:= publish) orelse (A =:= all))). --record(acl_metrics, { - allow = 'client.acl.allow', - deny = 'client.acl.deny', - ignore = 'client.acl.ignore' +-record(authz_metrics, { + allow = 'client.authorize.allow', + deny = 'client.authorize.deny', + ignore = 'client.authorize.ignore' }). -define(METRICS(Type), tl(tuple_to_list(#Type{}))). -define(METRICS(Type, K), #Type{}#Type.K). --define(ACL_METRICS, ?METRICS(acl_metrics)). --define(ACL_METRICS(K), ?METRICS(acl_metrics, K)). +-define(AUTHZ_METRICS, ?METRICS(authz_metrics)). +-define(AUTHZ_METRICS(K), ?METRICS(authz_metrics, K)). diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index 10801eca1..f79e10f85 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,6 +1,6 @@ {application, emqx_authz, [{description, "An OTP application"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 24393a4b0..51600ea81 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -15,6 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_authz). +-behaviour(emqx_config_handler). -include("emqx_authz.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -23,113 +24,166 @@ -export([ register_metrics/0 , init/0 - , compile/1 + , init_rule/1 , lookup/0 - , update/1 - , check_authz/5 + , update/2 + , authorize/5 , match/4 ]). +-export([post_config_update/3, pre_config_update/2]). + +-define(CONF_KEY_PATH, [authorization, rules]). + -spec(register_metrics() -> ok). register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?ACL_METRICS). + lists:foreach(fun emqx_metrics:ensure/1, ?AUTHZ_METRICS). init() -> ok = register_metrics(), - Conf = filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), - {ok, RawConf} = hocon:load(Conf), - #{<<"authz">> := #{<<"rules">> := Rules}} = hocon_schema:check_plain(emqx_authz_schema, RawConf), - ok = application:set_env(?APP, rules, Rules), - NRules = [compile(Rule) || Rule <- Rules], - ok = emqx_hooks:add('client.check_acl', {?MODULE, check_authz, [NRules]}, -1). + emqx_config_handler:add_handler(?CONF_KEY_PATH, ?MODULE), + NRules = [init_rule(Rule) || Rule <- lookup()], + ok = emqx_hooks:add('client.authorize', {?MODULE, authorize, [NRules]}, -1). lookup() -> - application:get_env(?APP, rules, []). + emqx_config:get(?CONF_KEY_PATH, []). -update(Rules) -> - ok = application:set_env(?APP, rules, Rules), - NRules = [compile(Rule) || Rule <- Rules], +update(Cmd, Rules) -> + emqx_config:update(emqx_authz_schema, ?CONF_KEY_PATH, {Cmd, Rules}). + +%% For now we only support re-creating the entire rule list +pre_config_update({head, Rule}, OldConf) when is_map(Rule), is_list(OldConf) -> + [Rule | OldConf]; +pre_config_update({tail, Rule}, OldConf) when is_map(Rule), is_list(OldConf) -> + OldConf ++ [Rule]; +pre_config_update({_, NewConf}, _OldConf) -> + %% overwrite the entire config! + case is_list(NewConf) of + true -> NewConf; + false -> [NewConf] + end. + +post_config_update(_, undefined, _OldConf) -> + %_ = [release_rules(Rule) || Rule <- OldConf], + ok; +post_config_update(_, NewRules, _OldConf) -> + %_ = [release_rules(Rule) || Rule <- OldConf], + InitedRules = [init_rule(Rule) || Rule <- NewRules], Action = find_action_in_hooks(), - ok = emqx_hooks:del('client.check_acl', Action), - ok = emqx_hooks:add('client.check_acl', {?MODULE, check_authz, [NRules]}, -1), - ok = emqx_acl_cache:empty_acl_cache(). + ok = emqx_hooks:del('client.authorize', Action), + ok = emqx_hooks:add('client.authorize', {?MODULE, authorize, [InitedRules]}, -1), + ok = emqx_authz_cache:drain_cache(). %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- find_action_in_hooks() -> - Callbacks = emqx_hooks:lookup('client.check_acl'), - [Action] = [Action || {callback,{?MODULE, check_authz, _} = Action, _, _} <- Callbacks ], + Callbacks = emqx_hooks:lookup('client.authorize'), + [Action] = [Action || {callback,{?MODULE, authorize, _} = Action, _, _} <- Callbacks ], Action. -create_resource(#{<<"type">> := DB, - <<"config">> := Config - } = Rule) -> - ResourceID = iolist_to_binary([io_lib:format("~s_~s",[?APP, DB]), "_", integer_to_list(erlang:system_time())]), - case emqx_resource:check_and_create( +gen_id(Type) -> + iolist_to_binary([io_lib:format("~s_~s",[?APP, Type]), "_", integer_to_list(erlang:system_time())]). + +create_resource(#{type := DB, + config := Config}) -> + ResourceID = gen_id(DB), + case emqx_resource:create( ResourceID, list_to_existing_atom(io_lib:format("~s_~s",[emqx_connector, DB])), - #{<<"config">> => Config }) + Config) of - {ok, _} -> - Rule#{<<"resource_id">> => ResourceID}; - {error, already_created} -> - Rule#{<<"resource_id">> => ResourceID}; - {error, Reason} -> - error({load_config_error, Reason}) + {ok, _} -> ResourceID; + {error, already_created} -> ResourceID; + {error, Reason} -> {error, Reason} end. --spec(compile(rule()) -> rule()). -compile(#{<<"topics">> := Topics, - <<"action">> := Action, - <<"permission">> := Permission, - <<"principal">> := Principal +-spec(init_rule(rule()) -> rule()). +init_rule(#{topics := Topics, + action := Action, + permission := Permission, + principal := Principal } = Rule) when ?ALLOW_DENY(Permission), ?PUBSUB(Action), is_list(Topics) -> - NTopics = [compile_topic(Topic) || Topic <- Topics], - Rule#{<<"principal">> => compile_principal(Principal), - <<"topics">> => NTopics + Rule#{annotations => + #{id => gen_id(simple), + principal => compile_principal(Principal), + topics => [compile_topic(Topic) || Topic <- Topics]} }; -compile(#{<<"principal">> := Principal, - <<"type">> := redis - } = Rule) -> - NRule = create_resource(Rule), - NRule#{<<"principal">> => compile_principal(Principal)}; +init_rule(#{principal := Principal, + enable := true, + type := http, + config := #{url := Url} = Config + } = Rule) -> + NConfig = maps:merge(Config, #{base_url => maps:remove(query, Url)}), + case create_resource(Rule#{config := NConfig}) of + {error, Reason} -> error({load_config_error, Reason}); + Id -> Rule#{annotations => + #{id => Id, + principal => compile_principal(Principal) + } + } + end; -compile(#{<<"principal">> := Principal, - <<"type">> := DB, - <<"sql">> := SQL +init_rule(#{principal := Principal, + enable := true, + type := DB + } = Rule) when DB =:= redis; + DB =:= mongo -> + case create_resource(Rule) of + {error, Reason} -> error({load_config_error, Reason}); + Id -> Rule#{annotations => + #{id => Id, + principal => compile_principal(Principal) + } + } + end; + +init_rule(#{principal := Principal, + enable := true, + type := DB, + sql := SQL } = Rule) when DB =:= mysql; DB =:= pgsql -> Mod = list_to_existing_atom(io_lib:format("~s_~s",[?APP, DB])), - NRule = create_resource(Rule), - NRule#{<<"principal">> => compile_principal(Principal), - <<"sql">> => Mod:parse_query(SQL) - }. + case create_resource(Rule) of + {error, Reason} -> error({load_config_error, Reason}); + Id -> Rule#{annotations => + #{id => Id, + principal => compile_principal(Principal), + sql => Mod:parse_query(SQL) + } + } + end; + +init_rule(#{enable := false, + type := _DB + } = Rule) -> + Rule. compile_principal(all) -> all; -compile_principal(#{<<"username">> := Username}) -> +compile_principal(#{username := Username}) -> {ok, MP} = re:compile(bin(Username)), - #{<<"username">> => MP}; -compile_principal(#{<<"clientid">> := Clientid}) -> + #{username => MP}; +compile_principal(#{clientid := Clientid}) -> {ok, MP} = re:compile(bin(Clientid)), - #{<<"clientid">> => MP}; -compile_principal(#{<<"ipaddress">> := IpAddress}) -> - #{<<"ipaddress">> => esockd_cidr:parse(b2l(IpAddress), true)}; -compile_principal(#{<<"and">> := Principals}) when is_list(Principals) -> - #{<<"and">> => [compile_principal(Principal) || Principal <- Principals]}; -compile_principal(#{<<"or">> := Principals}) when is_list(Principals) -> - #{<<"or">> => [compile_principal(Principal) || Principal <- Principals]}. + #{clientid => MP}; +compile_principal(#{ipaddress := IpAddress}) -> + #{ipaddress => esockd_cidr:parse(b2l(IpAddress), true)}; +compile_principal(#{'and' := Principals}) when is_list(Principals) -> + #{'and' => [compile_principal(Principal) || Principal <- Principals]}; +compile_principal(#{'or' := Principals}) when is_list(Principals) -> + #{'or' => [compile_principal(Principal) || Principal <- Principals]}. compile_topic(<<"eq ", Topic/binary>>) -> - compile_topic(#{<<"eq">> => Topic}); -compile_topic(#{<<"eq">> := Topic}) -> - #{<<"eq">> => emqx_topic:words(bin(Topic))}; + compile_topic(#{'eq' => Topic}); +compile_topic(#{'eq' := Topic}) -> + #{'eq' => emqx_topic:words(bin(Topic))}; compile_topic(Topic) when is_binary(Topic)-> Words = emqx_topic:words(bin(Topic)), case pattern(Words) of - true -> #{<<"pattern">> => Words}; + true -> #{pattern => Words}; false -> Words end. @@ -145,53 +199,57 @@ b2l(B) when is_list(B) -> B; b2l(B) when is_binary(B) -> binary_to_list(B). %%-------------------------------------------------------------------- -%% ACL callbacks +%% AuthZ callbacks %%-------------------------------------------------------------------- -%% @doc Check ACL --spec(check_authz(emqx_types:clientinfo(), emqx_types:all(), emqx_topic:topic(), emqx_permission_rule:acl_result(), rules()) - -> {ok, allow} | {ok, deny} | deny). -check_authz(#{username := Username, - peerhost := IpAddress - } = Client, PubSub, Topic, DefaultResult, Rules) -> - case do_check_authz(Client, PubSub, Topic, Rules) of +%% @doc Check AuthZ +-spec(authorize(emqx_types:clientinfo(), emqx_types:all(), emqx_topic:topic(), allow | deny, rules()) + -> {stop, allow} | {ok, deny}). +authorize(#{username := Username, + peerhost := IpAddress + } = Client, PubSub, Topic, _DefaultResult, Rules) -> + case do_authorize(Client, PubSub, Topic, Rules) of {matched, allow} -> - ?LOG(info, "Client succeeded authorizationa: Username: ~p, IP: ~p, Topic: ~p, Permission: allow", [Username, IpAddress, Topic]), - emqx_metrics:inc(?ACL_METRICS(allow)), + ?LOG(info, "Client succeeded authorization: Username: ~p, IP: ~p, Topic: ~p, Permission: allow", [Username, IpAddress, Topic]), + emqx_metrics:inc(?AUTHZ_METRICS(allow)), {stop, allow}; {matched, deny} -> - ?LOG(info, "Client failed authorizationa: Username: ~p, IP: ~p, Topic: ~p, Permission: deny", [Username, IpAddress, Topic]), - emqx_metrics:inc(?ACL_METRICS(deny)), + ?LOG(info, "Client failed authorization: Username: ~p, IP: ~p, Topic: ~p, Permission: deny", [Username, IpAddress, Topic]), + emqx_metrics:inc(?AUTHZ_METRICS(deny)), {stop, deny}; nomatch -> - ?LOG(info, "Client failed authorizationa: Username: ~p, IP: ~p, Topic: ~p, Reasion: ~p", [Username, IpAddress, Topic, "no-match rule"]), - DefaultResult + ?LOG(info, "Client failed authorization: Username: ~p, IP: ~p, Topic: ~p, Reasion: ~p", [Username, IpAddress, Topic, "no-match rule"]), + {stop, deny} end. -do_check_authz(Client, PubSub, Topic, - [Connector = #{<<"principal">> := Principal, - <<"type">> := DB} | Tail] ) -> +do_authorize(Client, PubSub, Topic, + [Connector = #{type := DB, + enable := true, + annotations := #{principal := Principal} + } | Tail] ) -> case match_principal(Client, Principal) of true -> Mod = list_to_existing_atom(io_lib:format("~s_~s",[emqx_authz, DB])), - case Mod:check_authz(Client, PubSub, Topic, Connector) of - nomatch -> do_check_authz(Client, PubSub, Topic, Tail); + case Mod:authorize(Client, PubSub, Topic, Connector) of + nomatch -> do_authorize(Client, PubSub, Topic, Tail); Matched -> Matched end; - false -> do_check_authz(Client, PubSub, Topic, Tail) + false -> do_authorize(Client, PubSub, Topic, Tail) end; -do_check_authz(Client, PubSub, Topic, - [#{<<"permission">> := Permission} = Rule | Tail]) -> +do_authorize(Client, PubSub, Topic, + [#{permission := Permission} = Rule | Tail]) -> case match(Client, PubSub, Topic, Rule) of true -> {matched, Permission}; - false -> do_check_authz(Client, PubSub, Topic, Tail) + false -> do_authorize(Client, PubSub, Topic, Tail) end; -do_check_authz(_Client, _PubSub, _Topic, []) -> nomatch. +do_authorize(_Client, _PubSub, _Topic, []) -> nomatch. match(Client, PubSub, Topic, - #{<<"principal">> := Principal, - <<"topics">> := TopicFilters, - <<"action">> := Action + #{action := Action, + annotations := #{ + principal := Principal, + topics := TopicFilters + } }) -> match_action(PubSub, Action) andalso match_principal(Client, Principal) andalso @@ -203,27 +261,27 @@ match_action(_, all) -> true; match_action(_, _) -> false. match_principal(_, all) -> true; -match_principal(#{username := undefined}, #{<<"username">> := _MP}) -> +match_principal(#{username := undefined}, #{username := _MP}) -> false; -match_principal(#{username := Username}, #{<<"username">> := MP}) -> +match_principal(#{username := Username}, #{username := MP}) -> case re:run(Username, MP) of {match, _} -> true; _ -> false end; -match_principal(#{clientid := Clientid}, #{<<"clientid">> := MP}) -> +match_principal(#{clientid := Clientid}, #{clientid := MP}) -> case re:run(Clientid, MP) of {match, _} -> true; _ -> false end; -match_principal(#{peerhost := undefined}, #{<<"ipaddress">> := _CIDR}) -> +match_principal(#{peerhost := undefined}, #{ipaddress := _CIDR}) -> false; -match_principal(#{peerhost := IpAddress}, #{<<"ipaddress">> := CIDR}) -> +match_principal(#{peerhost := IpAddress}, #{ipaddress := CIDR}) -> esockd_cidr:match(IpAddress, CIDR); -match_principal(ClientInfo, #{<<"and">> := Principals}) when is_list(Principals) -> +match_principal(ClientInfo, #{'and' := Principals}) when is_list(Principals) -> lists:foldl(fun(Principal, Permission) -> match_principal(ClientInfo, Principal) andalso Permission end, true, Principals); -match_principal(ClientInfo, #{<<"or">> := Principals}) when is_list(Principals) -> +match_principal(ClientInfo, #{'or' := Principals}) when is_list(Principals) -> lists:foldl(fun(Principal, Permission) -> match_principal(ClientInfo, Principal) orelse Permission end, false, Principals); @@ -231,7 +289,7 @@ match_principal(_, _) -> false. match_topics(_ClientInfo, _Topic, []) -> false; -match_topics(ClientInfo, Topic, [#{<<"pattern">> := PatternFilter}|Filters]) -> +match_topics(ClientInfo, Topic, [#{pattern := PatternFilter}|Filters]) -> TopicFilter = feed_var(ClientInfo, PatternFilter), match_topic(emqx_topic:words(Topic), TopicFilter) orelse match_topics(ClientInfo, Topic, Filters); @@ -239,7 +297,7 @@ match_topics(ClientInfo, Topic, [TopicFilter|Filters]) -> match_topic(emqx_topic:words(Topic), TopicFilter) orelse match_topics(ClientInfo, Topic, Filters). -match_topic(Topic, #{<<"eq">> := TopicFilter}) -> +match_topic(Topic, #{'eq' := TopicFilter}) -> Topic == TopicFilter; match_topic(Topic, TopicFilter) -> emqx_topic:match(Topic, TopicFilter). diff --git a/apps/emqx_authz/src/emqx_authz_api.erl b/apps/emqx_authz/src/emqx_authz_api.erl index 6b4ed6c74..974f72dbe 100644 --- a/apps/emqx_authz/src/emqx_authz_api.erl +++ b/apps/emqx_authz/src/emqx_authz_api.erl @@ -53,32 +53,26 @@ ]). lookup_authz(_Bindings, _Params) -> - minirest:return({ok, emqx_authz:lookup()}). + return({ok, emqx_authz:lookup()}). update_authz(_Bindings, Params) -> - Rules = get_rules(Params), - minirest:return(emqx_authz:update(Rules)). + Rules = form_rules(Params), + return(emqx_authz:update(replace, Rules)). append_authz(_Bindings, Params) -> - Rules = get_rules(Params), - NRules = lists:append(emqx_authz:lookup(), Rules), - minirest:return(emqx_authz:update(NRules)). + Rules = form_rules(Params), + return(emqx_authz:update(tail, Rules)). push_authz(_Bindings, Params) -> - Rules = get_rules(Params), - NRules = lists:append(Rules, emqx_authz:lookup()), - minirest:return(emqx_authz:update(NRules)). + Rules = form_rules(Params), + return(emqx_authz:update(head, Rules)). %%------------------------------------------------------------------------------ %% Interval Funcs %%------------------------------------------------------------------------------ -get_rules(Params) -> - % #{<<"authz">> := #{<<"rules">> := Rules}} = hocon_schema:check_plain(emqx_authz_schema, #{<<"authz">> => Params}), - {ok, Conf} = hocon:binary(jsx:encode(#{<<"authz">> => Params}), #{format => richmap}), - CheckConf = hocon_schema:check(emqx_authz_schema, Conf), - #{<<"authz">> := #{<<"rules">> := Rules}} = hocon_schema:richmap_to_map(CheckConf), - Rules. +form_rules(Params) -> + Params. %%-------------------------------------------------------------------- %% EUnits @@ -89,3 +83,7 @@ get_rules(Params) -> -endif. + +return(_) -> +%% TODO: V5 api + ok. \ No newline at end of file diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl new file mode 100644 index 000000000..c95d200e1 --- /dev/null +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -0,0 +1,99 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_http). + +-include("emqx_authz.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% AuthZ Callbacks +-export([ authorize/4 + , description/0 + ]). + +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-endif. + +description() -> + "AuthZ with http". + +authorize(Client, PubSub, Topic, + #{type := http, + config := #{url := #{path := Path} = Url, + headers := Headers, + method := Method, + request_timeout := RequestTimeout} = Config, + annotations := #{id := ResourceID} + }) -> + Request = case Method of + get -> + Query = maps:get(query, Url, ""), + Path1 = replvar(Path ++ "?" ++ Query, PubSub, Topic, Client), + {Path1, maps:to_list(Headers)}; + _ -> + Body0 = serialize_body( + maps:get('Accept', Headers, <<"application/json">>), + maps:get(body, Config, #{}) + ), + Body1 = replvar(Body0, PubSub, Topic, Client), + Path1 = replvar(Path, PubSub, Topic, Client), + {Path1, maps:to_list(Headers), Body1} + end, + case emqx_resource:query(ResourceID, {Method, Request, RequestTimeout}) of + {ok, 204, _Headers} -> {matched, allow}; + {ok, 200, _Headers, _Body} -> {matched, allow}; + _ -> nomatch + end. + +query_string(Body) -> + query_string(maps:to_list(Body), []). + +query_string([], Acc) -> + <<$&, Str/binary>> = iolist_to_binary(lists:reverse(Acc)), + Str; +query_string([{K, V} | More], Acc) -> + query_string(More, [["&", emqx_http_lib:uri_encode(K), "=", emqx_http_lib:uri_encode(V)] | Acc]). + +serialize_body(<<"application/json">>, Body) -> + jsx:encode(Body); +serialize_body(<<"application/x-www-form-urlencoded">>, Body) -> + query_string(Body). + +replvar(Str0, PubSub, Topic, + #{username := Username, + clientid := Clientid, + peerhost := IpAddress, + protocol := Protocol, + mountpoint := Mountpoint + }) when is_list(Str0); + is_binary(Str0) -> + NTopic = emqx_http_lib:uri_encode(Topic), + Str1 = re:replace(Str0, "%c", Clientid, [global, {return, binary}]), + Str2 = re:replace(Str1, "%u", bin(Username), [global, {return, binary}]), + Str3 = re:replace(Str2, "%a", inet_parse:ntoa(IpAddress), [global, {return, binary}]), + Str4 = re:replace(Str3, "%r", bin(Protocol), [global, {return, binary}]), + Str5 = re:replace(Str4, "%m", Mountpoint, [global, {return, binary}]), + Str6 = re:replace(Str5, "%t", NTopic, [global, {return, binary}]), + Str7 = re:replace(Str6, "%A", bin(PubSub), [global, {return, binary}]), + Str7. + +bin(A) when is_atom(A) -> atom_to_binary(A, utf8); +bin(B) when is_binary(B) -> B; +bin(L) when is_list(L) -> list_to_binary(L); +bin(X) -> X. diff --git a/apps/emqx_authz/src/emqx_authz_mongo.erl b/apps/emqx_authz/src/emqx_authz_mongo.erl new file mode 100644 index 000000000..c015f8208 --- /dev/null +++ b/apps/emqx_authz/src/emqx_authz_mongo.erl @@ -0,0 +1,104 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_mongo). + +-include("emqx_authz.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% AuthZ Callbacks +-export([ authorize/4 + , description/0 + ]). + +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-endif. + +description() -> + "AuthZ with Mongo". + +authorize(Client, PubSub, Topic, + #{collection := Collection, + find := Find, + annotations := #{id := ResourceID} + }) -> + case emqx_resource:query(ResourceID, {find, Collection, replvar(Find, Client), #{}}) of + {error, Reason} -> + ?LOG(error, "[AuthZ] Query mongo error: ~p", [Reason]), + nomatch; + [] -> nomatch; + Rows -> + do_authorize(Client, PubSub, Topic, Rows) + end. + +do_authorize(_Client, _PubSub, _Topic, []) -> + nomatch; +do_authorize(Client, PubSub, Topic, [Rule | Tail]) -> + case match(Client, PubSub, Topic, Rule) of + {matched, Permission} -> {matched, Permission}; + nomatch -> do_authorize(Client, PubSub, Topic, Tail) + end. + +match(Client, PubSub, Topic, + #{<<"topics">> := Topics, + <<"permission">> := Permission, + <<"action">> := Action + }) -> + Rule = #{<<"permission">> => Permission, + <<"topics">> => Topics, + <<"action">> => Action + }, + #{simple_rule := + #{permission := NPermission} = NRule + } = hocon_schema:check_plain( + emqx_authz_schema, + #{<<"simple_rule">> => Rule}, + #{atom_key => true}, + [simple_rule]), + case emqx_authz:match(Client, PubSub, Topic, emqx_authz:init_rule(NRule)) of + true -> {matched, NPermission}; + false -> nomatch + end. + +replvar(Find, #{clientid := Clientid, + username := Username, + peerhost := IpAddress + }) -> + Fun = fun + _Fun(K, V, AccIn) when is_map(V) -> maps:put(K, maps:fold(_Fun, AccIn, V), AccIn); + _Fun(K, V, AccIn) when is_list(V) -> + maps:put(K, [ begin + [{K1, V1}] = maps:to_list(M), + _Fun(K1, V1, AccIn) + end || M <- V], + AccIn); + _Fun(K, V, AccIn) when is_binary(V) -> + V1 = re:replace(V, "%c", bin(Clientid), [global, {return, binary}]), + V2 = re:replace(V1, "%u", bin(Username), [global, {return, binary}]), + V3 = re:replace(V2, "%a", inet_parse:ntoa(IpAddress), [global, {return, binary}]), + maps:put(K, V3, AccIn); + _Fun(K, V, AccIn) -> maps:put(K, V, AccIn) + end, + maps:fold(Fun, #{}, Find). + +bin(A) when is_atom(A) -> atom_to_binary(A, utf8); +bin(B) when is_binary(B) -> B; +bin(L) when is_list(L) -> list_to_binary(L); +bin(X) -> X. + diff --git a/apps/emqx_authz/src/emqx_authz_mysql.erl b/apps/emqx_authz/src/emqx_authz_mysql.erl index c1ab20125..2ce991eba 100644 --- a/apps/emqx_authz/src/emqx_authz_mysql.erl +++ b/apps/emqx_authz/src/emqx_authz_mysql.erl @@ -20,10 +20,10 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). -%% ACL Callbacks +%% AuthZ Callbacks -export([ description/0 , parse_query/1 - , check_authz/4 + , authorize/4 ]). -ifdef(TEST). @@ -45,25 +45,26 @@ parse_query(Sql) -> {Sql, []} end. -check_authz(Client, PubSub, Topic, - #{<<"resource_id">> := ResourceID, - <<"sql">> := {SQL, Params} +authorize(Client, PubSub, Topic, + #{annotations := #{id := ResourceID, + sql := {SQL, Params} + } }) -> case emqx_resource:query(ResourceID, {sql, SQL, replvar(Params, Client)}) of {ok, _Columns, []} -> nomatch; {ok, Columns, Rows} -> - do_check_authz(Client, PubSub, Topic, Columns, Rows); + do_authorize(Client, PubSub, Topic, Columns, Rows); {error, Reason} -> ?LOG(error, "[AuthZ] Query mysql error: ~p~n", [Reason]), nomatch end. -do_check_authz(_Client, _PubSub, _Topic, _Columns, []) -> +do_authorize(_Client, _PubSub, _Topic, _Columns, []) -> nomatch; -do_check_authz(Client, PubSub, Topic, Columns, [Row | Tail]) -> +do_authorize(Client, PubSub, Topic, Columns, [Row | Tail]) -> case match(Client, PubSub, Topic, format_result(Columns, Row)) of {matched, Permission} -> {matched, Permission}; - nomatch -> do_check_authz(Client, PubSub, Topic, Columns, Tail) + nomatch -> do_authorize(Client, PubSub, Topic, Columns, Tail) end. format_result(Columns, Row) -> @@ -77,41 +78,24 @@ format_result(Columns, Row) -> match(Client, PubSub, Topic, #{<<"permission">> := Permission, <<"action">> := Action, - <<"clientid">> := ClientId, - <<"username">> := Username, - <<"ipaddress">> := IpAddress, <<"topic">> := TopicFilter }) -> - Rule = #{<<"principal">> => principal(IpAddress, Username, ClientId), - <<"topics">> => [TopicFilter], + Rule = #{<<"topics">> => [TopicFilter], <<"action">> => Action, <<"permission">> => Permission }, - #{<<"simple_rule">> := - #{<<"permission">> := NPermission} = NRule + #{simple_rule := + #{permission := NPermission} = NRule } = hocon_schema:check_plain( emqx_authz_schema, #{<<"simple_rule">> => Rule}, - #{}, + #{atom_key => true}, [simple_rule]), - case emqx_authz:match(Client, PubSub, Topic, emqx_authz:compile(NRule)) of + case emqx_authz:match(Client, PubSub, Topic, emqx_authz:init_rule(NRule)) of true -> {matched, NPermission}; false -> nomatch end. -principal(CIDR, Username, ClientId) -> - Cols = [{<<"ipaddress">>, CIDR}, {<<"username">>, Username}, {<<"clientid">>, ClientId}], - case [#{C => V} || {C, V} <- Cols, not empty(V)] of - [] -> throw(undefined_who); - [Who] -> Who; - Conds -> #{<<"and">> => Conds} - end. - -empty(null) -> true; -empty("") -> true; -empty(<<>>) -> true; -empty(_) -> false. - replvar(Params, ClientInfo) -> replvar(Params, ClientInfo, []). diff --git a/apps/emqx_authz/src/emqx_authz_pgsql.erl b/apps/emqx_authz/src/emqx_authz_pgsql.erl index edea8102f..f3e793763 100644 --- a/apps/emqx_authz/src/emqx_authz_pgsql.erl +++ b/apps/emqx_authz/src/emqx_authz_pgsql.erl @@ -20,10 +20,10 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). -%% ACL Callbacks +%% AuthZ Callbacks -export([ description/0 , parse_query/1 - , check_authz/4 + , authorize/4 ]). -ifdef(TEST). @@ -49,25 +49,26 @@ parse_query(Sql) -> {Sql, []} end. -check_authz(Client, PubSub, Topic, - #{<<"resource_id">> := ResourceID, - <<"sql">> := {SQL, Params} +authorize(Client, PubSub, Topic, + #{annotations := #{id := ResourceID, + sql := {SQL, Params} + } }) -> case emqx_resource:query(ResourceID, {sql, SQL, replvar(Params, Client)}) of {ok, _Columns, []} -> nomatch; {ok, Columns, Rows} -> - do_check_authz(Client, PubSub, Topic, Columns, Rows); + do_authorize(Client, PubSub, Topic, Columns, Rows); {error, Reason} -> ?LOG(error, "[AuthZ] Query pgsql error: ~p~n", [Reason]), nomatch end. -do_check_authz(_Client, _PubSub, _Topic, _Columns, []) -> +do_authorize(_Client, _PubSub, _Topic, _Columns, []) -> nomatch; -do_check_authz(Client, PubSub, Topic, Columns, [Row | Tail]) -> +do_authorize(Client, PubSub, Topic, Columns, [Row | Tail]) -> case match(Client, PubSub, Topic, format_result(Columns, Row)) of {matched, Permission} -> {matched, Permission}; - nomatch -> do_check_authz(Client, PubSub, Topic, Columns, Tail) + nomatch -> do_authorize(Client, PubSub, Topic, Columns, Tail) end. format_result(Columns, Row) -> @@ -81,41 +82,24 @@ format_result(Columns, Row) -> match(Client, PubSub, Topic, #{<<"permission">> := Permission, <<"action">> := Action, - <<"clientid">> := ClientId, - <<"username">> := Username, - <<"ipaddress">> := IpAddress, <<"topic">> := TopicFilter }) -> - Rule = #{<<"principal">> => principal(IpAddress, Username, ClientId), - <<"topics">> => [TopicFilter], + Rule = #{<<"topics">> => [TopicFilter], <<"action">> => Action, <<"permission">> => Permission }, - #{<<"simple_rule">> := - #{<<"permission">> := NPermission} = NRule + #{simple_rule := + #{permission := NPermission} = NRule } = hocon_schema:check_plain( emqx_authz_schema, #{<<"simple_rule">> => Rule}, - #{}, + #{atom_key => true}, [simple_rule]), - case emqx_authz:match(Client, PubSub, Topic, emqx_authz:compile(NRule)) of + case emqx_authz:match(Client, PubSub, Topic, emqx_authz:init_rule(NRule)) of true -> {matched, NPermission}; false -> nomatch end. -principal(CIDR, Username, ClientId) -> - Cols = [{<<"ipaddress">>, CIDR}, {<<"username">>, Username}, {<<"clientid">>, ClientId}], - case [#{C => V} || {C, V} <- Cols, not empty(V)] of - [] -> throw(undefined_who); - [Who] -> Who; - Conds -> #{<<"and">> => Conds} - end. - -empty(null) -> true; -empty("") -> true; -empty(<<>>) -> true; -empty(_) -> false. - replvar(Params, ClientInfo) -> replvar(Params, ClientInfo, []). diff --git a/apps/emqx_authz/src/emqx_authz_redis.erl b/apps/emqx_authz/src/emqx_authz_redis.erl index 7a85b26af..8f6731fd8 100644 --- a/apps/emqx_authz/src/emqx_authz_redis.erl +++ b/apps/emqx_authz/src/emqx_authz_redis.erl @@ -20,8 +20,8 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). -%% ACL Callbacks --export([ check_authz/4 +%% AuthZ Callbacks +-export([ authorize/4 , description/0 ]). @@ -33,33 +33,33 @@ description() -> "AuthZ with redis". -check_authz(Client, PubSub, Topic, - #{<<"resource_id">> := ResourceID, - <<"cmd">> := CMD +authorize(Client, PubSub, Topic, + #{cmd := CMD, + annotations := #{id := ResourceID} }) -> NCMD = string:tokens(replvar(CMD, Client), " "), case emqx_resource:query(ResourceID, {cmd, NCMD}) of {ok, []} -> nomatch; {ok, Rows} -> - do_check_authz(Client, PubSub, Topic, Rows); + do_authorize(Client, PubSub, Topic, Rows); {error, Reason} -> ?LOG(error, "[AuthZ] Query redis error: ~p", [Reason]), nomatch end. -do_check_authz(_Client, _PubSub, _Topic, []) -> +do_authorize(_Client, _PubSub, _Topic, []) -> nomatch; -do_check_authz(Client, PubSub, Topic, [TopicFilter, Action | Tail]) -> - case match(Client, PubSub, Topic, +do_authorize(Client, PubSub, Topic, [TopicFilter, Action | Tail]) -> + case match(Client, PubSub, Topic, #{topics => TopicFilter, action => Action - }) + }) of {matched, Permission} -> {matched, Permission}; - nomatch -> do_check_authz(Client, PubSub, Topic, Tail) + nomatch -> do_authorize(Client, PubSub, Topic, Tail) end. -match(Client, PubSub, Topic, +match(Client, PubSub, Topic, #{topics := TopicFilter, action := Action }) -> @@ -68,13 +68,13 @@ match(Client, PubSub, Topic, <<"action">> => Action, <<"permission">> => allow }, - #{<<"simple_rule">> := NRule + #{simple_rule := NRule } = hocon_schema:check_plain( emqx_authz_schema, #{<<"simple_rule">> => Rule}, - #{}, + #{atom_key => true}, [simple_rule]), - case emqx_authz:match(Client, PubSub, Topic, emqx_authz:compile(NRule)) of + case emqx_authz:match(Client, PubSub, Topic, emqx_authz:init_rule(NRule)) of true -> {matched, allow}; false -> nomatch end. diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 04d1b2268..cc109534f 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -2,44 +2,110 @@ -include_lib("typerefl/include/types.hrl"). --type action() :: publish | subscribe | all. --type permission() :: allow | deny. - -reflect_type([ permission/0 , action/0 + , url/0 ]). --export([structs/0, fields/1]). +-typerefl_from_string({url/0, emqx_http_lib, uri_parse}). -structs() -> [authz]. +-type action() :: publish | subscribe | all. +-type permission() :: allow | deny. +-type url() :: emqx_http_lib:uri_map(). -fields(authz) -> +-export([ structs/0 + , fields/1 + ]). + +structs() -> ["authorization"]. + +fields("authorization") -> [ {rules, rules()} ]; -fields(redis_connector) -> +fields(http) -> [ {principal, principal()} - , {type, #{type => hoconsc:enum([redis])}} - , {config, #{type => hoconsc:union( - [ hoconsc:ref(emqx_connector_redis, cluster) - , hoconsc:ref(emqx_connector_redis, sentinel) - , hoconsc:ref(emqx_connector_redis, single) - ])} + , {type, #{type => http}} + , {enable, #{type => boolean(), + default => true}} + , {config, #{type => hoconsc:union([ hoconsc:ref(?MODULE, http_get) + , hoconsc:ref(?MODULE, http_post) + ])} } - , {cmd, query()} ]; - -fields(sql_connector) -> - [ {principal, principal() } - , {type, #{type => hoconsc:enum([mysql, pgsql])}} - , {config, #{type => map()}} - , {sql, query()} +fields(http_get) -> + [ {url, #{type => url()}} + , {headers, #{type => map(), + default => #{ <<"accept">> => <<"application/json">> + , <<"cache-control">> => <<"no-cache">> + , <<"connection">> => <<"keep-alive">> + , <<"keep-alive">> => <<"timeout=5">> + }, + converter => fun (Headers0) -> + Headers1 = maps:fold(fun(K0, V, AccIn) -> + K1 = iolist_to_binary(string:to_lower(binary_to_list(K0))), + maps:put(K1, V, AccIn) + end, #{}, Headers0), + maps:merge(#{ <<"accept">> => <<"application/json">> + , <<"cache-control">> => <<"no-cache">> + , <<"connection">> => <<"keep-alive">> + , <<"keep-alive">> => <<"timeout=5">> + }, Headers1) + end + } + } + , {method, #{type => get, default => get }} + , {request_timeout, #{type => timeout(), default => 30000 }} + ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)); +fields(http_post) -> + [ {url, #{type => url()}} + , {headers, #{type => map(), + default => #{ <<"accept">> => <<"application/json">> + , <<"cache-control">> => <<"no-cache">> + , <<"connection">> => <<"keep-alive">> + , <<"content-type">> => <<"application/json">> + , <<"keep-alive">> => <<"timeout=5">> + }, + converter => fun (Headers0) -> + Headers1 = maps:fold(fun(K0, V, AccIn) -> + K1 = iolist_to_binary(string:to_lower(binary_to_list(K0))), + maps:put(K1, V, AccIn) + end, #{}, Headers0), + maps:merge(#{ <<"accept">> => <<"application/json">> + , <<"cache-control">> => <<"no-cache">> + , <<"connection">> => <<"keep-alive">> + , <<"content-type">> => <<"application/json">> + , <<"keep-alive">> => <<"timeout=5">> + }, Headers1) + end + } + } + , {method, #{type => hoconsc:enum([post, put]), + default => get}} + , {body, #{type => map(), + nullable => true + } + } + ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)); +fields(mongo) -> + connector_fields(mongo) ++ + [ {collection, #{type => atom()}} + , {find, #{type => map()}} ]; +fields(redis) -> + connector_fields(redis) ++ + [ {cmd, query()} ]; +fields(mysql) -> + connector_fields(mysql) ++ + [ {sql, query()} ]; +fields(pgsql) -> + connector_fields(pgsql) ++ + [ {sql, query()} ]; fields(simple_rule) -> [ {permission, #{type => permission()}} , {action, #{type => action()}} , {topics, #{type => union_array( [ binary() - , hoconsc:ref(eq_topic) + , hoconsc:ref(?MODULE, eq_topic) ] )}} , {principal, principal()} @@ -52,18 +118,18 @@ fields(ipaddress) -> [{ipaddress, #{type => string()}}]; fields(andlist) -> [{'and', #{type => union_array( - [ hoconsc:ref(username) - , hoconsc:ref(clientid) - , hoconsc:ref(ipaddress) + [ hoconsc:ref(?MODULE, username) + , hoconsc:ref(?MODULE, clientid) + , hoconsc:ref(?MODULE, ipaddress) ]) } } ]; fields(orlist) -> [{'or', #{type => union_array( - [ hoconsc:ref(username) - , hoconsc:ref(clientid) - , hoconsc:ref(ipaddress) + [ hoconsc:ref(?MODULE, username) + , hoconsc:ref(?MODULE, clientid) + , hoconsc:ref(?MODULE, ipaddress) ]) } } @@ -79,11 +145,14 @@ fields(eq_topic) -> union_array(Item) when is_list(Item) -> hoconsc:array(hoconsc:union(Item)). -rules() -> +rules() -> #{type => union_array( - [ hoconsc:ref(simple_rule) - , hoconsc:ref(sql_connector) - , hoconsc:ref(redis_connector) + [ hoconsc:ref(?MODULE, simple_rule) + , hoconsc:ref(?MODULE, http) + , hoconsc:ref(?MODULE, mysql) + , hoconsc:ref(?MODULE, pgsql) + , hoconsc:ref(?MODULE, redis) + , hoconsc:ref(?MODULE, mongo) ]) }. @@ -91,11 +160,11 @@ principal() -> #{default => all, type => hoconsc:union( [ all - , hoconsc:ref(username) - , hoconsc:ref(clientid) - , hoconsc:ref(ipaddress) - , hoconsc:ref(andlist) - , hoconsc:ref(orlist) + , hoconsc:ref(?MODULE, username) + , hoconsc:ref(?MODULE, clientid) + , hoconsc:ref(?MODULE, ipaddress) + , hoconsc:ref(?MODULE, andlist) + , hoconsc:ref(?MODULE, orlist) ]) }. @@ -108,3 +177,19 @@ query() -> end end }. + +connector_fields(DB) -> + Mod0 = io_lib:format("~s_~s",[emqx_connector, DB]), + Mod = try + list_to_existing_atom(Mod0) + catch + error:badarg -> + list_to_atom(Mod0); + Error -> + erlang:error(Error) + end, + [ {principal, principal()} + , {type, #{type => DB}} + , {enable, #{type => boolean(), + default => true}} + ] ++ Mod:fields(""). diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index 88e250377..dd3f38519 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -29,140 +29,132 @@ groups() -> []. init_per_suite(Config) -> - ok = emqx_ct_helpers:start_apps([emqx_authz], fun set_special_configs/1), + ok = emqx_ct_helpers:start_apps([emqx_authz]), + ok = emqx_config:update([zones, default, authorization, cache, enable], false), + ok = emqx_config:update([zones, default, authorization, enable], true), + emqx_authz:update(replace, []), Config. end_per_suite(_Config) -> - file:delete(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf')), emqx_ct_helpers:stop_apps([emqx_authz]). -set_special_configs(emqx) -> - application:set_env(emqx, allow_anonymous, true), - application:set_env(emqx, enable_acl_cache, false), - application:set_env(emqx, acl_nomatch, deny), - ok; -set_special_configs(emqx_authz) -> - application:set_env(emqx, plugins_etc_dir, - emqx_ct_helpers:deps_path(emqx_authz, "test")), - Conf = #{<<"authz">> => #{<<"rules">> => []}}, - ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)), - ok; -set_special_configs(_App) -> - ok. - --define(RULE1, #{<<"principal">> => all, - <<"topics">> => [<<"#">>], - <<"action">> => all, - <<"permission">> => deny} +-define(RULE1, #{principal => all, + topics => [<<"#">>], + action => all, + permission => deny} ). --define(RULE2, #{<<"principal">> => - #{<<"ipaddress">> => <<"127.0.0.1">>}, - <<"topics">> => - [#{<<"eq">> => <<"#">>}, - #{<<"eq">> => <<"+">>} +-define(RULE2, #{principal => + #{ipaddress => <<"127.0.0.1">>}, + topics => + [#{eq => <<"#">>}, + #{eq => <<"+">>} ] , - <<"action">> => all, - <<"permission">> => allow} + action => all, + permission => allow} ). --define(RULE3,#{<<"principal">> => - #{<<"and">> => [#{<<"username">> => "^test?"}, - #{<<"clientid">> => "^test?"} +-define(RULE3,#{principal => + #{'and' => [#{username => "^test?"}, + #{clientid => "^test?"} ]}, - <<"topics">> => [<<"test">>], - <<"action">> => publish, - <<"permission">> => allow} + topics => [<<"test">>], + action => publish, + permission => allow} ). --define(RULE4,#{<<"principal">> => - #{<<"or">> => [#{<<"username">> => <<"^test">>}, - #{<<"clientid">> => <<"test?">>} - ]}, - <<"topics">> => [<<"%u">>,<<"%c">>], - <<"action">> => publish, - <<"permission">> => deny} +-define(RULE4,#{principal => + #{'or' => [#{username => <<"^test">>}, + #{clientid => <<"test?">>} + ]}, + topics => [<<"%u">>,<<"%c">>], + action => publish, + permission => deny} ). %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ -t_compile(_) -> - ?assertEqual(#{<<"permission">> => deny, - <<"action">> => all, - <<"principal">> => all, - <<"topics">> => [['#']] - },emqx_authz:compile(?RULE1)), - ?assertEqual(#{<<"permission">> => allow, - <<"action">> => all, - <<"principal">> => - #{<<"ipaddress">> => {{127,0,0,1},{127,0,0,1},32}}, - <<"topics">> => [#{<<"eq">> => ['#']}, - #{<<"eq">> => ['+']}] - }, emqx_authz:compile(?RULE2)), - ?assertMatch( - #{<<"permission">> := allow, - <<"action">> := publish, - <<"principal">> := - #{<<"and">> := [#{<<"username">> := {re_pattern, _, _, _, _}}, - #{<<"clientid">> := {re_pattern, _, _, _, _}} - ] - }, - <<"topics">> := [[<<"test">>]] - }, emqx_authz:compile(?RULE3)), - ?assertMatch( - #{<<"permission">> := deny, - <<"action">> := publish, - <<"principal">> := - #{<<"or">> := [#{<<"username">> := {re_pattern, _, _, _, _}}, - #{<<"clientid">> := {re_pattern, _, _, _, _}} - ] - }, - <<"topics">> := [#{<<"pattern">> := [<<"%u">>]}, - #{<<"pattern">> := [<<"%c">>]} - ] - }, emqx_authz:compile(?RULE4)), +t_init_rule(_) -> + ?assertMatch(#{annotations := #{id := _ID, + principal := all, + topics := [['#']]} + }, emqx_authz:init_rule(?RULE1)), + ?assertMatch(#{annotations := #{principal := + #{ipaddress := {{127,0,0,1},{127,0,0,1},32}}, + topics := [#{eq := ['#']}, + #{eq := ['+']}], + id := _ID} + }, emqx_authz:init_rule(?RULE2)), + ?assertMatch(#{annotations := + #{principal := + #{'and' := [#{username := {re_pattern, _, _, _, _}}, + #{clientid := {re_pattern, _, _, _, _}} + ] + }, + topics := [[<<"test">>]], + id := _ID} + }, emqx_authz:init_rule(?RULE3)), + ?assertMatch(#{annotations := + #{principal := + #{'or' := [#{username := {re_pattern, _, _, _, _}}, + #{clientid := {re_pattern, _, _, _, _}} + ] + }, + topics := [#{pattern := [<<"%u">>]}, + #{pattern := [<<"%c">>]} + ], + id := _ID} + }, emqx_authz:init_rule(?RULE4)), ok. t_authz(_) -> ClientInfo1 = #{clientid => <<"test">>, username => <<"test">>, - peerhost => {127,0,0,1} + peerhost => {127,0,0,1}, + zone => default, + listener => mqtt_tcp }, ClientInfo2 = #{clientid => <<"test">>, username => <<"test">>, - peerhost => {192,168,0,10} + peerhost => {192,168,0,10}, + zone => default, + listener => mqtt_tcp }, ClientInfo3 = #{clientid => <<"test">>, username => <<"fake">>, - peerhost => {127,0,0,1} + peerhost => {127,0,0,1}, + zone => default, + listener => mqtt_tcp }, ClientInfo4 = #{clientid => <<"fake">>, username => <<"test">>, - peerhost => {127,0,0,1} + peerhost => {127,0,0,1}, + zone => default, + listener => mqtt_tcp }, - Rules1 = [emqx_authz:compile(Rule) || Rule <- [?RULE1, ?RULE2]], - Rules2 = [emqx_authz:compile(Rule) || Rule <- [?RULE2, ?RULE1]], - Rules3 = [emqx_authz:compile(Rule) || Rule <- [?RULE3, ?RULE4]], - Rules4 = [emqx_authz:compile(Rule) || Rule <- [?RULE4, ?RULE1]], + Rules1 = [emqx_authz:init_rule(Rule) || Rule <- [?RULE1, ?RULE2]], + Rules2 = [emqx_authz:init_rule(Rule) || Rule <- [?RULE2, ?RULE1]], + Rules3 = [emqx_authz:init_rule(Rule) || Rule <- [?RULE3, ?RULE4]], + Rules4 = [emqx_authz:init_rule(Rule) || Rule <- [?RULE4, ?RULE1]], - ?assertEqual(deny, - emqx_authz:check_authz(ClientInfo1, subscribe, <<"#">>, deny, [])), ?assertEqual({stop, deny}, - emqx_authz:check_authz(ClientInfo1, subscribe, <<"+">>, deny, Rules1)), + emqx_authz:authorize(ClientInfo1, subscribe, <<"#">>, deny, [])), + ?assertEqual({stop, deny}, + emqx_authz:authorize(ClientInfo1, subscribe, <<"+">>, deny, Rules1)), ?assertEqual({stop, allow}, - emqx_authz:check_authz(ClientInfo1, subscribe, <<"+">>, deny, Rules2)), + emqx_authz:authorize(ClientInfo1, subscribe, <<"+">>, deny, Rules2)), ?assertEqual({stop, allow}, - emqx_authz:check_authz(ClientInfo1, publish, <<"test">>, deny, Rules3)), + emqx_authz:authorize(ClientInfo1, publish, <<"test">>, deny, Rules3)), ?assertEqual({stop, deny}, - emqx_authz:check_authz(ClientInfo1, publish, <<"test">>, deny, Rules4)), + emqx_authz:authorize(ClientInfo1, publish, <<"test">>, deny, Rules4)), ?assertEqual({stop, deny}, - emqx_authz:check_authz(ClientInfo2, subscribe, <<"#">>, deny, Rules2)), + emqx_authz:authorize(ClientInfo2, subscribe, <<"#">>, deny, Rules2)), ?assertEqual({stop, deny}, - emqx_authz:check_authz(ClientInfo3, publish, <<"test">>, deny, Rules3)), + emqx_authz:authorize(ClientInfo3, publish, <<"test">>, deny, Rules3)), ?assertEqual({stop, deny}, - emqx_authz:check_authz(ClientInfo3, publish, <<"fake">>, deny, Rules4)), + emqx_authz:authorize(ClientInfo3, publish, <<"fake">>, deny, Rules4)), ?assertEqual({stop, deny}, - emqx_authz:check_authz(ClientInfo4, publish, <<"test">>, deny, Rules3)), + emqx_authz:authorize(ClientInfo4, publish, <<"test">>, deny, Rules3)), ?assertEqual({stop, deny}, - emqx_authz:check_authz(ClientInfo4, publish, <<"fake">>, deny, Rules4)), + emqx_authz:authorize(ClientInfo4, publish, <<"fake">>, deny, Rules4)), ok. diff --git a/apps/emqx_authz/test/emqx_authz_api_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_SUITE.erl index 4871e0d7f..27c571d1c 100644 --- a/apps/emqx_authz/test/emqx_authz_api_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_SUITE.erl @@ -18,58 +18,70 @@ -compile(nowarn_export_all). -compile(export_all). --include("emqx_authz.hrl"). --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). +% -include("emqx_authz.hrl"). +% -include_lib("eunit/include/eunit.hrl"). +% -include_lib("common_test/include/ct.hrl"). --import(emqx_ct_http, [ request_api/3 - , request_api/5 - , get_http_data/1 - , create_default_app/0 - , delete_default_app/0 - , default_auth_header/0 - ]). +% -import(emqx_ct_http, [ request_api/3 +% , request_api/5 +% , get_http_data/1 +% , create_default_app/0 +% , delete_default_app/0 +% , default_auth_header/0 +% ]). --define(HOST, "http://127.0.0.1:8081/"). --define(API_VERSION, "v4"). --define(BASE_PATH, "api"). +% -define(HOST, "http://127.0.0.1:8081/"). +% -define(API_VERSION, "v4"). +% -define(BASE_PATH, "api"). + +-define(CONF_DEFAULT, <<""" +authorization:{ + rules: [ + ] +} +""">>). all() -> - emqx_ct:all(?MODULE). +%% TODO: V5 API +%% emqx_ct:all(?MODULE). + [t_api_unit_test]. groups() -> []. init_per_suite(Config) -> - ok = emqx_ct_helpers:start_apps([emqx_authz, emqx_management], fun set_special_configs/1), - create_default_app(), + ok = emqx_config:init_load(emqx_authz_schema, ?CONF_DEFAULT), + ok = emqx_ct_helpers:start_apps([emqx_authz]), + + %create_default_app(), Config. end_per_suite(_Config) -> - delete_default_app(), - file:delete(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf')), - emqx_ct_helpers:stop_apps([emqx_authz, emqx_management]). + %delete_default_app(), + emqx_ct_helpers:stop_apps([emqx_authz]). -set_special_configs(emqx) -> - application:set_env(emqx, allow_anonymous, true), - application:set_env(emqx, enable_acl_cache, false), - ok; -set_special_configs(emqx_authz) -> - application:set_env(emqx, plugins_etc_dir, - emqx_ct_helpers:deps_path(emqx_authz, "test")), - Conf = #{<<"authz">> => #{<<"rules">> => []}}, - ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)), +% set_special_configs(emqx) -> +% application:set_env(emqx, allow_anonymous, true), +% application:set_env(emqx, enable_authz_cache, false), +% ok; +% set_special_configs(emqx_authz) -> +% emqx_config:put([emqx_authz], #{rules => []}), +% ok; - ok; +% set_special_configs(emqx_management) -> +% emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], +% applications =>[#{id => "admin", secret => "public"}]}), +% ok; -set_special_configs(_App) -> - ok. +% set_special_configs(_App) -> +% ok. -%%------------------------------------------------------------------------------ -%% Testcases -%%------------------------------------------------------------------------------ +% %%------------------------------------------------------------------------------ +% %% Testcases +% %%------------------------------------------------------------------------------ -t_api(_Config) -> +t_api_unit_test(_Config) -> + %% TODO: Decode from JSON or HOCON, instead of hand-crafting decode result Rule1 = #{<<"principal">> => #{<<"and">> => [#{<<"username">> => <<"^test?">>}, #{<<"clientid">> => <<"^test?">>} @@ -78,53 +90,70 @@ t_api(_Config) -> <<"topics">> => [<<"%u">>], <<"permission">> => <<"allow">> }, - {ok, _} = request_http_rest_add(["authz/push"], #{rules => [Rule1]}), - {ok, Result1} = request_http_rest_lookup(["authz"]), - ?assertMatch([Rule1 | _ ], get_http_data(Result1)), + ok = emqx_authz_api:push_authz(#{}, Rule1), + [#{action := subscribe, + permission := allow, + principal := + #{'and' := [#{username := <<"^test?">>}, + #{clientid := <<"^test?">>}]}, + topics := [<<"%u">>]}] = emqx_config:get([authorization, rules]). - Rule2 = #{<<"principal">> => #{<<"ipaddress">> => <<"127.0.0.1">>}, - <<"action">> => <<"publish">>, - <<"topics">> => [#{<<"eq">> => <<"#">>}, - #{<<"eq">> => <<"+">>} - ], - <<"permission">> => <<"deny">> - }, - {ok, _} = request_http_rest_add(["authz/append"], #{rules => [Rule2]}), - {ok, Result2} = request_http_rest_lookup(["authz"]), - ?assertEqual(Rule2#{<<"principal">> => #{<<"ipaddress">> => "127.0.0.1"}}, - lists:last(get_http_data(Result2))), +% t_api(_Config) -> +% Rule1 = #{<<"principal">> => +% #{<<"and">> => [#{<<"username">> => <<"^test?">>}, +% #{<<"clientid">> => <<"^test?">>} +% ]}, +% <<"action">> => <<"subscribe">>, +% <<"topics">> => [<<"%u">>], +% <<"permission">> => <<"allow">> +% }, +% {ok, _} = request_http_rest_add(["authz/push"], #{rules => [Rule1]}), +% {ok, Result1} = request_http_rest_lookup(["authz"]), +% ?assertMatch([Rule1 | _ ], get_http_data(Result1)), - {ok, _} = request_http_rest_update(["authz"], #{rules => []}), - {ok, Result3} = request_http_rest_lookup(["authz"]), - ?assertEqual([], get_http_data(Result3)), - ok. +% Rule2 = #{<<"principal">> => #{<<"ipaddress">> => <<"127.0.0.1">>}, +% <<"action">> => <<"publish">>, +% <<"topics">> => [#{<<"eq">> => <<"#">>}, +% #{<<"eq">> => <<"+">>} +% ], +% <<"permission">> => <<"deny">> +% }, +% {ok, _} = request_http_rest_add(["authz/append"], #{rules => [Rule2]}), +% {ok, Result2} = request_http_rest_lookup(["authz"]), +% ?assertEqual(Rule2#{<<"principal">> => #{<<"ipaddress">> => "127.0.0.1"}}, +% lists:last(get_http_data(Result2))), -%%-------------------------------------------------------------------- -%% HTTP Request -%%-------------------------------------------------------------------- +% {ok, _} = request_http_rest_update(["authz"], #{rules => []}), +% {ok, Result3} = request_http_rest_lookup(["authz"]), +% ?assertEqual([], get_http_data(Result3)), +% ok. -request_http_rest_list(Path) -> - request_api(get, uri(Path), default_auth_header()). +% %%-------------------------------------------------------------------- +% %% HTTP Request +% %%-------------------------------------------------------------------- -request_http_rest_lookup(Path) -> - request_api(get, uri([Path]), default_auth_header()). +% request_http_rest_list(Path) -> +% request_api(get, uri(Path), default_auth_header()). -request_http_rest_add(Path, Params) -> - request_api(post, uri(Path), [], default_auth_header(), Params). +% request_http_rest_lookup(Path) -> +% request_api(get, uri([Path]), default_auth_header()). -request_http_rest_update(Path, Params) -> - request_api(put, uri([Path]), [], default_auth_header(), Params). +% request_http_rest_add(Path, Params) -> +% request_api(post, uri(Path), [], default_auth_header(), Params). -request_http_rest_delete(Login) -> - request_api(delete, uri([Login]), default_auth_header()). +% request_http_rest_update(Path, Params) -> +% request_api(put, uri([Path]), [], default_auth_header(), Params). -uri() -> uri([]). -uri(Parts) when is_list(Parts) -> - NParts = [b2l(E) || E <- Parts], - ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION | NParts]). +% request_http_rest_delete(Login) -> +% request_api(delete, uri([Login]), default_auth_header()). -%% @private -b2l(B) when is_binary(B) -> - binary_to_list(B); -b2l(L) when is_list(L) -> - L. +% uri() -> uri([]). +% uri(Parts) when is_list(Parts) -> +% NParts = [b2l(E) || E <- Parts], +% ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION | NParts]). + +% %% @private +% b2l(B) when is_binary(B) -> +% binary_to_list(B); +% b2l(L) when is_list(L) -> +% L. diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl new file mode 100644 index 000000000..f387be77a --- /dev/null +++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl @@ -0,0 +1,84 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_http_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include("emqx_authz.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_ct:all(?MODULE). + +groups() -> + []. + +init_per_suite(Config) -> + meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end), + + ok = emqx_ct_helpers:start_apps([emqx_authz]), + + ok = emqx_config:update([zones, default, authorization, cache, enable], false), + ok = emqx_config:update([zones, default, authorization, enable], true), + Rules = [#{ <<"config">> => #{ + <<"url">> => <<"https://fake.com:443/">>, + <<"headers">> => #{}, + <<"method">> => <<"get">>, + <<"request_timeout">> => 5000 + }, + <<"principal">> => <<"all">>, + <<"type">> => <<"http">>} + ], + ok = emqx_authz:update(replace, Rules), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_authz, emqx_resource]), + meck:unload(emqx_resource). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_authz(_) -> + ClientInfo = #{clientid => <<"clientid">>, + username => <<"username">>, + peerhost => {127,0,0,1}, + protocol => mqtt, + mountpoint => <<"fake">>, + zone => default, + listener => mqtt_tcp + }, + + meck:expect(emqx_resource, query, fun(_, _) -> {ok, 204, fake_headers} end), + ?assertEqual(allow, + emqx_access_control:authorize(ClientInfo, subscribe, <<"#">>)), + + meck:expect(emqx_resource, query, fun(_, _) -> {ok, 200, fake_headers, fake_body} end), + ?assertEqual(allow, + emqx_access_control:authorize(ClientInfo, publish, <<"#">>)), + + + meck:expect(emqx_resource, query, fun(_, _) -> {error, other} end), + ?assertEqual(deny, + emqx_access_control:authorize(ClientInfo, subscribe, <<"+">>)), + ?assertEqual(deny, + emqx_access_control:authorize(ClientInfo, publish, <<"+">>)), + ok. + diff --git a/apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl new file mode 100644 index 000000000..a5f59ac64 --- /dev/null +++ b/apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl @@ -0,0 +1,113 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_mongo_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include("emqx_authz.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_ct:all(?MODULE). + +groups() -> + []. + +init_per_suite(Config) -> + meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end), + + ok = emqx_ct_helpers:start_apps([emqx_authz]), + ok = emqx_config:update([zones, default, authorization, cache, enable], false), + ok = emqx_config:update([zones, default, authorization, enable], true), + Rules = [#{ <<"config">> => #{ + <<"mongo_type">> => <<"single">>, + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"ssl">> => #{<<"enable">> => false}}, + <<"principal">> => <<"all">>, + <<"collection">> => <<"fake">>, + <<"find">> => #{<<"a">> => <<"b">>}, + <<"type">> => <<"mongo">>} + ], + ok = emqx_authz:update(replace, Rules), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_authz, emqx_resource]), + meck:unload(emqx_resource). + +-define(RULE1,[#{<<"topics">> => [<<"#">>], + <<"permission">> => <<"deny">>, + <<"action">> => <<"all">>}]). +-define(RULE2,[#{<<"topics">> => [<<"eq #">>], + <<"permission">> => <<"allow">>, + <<"action">> => <<"all">>}]). +-define(RULE3,[#{<<"topics">> => [<<"test/%c">>], + <<"permission">> => <<"allow">>, + <<"action">> => <<"subscribe">>}]). +-define(RULE4,[#{<<"topics">> => [<<"test/%u">>], + <<"permission">> => <<"allow">>, + <<"action">> => <<"publish">>}]). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_authz(_) -> + ClientInfo1 = #{clientid => <<"test">>, + username => <<"test">>, + peerhost => {127,0,0,1}, + zone => default, + listener => mqtt_tcp + }, + ClientInfo2 = #{clientid => <<"test_clientid">>, + username => <<"test_username">>, + peerhost => {192,168,0,10}, + zone => default, + listener => mqtt_tcp + }, + ClientInfo3 = #{clientid => <<"test_clientid">>, + username => <<"fake_username">>, + peerhost => {127,0,0,1}, + zone => default, + listener => mqtt_tcp + }, + + meck:expect(emqx_resource, query, fun(_, _) -> [] end), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo1, subscribe, <<"#">>)), % nomatch + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo1, publish, <<"#">>)), % nomatch + + meck:expect(emqx_resource, query, fun(_, _) -> ?RULE1 ++ ?RULE2 end), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo1, subscribe, <<"+">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo1, publish, <<"+">>)), + + meck:expect(emqx_resource, query, fun(_, _) -> ?RULE2 ++ ?RULE1 end), + ?assertEqual(allow, emqx_access_control:authorize(ClientInfo1, subscribe, <<"#">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo1, subscribe, <<"+">>)), + + meck:expect(emqx_resource, query, fun(_, _) -> ?RULE3 ++ ?RULE4 end), + ?assertEqual(allow, emqx_access_control:authorize(ClientInfo2, subscribe, <<"test/test_clientid">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo2, publish, <<"test/test_clientid">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo2, subscribe, <<"test/test_username">>)), + ?assertEqual(allow, emqx_access_control:authorize(ClientInfo2, publish, <<"test/test_username">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo3, subscribe, <<"test">>)), % nomatch + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo3, publish, <<"test">>)), % nomatch + ok. + diff --git a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl index 4f2148522..050234478 100644 --- a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl @@ -30,37 +30,31 @@ groups() -> init_per_suite(Config) -> meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_resource, check_and_create, fun(_, _, _) -> {ok, meck_data} end ), - ok = emqx_ct_helpers:start_apps([emqx_authz], fun set_special_configs/1), + meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end ), + + ok = emqx_ct_helpers:start_apps([emqx_authz]), + + ok = emqx_config:update([zones, default, authorization, cache, enable], false), + ok = emqx_config:update([zones, default, authorization, enable], true), + Rules = [#{ <<"config">> => #{ + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"username">> => <<"xx">>, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false} + }, + <<"principal">> => <<"all">>, + <<"sql">> => <<"abcb">>, + <<"type">> => <<"mysql">> }], + emqx_authz:update(replace, Rules), Config. end_per_suite(_Config) -> - file:delete(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf')), emqx_ct_helpers:stop_apps([emqx_authz, emqx_resource]), meck:unload(emqx_resource). -set_special_configs(emqx) -> - application:set_env(emqx, allow_anonymous, false), - application:set_env(emqx, enable_acl_cache, false), - application:set_env(emqx, acl_nomatch, deny), - application:set_env(emqx, plugins_loaded_file, - emqx_ct_helpers:deps_path(emqx, "test/loaded_plguins")), - ok; -set_special_configs(emqx_authz) -> - application:set_env(emqx, plugins_etc_dir, - emqx_ct_helpers:deps_path(emqx_authz, "test")), - Conf = #{<<"authz">> => - #{<<"rules">> => - [#{<<"config">> =>#{<<"meck">> => <<"fake">>}, - <<"principal">> => all, - <<"sql">> => <<"fake sql">>, - <<"type">> => mysql} - ]}}, - ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)), - ok; -set_special_configs(_App) -> - ok. - -define(COLUMNS, [ <<"ipaddress">> , <<"username">> , <<"clientid">> @@ -81,37 +75,40 @@ t_authz(_) -> ClientInfo1 = #{clientid => <<"test">>, username => <<"test">>, peerhost => {127,0,0,1}, - zone => zone + zone => default, + listener => mqtt_tcp }, ClientInfo2 = #{clientid => <<"test_clientid">>, username => <<"test_username">>, peerhost => {192,168,0,10}, - zone => zone + zone => default, + listener => mqtt_tcp }, ClientInfo3 = #{clientid => <<"test_clientid">>, username => <<"fake_username">>, peerhost => {127,0,0,1}, - zone => zone + zone => default, + listener => mqtt_tcp }, meck:expect(emqx_resource, query, fun(_, _) -> {ok, ?COLUMNS, []} end), - ?assertEqual(deny, emqx_access_control:check_acl(ClientInfo1, subscribe, <<"#">>)), % nomatch - ?assertEqual(deny, emqx_access_control:check_acl(ClientInfo1, publish, <<"#">>)), % nomatch + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo1, subscribe, <<"#">>)), % nomatch + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo1, publish, <<"#">>)), % nomatch meck:expect(emqx_resource, query, fun(_, _) -> {ok, ?COLUMNS, ?RULE1 ++ ?RULE2} end), - ?assertEqual(deny, emqx_access_control:check_acl(ClientInfo1, subscribe, <<"+">>)), - ?assertEqual(deny, emqx_access_control:check_acl(ClientInfo1, publish, <<"+">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo1, subscribe, <<"+">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo1, publish, <<"+">>)), meck:expect(emqx_resource, query, fun(_, _) -> {ok, ?COLUMNS, ?RULE2 ++ ?RULE1} end), - ?assertEqual(allow, emqx_access_control:check_acl(ClientInfo1, subscribe, <<"#">>)), - ?assertEqual(deny, emqx_access_control:check_acl(ClientInfo1, subscribe, <<"+">>)), + ?assertEqual(allow, emqx_access_control:authorize(ClientInfo1, subscribe, <<"#">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo1, subscribe, <<"+">>)), meck:expect(emqx_resource, query, fun(_, _) -> {ok, ?COLUMNS, ?RULE3 ++ ?RULE4} end), - ?assertEqual(allow, emqx_access_control:check_acl(ClientInfo2, subscribe, <<"test/test_clientid">>)), - ?assertEqual(deny, emqx_access_control:check_acl(ClientInfo2, publish, <<"test/test_clientid">>)), - ?assertEqual(deny, emqx_access_control:check_acl(ClientInfo2, subscribe, <<"test/test_username">>)), - ?assertEqual(allow, emqx_access_control:check_acl(ClientInfo2, publish, <<"test/test_username">>)), - ?assertEqual(deny, emqx_access_control:check_acl(ClientInfo3, subscribe, <<"test">>)), % nomatch - ?assertEqual(deny, emqx_access_control:check_acl(ClientInfo3, publish, <<"test">>)), % nomatch + ?assertEqual(allow, emqx_access_control:authorize(ClientInfo2, subscribe, <<"test/test_clientid">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo2, publish, <<"test/test_clientid">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo2, subscribe, <<"test/test_username">>)), + ?assertEqual(allow, emqx_access_control:authorize(ClientInfo2, publish, <<"test/test_username">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo3, subscribe, <<"test">>)), % nomatch + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo3, publish, <<"test">>)), % nomatch ok. diff --git a/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl index dcc820a4c..439dce14f 100644 --- a/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl @@ -30,37 +30,30 @@ groups() -> init_per_suite(Config) -> meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_resource, check_and_create, fun(_, _, _) -> {ok, meck_data} end ), - ok = emqx_ct_helpers:start_apps([emqx_authz], fun set_special_configs/1), + meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end ), + + ok = emqx_ct_helpers:start_apps([emqx_authz]), + + ok = emqx_config:update([zones, default, authorization, cache, enable], false), + ok = emqx_config:update([zones, default, authorization, enable], true), + Rules = [#{ <<"config">> => #{ + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"username">> => <<"xx">>, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false} + }, + <<"sql">> => <<"abcb">>, + <<"type">> => <<"pgsql">> }], + emqx_authz:update(replace, Rules), Config. end_per_suite(_Config) -> - file:delete(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf')), emqx_ct_helpers:stop_apps([emqx_authz, emqx_resource]), meck:unload(emqx_resource). -set_special_configs(emqx) -> - application:set_env(emqx, allow_anonymous, false), - application:set_env(emqx, enable_acl_cache, false), - application:set_env(emqx, acl_nomatch, deny), - application:set_env(emqx, plugins_loaded_file, - emqx_ct_helpers:deps_path(emqx, "test/loaded_plguins")), - ok; -set_special_configs(emqx_authz) -> - application:set_env(emqx, plugins_etc_dir, - emqx_ct_helpers:deps_path(emqx_authz, "test")), - Conf = #{<<"authz">> => - #{<<"rules">> => - [#{<<"config">> =>#{<<"meck">> => <<"fake">>}, - <<"principal">> => all, - <<"sql">> => <<"fake sql">>, - <<"type">> => pgsql} - ]}}, - ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)), - ok; -set_special_configs(_App) -> - ok. - -define(COLUMNS, [ {column, <<"ipaddress">>, meck, meck, meck, meck, meck, meck, meck} , {column, <<"username">>, meck, meck, meck, meck, meck, meck, meck} , {column, <<"clientid">>, meck, meck, meck, meck, meck, meck, meck} @@ -81,37 +74,40 @@ t_authz(_) -> ClientInfo1 = #{clientid => <<"test">>, username => <<"test">>, peerhost => {127,0,0,1}, - zone => zone + zone => default, + listener => mqtt_tcp }, ClientInfo2 = #{clientid => <<"test_clientid">>, username => <<"test_username">>, peerhost => {192,168,0,10}, - zone => zone + zone => default, + listener => mqtt_tcp }, ClientInfo3 = #{clientid => <<"test_clientid">>, username => <<"fake_username">>, peerhost => {127,0,0,1}, - zone => zone + zone => default, + listener => mqtt_tcp }, meck:expect(emqx_resource, query, fun(_, _) -> {ok, ?COLUMNS, []} end), - ?assertEqual(deny, emqx_access_control:check_acl(ClientInfo1, subscribe, <<"#">>)), % nomatch - ?assertEqual(deny, emqx_access_control:check_acl(ClientInfo1, publish, <<"#">>)), % nomatch + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo1, subscribe, <<"#">>)), % nomatch + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo1, publish, <<"#">>)), % nomatch meck:expect(emqx_resource, query, fun(_, _) -> {ok, ?COLUMNS, ?RULE1 ++ ?RULE2} end), - ?assertEqual(deny, emqx_access_control:check_acl(ClientInfo1, subscribe, <<"+">>)), - ?assertEqual(deny, emqx_access_control:check_acl(ClientInfo1, publish, <<"+">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo1, subscribe, <<"+">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo1, publish, <<"+">>)), meck:expect(emqx_resource, query, fun(_, _) -> {ok, ?COLUMNS, ?RULE2 ++ ?RULE1} end), - ?assertEqual(allow, emqx_access_control:check_acl(ClientInfo1, subscribe, <<"#">>)), - ?assertEqual(deny, emqx_access_control:check_acl(ClientInfo2, subscribe, <<"+">>)), + ?assertEqual(allow, emqx_access_control:authorize(ClientInfo1, subscribe, <<"#">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo2, subscribe, <<"+">>)), meck:expect(emqx_resource, query, fun(_, _) -> {ok, ?COLUMNS, ?RULE3 ++ ?RULE4} end), - ?assertEqual(allow, emqx_access_control:check_acl(ClientInfo2, subscribe, <<"test/test_clientid">>)), - ?assertEqual(deny, emqx_access_control:check_acl(ClientInfo2, publish, <<"test/test_clientid">>)), - ?assertEqual(deny, emqx_access_control:check_acl(ClientInfo2, subscribe, <<"test/test_username">>)), - ?assertEqual(allow, emqx_access_control:check_acl(ClientInfo2, publish, <<"test/test_username">>)), - ?assertEqual(deny, emqx_access_control:check_acl(ClientInfo3, subscribe, <<"test">>)), % nomatch - ?assertEqual(deny, emqx_access_control:check_acl(ClientInfo3, publish, <<"test">>)), % nomatch + ?assertEqual(allow, emqx_access_control:authorize(ClientInfo2, subscribe, <<"test/test_clientid">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo2, publish, <<"test/test_clientid">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo2, subscribe, <<"test/test_username">>)), + ?assertEqual(allow, emqx_access_control:authorize(ClientInfo2, publish, <<"test/test_username">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo3, subscribe, <<"test">>)), % nomatch + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo3, publish, <<"test">>)), % nomatch ok. diff --git a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl index 3f8bea166..7c7990dbf 100644 --- a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl @@ -30,42 +30,29 @@ groups() -> init_per_suite(Config) -> meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_resource, check_and_create, fun(_, _, _) -> {ok, meck_data} end ), - ok = emqx_ct_helpers:start_apps([emqx_authz], fun set_special_configs/1), + meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end ), + + ok = emqx_ct_helpers:start_apps([emqx_authz]), + + ok = emqx_config:update([zones, default, authorization, cache, enable], false), + ok = emqx_config:update([zones, default, authorization, enable], true), + Rules = [#{ <<"config">> => #{ + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => 0, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false} + }, + <<"cmd">> => <<"HGETALL mqtt_authz:%u">>, + <<"type">> => <<"redis">> }], + emqx_authz:update(replace, Rules), Config. end_per_suite(_Config) -> - file:delete(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf')), emqx_ct_helpers:stop_apps([emqx_authz, emqx_resource]), meck:unload(emqx_resource). -set_special_configs(emqx) -> - application:set_env(emqx, allow_anonymous, true), - application:set_env(emqx, enable_acl_cache, false), - application:set_env(emqx, acl_nomatch, deny), - application:set_env(emqx, plugins_loaded_file, - emqx_ct_helpers:deps_path(emqx, "test/loaded_plguins")), - ok; -set_special_configs(emqx_authz) -> - application:set_env(emqx, plugins_etc_dir, - emqx_ct_helpers:deps_path(emqx_authz, "test")), - Conf = #{<<"authz">> => - #{<<"rules">> => - [#{<<"config">> =>#{ - <<"server">> => <<"127.0.0.1:6379">>, - <<"password">> => <<"public">>, - <<"pool_size">> => 1, - <<"auto_reconnect">> => true - }, - <<"principal">> => all, - <<"cmd">> => <<"fake cmd">>, - <<"type">> => redis} - ]}}, - ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)), - ok; -set_special_configs(_App) -> - ok. - -define(RULE1, [<<"test/%u">>, <<"publish">>]). -define(RULE2, [<<"test/%c">>, <<"publish">>]). -define(RULE3, [<<"#">>, <<"subscribe">>]). @@ -78,36 +65,37 @@ t_authz(_) -> ClientInfo = #{clientid => <<"clientid">>, username => <<"username">>, peerhost => {127,0,0,1}, - zone => zone + zone => default, + listener => mqtt_tcp }, meck:expect(emqx_resource, query, fun(_, _) -> {ok, []} end), % nomatch ?assertEqual(deny, - emqx_access_control:check_acl(ClientInfo, subscribe, <<"#">>)), + emqx_access_control:authorize(ClientInfo, subscribe, <<"#">>)), ?assertEqual(deny, - emqx_access_control:check_acl(ClientInfo, publish, <<"#">>)), + emqx_access_control:authorize(ClientInfo, publish, <<"#">>)), meck:expect(emqx_resource, query, fun(_, _) -> {ok, ?RULE1 ++ ?RULE2} end), % nomatch ?assertEqual(deny, - emqx_access_control:check_acl(ClientInfo, subscribe, <<"+">>)), + emqx_access_control:authorize(ClientInfo, subscribe, <<"+">>)), % nomatch ?assertEqual(deny, - emqx_access_control:check_acl(ClientInfo, subscribe, <<"test/username">>)), + emqx_access_control:authorize(ClientInfo, subscribe, <<"test/username">>)), ?assertEqual(allow, - emqx_access_control:check_acl(ClientInfo, publish, <<"test/clientid">>)), + emqx_access_control:authorize(ClientInfo, publish, <<"test/clientid">>)), ?assertEqual(allow, - emqx_access_control:check_acl(ClientInfo, publish, <<"test/clientid">>)), + emqx_access_control:authorize(ClientInfo, publish, <<"test/clientid">>)), meck:expect(emqx_resource, query, fun(_, _) -> {ok, ?RULE3} end), ?assertEqual(allow, - emqx_access_control:check_acl(ClientInfo, subscribe, <<"#">>)), + emqx_access_control:authorize(ClientInfo, subscribe, <<"#">>)), % nomatch ?assertEqual(deny, - emqx_access_control:check_acl(ClientInfo, publish, <<"#">>)), + emqx_access_control:authorize(ClientInfo, publish, <<"#">>)), ok. diff --git a/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf b/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf index 3cc719e40..c34567ee4 100644 --- a/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf +++ b/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf @@ -2,173 +2,57 @@ ## Configuration for EMQ X MQTT Broker Bridge ##==================================================================== -##-------------------------------------------------------------------- -## Bridges to aws -##-------------------------------------------------------------------- - -## Bridge address: node name for local bridge, host:port for remote. -## -## Value: String -## Example: emqx@127.0.0.1, "127.0.0.1:1883" -bridge.mqtt.aws.address = "127.0.0.1:1883" - -## Protocol version of the bridge. -## -## Value: Enum -## - mqttv5 -## - mqttv4 -## - mqttv3 -bridge.mqtt.aws.proto_ver = mqttv4 - -## Start type of the bridge. -## -## Value: enum -## manual -## auto -bridge.mqtt.aws.start_type = manual - -## Whether to enable bridge mode for mqtt bridge -## -## This option is prepared for the mqtt broker which does not -## support bridge_mode such as the mqtt-plugin of the rabbitmq -## -## Value: boolean -#bridge.mqtt.aws.bridge_mode = false - -## The ClientId of a remote bridge. -## -## Placeholders: -## ${node}: Node name -## -## Value: String -bridge.mqtt.aws.clientid = bridge_aws - -## The Clean start flag of a remote bridge. -## -## Value: boolean -## Default: true -## -## NOTE: Some IoT platforms require clean_start -## must be set to 'true' -bridge.mqtt.aws.clean_start = true - -## The username for a remote bridge. -## -## Value: String -bridge.mqtt.aws.username = user - -## The password for a remote bridge. -## -## Value: String -bridge.mqtt.aws.password = passwd - -## Topics that need to be forward to AWS IoTHUB -## -## Value: String -## Example: "topic1/#,topic2/#" -bridge.mqtt.aws.forwards = "topic1/#,topic2/#" - -## Forward messages to the mountpoint of an AWS IoTHUB -## -## Value: String -bridge.mqtt.aws.forward_mountpoint = "bridge/aws/${node}/" - -## Need to subscribe to AWS topics -## -## Value: String -## bridge.mqtt.aws.subscription.1.topic = "cmd/topic1" - -## Need to subscribe to AWS topics QoS. -## -## Value: Number -## bridge.mqtt.aws.subscription.1.qos = 1 - -## A mountpoint that receives messages from AWS IoTHUB -## -## Value: String -## bridge.mqtt.aws.receive_mountpoint = "receive/aws/" - - -## Bribge to remote server via SSL. -## -## Value: on | off -bridge.mqtt.aws.ssl = off - -## PEM-encoded CA certificates of the bridge. -## -## Value: File -bridge.mqtt.aws.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" - -## Client SSL Certfile of the bridge. -## -## Value: File -bridge.mqtt.aws.certfile = "{{ platform_etc_dir }}/certs/client-cert.pem" - -## Client SSL Keyfile of the bridge. -## -## Value: File -bridge.mqtt.aws.keyfile = "{{ platform_etc_dir }}/certs/client-key.pem" - -## SSL Ciphers used by the bridge. -## -## Value: String -bridge.mqtt.aws.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" - -## Ciphers for TLS PSK. -## Note that 'bridge.${BridgeName}.ciphers' and 'bridge.${BridgeName}.psk_ciphers' cannot -## be configured at the same time. -## See 'https://tools.ietf.org/html/rfc4279#section-2'. -#bridge.mqtt.aws.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" - -## Ping interval of a down bridge. -## -## Value: Duration -## Default: 10 seconds -bridge.mqtt.aws.keepalive = 60s - -## TLS versions used by the bridge. -## -## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier -## Value: String -bridge.mqtt.aws.tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" - -## Bridge reconnect time. -## -## Value: Duration -## Default: 30 seconds -bridge.mqtt.aws.reconnect_interval = 30s - -## Retry interval for bridge QoS1 message delivering. -## -## Value: Duration -bridge.mqtt.aws.retry_interval = 20s - -## Publish messages in batches, only RPC Bridge supports -## -## Value: Integer -## default: 32 -bridge.mqtt.aws.batch_size = 32 - -## Inflight size. -## 0 means infinity (no limit on the inflight window) -## -## Value: Integer -bridge.mqtt.aws.max_inflight_size = 32 - -## Base directory for replayq to store messages on disk -## If this config entry is missing or set to undefined, -## replayq works in a mem-only manner. -## -## Value: String -bridge.mqtt.aws.queue.replayq_dir = "{{ platform_data_dir }}/replayq/emqx_aws_bridge/" - -## Replayq segment size -## -## Value: Bytesize -bridge.mqtt.aws.queue.replayq_seg_bytes = 10MB - -## Replayq max total size -## -## Value: Bytesize -bridge.mqtt.aws.queue.max_total_size = 5GB - +emqx_bridge_mqtt:{ + bridges:[ + # { + # name: "mqtt1" + # start_type: auto + # forwards: ["test/#"], + # forward_mountpoint: "" + # reconnect_interval: "30s" + # batch_size: 100 + # queue:{ + # replayq_dir: "{{ platform_data_dir }}/replayq/bridge_mqtt/" + # replayq_seg_bytes: "100MB" + # replayq_offload_mode: false + # replayq_max_total_bytes: "1GB" + # }, + # config:{ + # conn_type: mqtt + # address: "127.0.0.1:1883" + # proto_ver: v4 + # bridge_mode: true + # clientid: "client1" + # clean_start: true + # username: "username1" + # password: "" + # keepalive: 300 + # subscriptions: [{ + # topic: "t/#" + # qos: 1 + # }] + # receive_mountpoint: "" + # retry_interval: "30s" + # max_inflight: 32 + # } + # }, + # { + # name: "rpc1" + # start_type: auto + # forwards: ["test/#"], + # forward_mountpoint: "" + # reconnect_interval: "30s" + # batch_size: 100 + # queue:{ + # replayq_dir: "{{ platform_data_dir }}/replayq/bridge_mqtt/" + # replayq_seg_bytes: "100MB" + # replayq_offload_mode: false + # replayq_max_total_bytes: "1GB" + # }, + # config:{ + # conn_type: rpc + # node: "emqx@127.0.0.1" + # } + # } + ] +} diff --git a/apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema b/apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema deleted file mode 100644 index 3168bfc14..000000000 --- a/apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema +++ /dev/null @@ -1,244 +0,0 @@ -%%-*- mode: erlang -*- -%%-------------------------------------------------------------------- -%% Bridges -%%-------------------------------------------------------------------- -{mapping, "bridge.mqtt.$name.address", "emqx_bridge_mqtt.bridges", [ - {datatype, string} -]}. - -{mapping, "bridge.mqtt.$name.proto_ver", "emqx_bridge_mqtt.bridges", [ - {datatype, {enum, [mqttv3, mqttv4, mqttv5]}} -]}. - -{mapping, "bridge.mqtt.$name.bridge_mode", "emqx_bridge_mqtt.bridges", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "bridge.mqtt.$name.start_type", "emqx_bridge_mqtt.bridges", [ - {datatype, {enum, [manual, auto]}}, - {default, auto} -]}. - -{mapping, "bridge.mqtt.$name.clientid", "emqx_bridge_mqtt.bridges", [ - {datatype, string} -]}. - -{mapping, "bridge.mqtt.$name.clean_start", "emqx_bridge_mqtt.bridges", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "bridge.mqtt.$name.username", "emqx_bridge_mqtt.bridges", [ - {datatype, string} -]}. - -{mapping, "bridge.mqtt.$name.password", "emqx_bridge_mqtt.bridges", [ - {datatype, string} -]}. - -{mapping, "bridge.mqtt.$name.forwards", "emqx_bridge_mqtt.bridges", [ - {datatype, string}, - {default, ""} -]}. - -{mapping, "bridge.mqtt.$name.forward_mountpoint", "emqx_bridge_mqtt.bridges", [ - {datatype, string} -]}. - -{mapping, "bridge.mqtt.$name.subscription.$id.topic", "emqx_bridge_mqtt.bridges", [ - {datatype, string} -]}. - -{mapping, "bridge.mqtt.$name.subscription.$id.qos", "emqx_bridge_mqtt.bridges", [ - {datatype, integer} -]}. - -{mapping, "bridge.mqtt.$name.receive_mountpoint", "emqx_bridge_mqtt.bridges", [ - {datatype, string} -]}. - -{mapping, "bridge.mqtt.$name.ssl", "emqx_bridge_mqtt.bridges", [ - {datatype, flag}, - {default, off} -]}. - -{mapping, "bridge.mqtt.$name.cacertfile", "emqx_bridge_mqtt.bridges", [ - {datatype, string} -]}. - -{mapping, "bridge.mqtt.$name.certfile", "emqx_bridge_mqtt.bridges", [ - {datatype, string} -]}. - -{mapping, "bridge.mqtt.$name.keyfile", "emqx_bridge_mqtt.bridges", [ - {datatype, string} -]}. - -{mapping, "bridge.mqtt.$name.ciphers", "emqx_bridge_mqtt.bridges", [ - {datatype, string} -]}. - -{mapping, "bridge.mqtt.$name.psk_ciphers", "emqx_bridge_mqtt.bridges", [ - {datatype, string} -]}. - -{mapping, "bridge.mqtt.$name.keepalive", "emqx_bridge_mqtt.bridges", [ - {default, "10s"}, - {datatype, {duration, s}} -]}. - -{mapping, "bridge.mqtt.$name.tls_versions", "emqx_bridge_mqtt.bridges", [ - {datatype, string}, - {default, "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1"} -]}. - -{mapping, "bridge.mqtt.$name.reconnect_interval", "emqx_bridge_mqtt.bridges", [ - {default, "30s"}, - {datatype, {duration, ms}} -]}. - -{mapping, "bridge.mqtt.$name.retry_interval", "emqx_bridge_mqtt.bridges", [ - {default, "20s"}, - {datatype, {duration, s}} -]}. - -{mapping, "bridge.mqtt.$name.max_inflight_size", "emqx_bridge_mqtt.bridges", [ - {default, 0}, - {datatype, integer} - ]}. - -{mapping, "bridge.mqtt.$name.batch_size", "emqx_bridge_mqtt.bridges", [ - {default, 0}, - {datatype, integer} -]}. - -{mapping, "bridge.mqtt.$name.queue.replayq_dir", "emqx_bridge_mqtt.bridges", [ - {datatype, string} -]}. - -{mapping, "bridge.mqtt.$name.queue.replayq_seg_bytes", "emqx_bridge_mqtt.bridges", [ - {datatype, bytesize} -]}. - -{mapping, "bridge.mqtt.$name.queue.max_total_size", "emqx_bridge_mqtt.bridges", [ - {datatype, bytesize} -]}. - -{translation, "emqx_bridge_mqtt.bridges", fun(Conf) -> - - MapPSKCiphers = fun(PSKCiphers) -> - lists:map( - fun("PSK-AES128-CBC-SHA") -> {psk, aes_128_cbc, sha}; - ("PSK-AES256-CBC-SHA") -> {psk, aes_256_cbc, sha}; - ("PSK-3DES-EDE-CBC-SHA") -> {psk, '3des_ede_cbc', sha}; - ("PSK-RC4-SHA") -> {psk, rc4_128, sha} - end, PSKCiphers) - end, - - Split = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end, - - IsSsl = fun(cacertfile) -> true; - (certfile) -> true; - (keyfile) -> true; - (ciphers) -> true; - (psk_ciphers) -> true; - (tls_versions) -> true; - (_Opt) -> false - end, - - Parse = fun(tls_versions, Vers) -> - [{versions, [list_to_atom(S) || S <- Split(Vers)]}]; - (ciphers, Ciphers) -> - [{ciphers, Split(Ciphers)}]; - (psk_ciphers, Ciphers) -> - [{ciphers, MapPSKCiphers(Split(Ciphers))}, {user_lookup_fun, {fun emqx_psk:lookup/3, <<>>}}]; - (Opt, Val) -> - [{Opt, Val}] - end, - - Merge = fun(forwards, Val, Opts) -> - [{forwards, string:tokens(Val, ",")}|Opts]; - (Opt, Val, Opts) -> - case IsSsl(Opt) of - true -> - SslOpts = Parse(Opt, Val) ++ proplists:get_value(ssl_opts, Opts, []), - lists:ukeymerge(1, [{ssl_opts, SslOpts}], lists:usort(Opts)); - false -> - [{Opt, Val}|Opts] - end - end, - Queue = fun(Name) -> - Configs = cuttlefish_variable:filter_by_prefix("bridge.mqtt." ++ Name ++ ".queue", Conf), - - QOpts = [{list_to_atom(QOpt), QValue}|| {[_, _, _, "queue", QOpt], QValue} <- Configs], - maps:from_list(QOpts) - end, - Subscriptions = fun(Name) -> - Configs = cuttlefish_variable:filter_by_prefix("bridge.mqtt." ++ Name ++ ".subscription", Conf), - lists:zip([Topic || {_, Topic} <- lists:sort([{I, Topic} || {[_, _, _, "subscription", I, "topic"], Topic} <- Configs])], - [QoS || {_, QoS} <- lists:sort([{I, QoS} || {[_, _, _, "subscription", I, "qos"], QoS} <- Configs])]) - end, - IsNodeAddr = fun(Addr) -> - case string:tokens(Addr, "@") of - [_NodeName, _Hostname] -> true; - _ -> false - end - end, - ConnMod = fun(Name) -> - - [AddrConfig] = cuttlefish_variable:filter_by_prefix("bridge.mqtt." ++ Name ++ ".address", Conf), - {_, Addr} = AddrConfig, - - Subs = Subscriptions(Name), - case IsNodeAddr(Addr) of - true when Subs =/= [] -> - error({"subscriptions are not supported when bridging between emqx nodes", Name, Subs}); - true -> - emqx_bridge_rpc; - false -> - emqx_bridge_mqtt - end - end, - - %% to be backward compatible - Translate = - fun Tr(queue, Q, Cfg) -> - NewQ = maps:fold(Tr, #{}, Q), - Cfg#{queue => NewQ}; - Tr(address, Addr0, Cfg) -> - Addr = case IsNodeAddr(Addr0) of - true -> list_to_atom(Addr0); - false -> Addr0 - end, - Cfg#{address => Addr}; - Tr(reconnect_interval, Ms, Cfg) -> - Cfg#{reconnect_delay_ms => Ms}; - Tr(proto_ver, Ver, Cfg) -> - Cfg#{proto_ver => - case Ver of - mqttv3 -> v3; - mqttv4 -> v4; - mqttv5 -> v5; - _ -> v4 - end}; - Tr(max_inflight_size, Size, Cfg) -> - Cfg#{max_inflight => Size}; - Tr(Key, Value, Cfg) -> - Cfg#{Key => Value} - end, - C = lists:foldl( - fun({["bridge", "mqtt", Name, Opt], Val}, Acc) -> - %% e.g #{aws => [{OptKey, OptVal}]} - Init = [{list_to_atom(Opt), Val}, - {connect_module, ConnMod(Name)}, - {subscriptions, Subscriptions(Name)}, - {queue, Queue(Name)}], - maps:update_with(list_to_atom(Name), fun(Opts) -> Merge(list_to_atom(Opt), Val, Opts) end, Init, Acc); - (_, Acc) -> Acc - end, #{}, lists:usort(cuttlefish_variable:filter_by_prefix("bridge.mqtt", Conf))), - C1 = maps:map(fun(Bn, Bc) -> - maps:to_list(maps:fold(Translate, #{}, maps:from_list(Bc))) - end, C), - maps:to_list(C1) -end}. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_connect.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_connect.erl deleted file mode 100644 index ece6002a7..000000000 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_connect.erl +++ /dev/null @@ -1,74 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_bridge_connect). - --export([start/2]). - --export_type([config/0, connection/0]). - --optional_callbacks([ensure_subscribed/3, ensure_unsubscribed/2]). - -%% map fields depend on implementation --type(config() :: map()). --type(connection() :: term()). --type(batch() :: emqx_protal:batch()). --type(ack_ref() :: emqx_bridge_worker:ack_ref()). --type(topic() :: emqx_topic:topic()). --type(qos() :: emqx_mqtt_types:qos()). - --include_lib("emqx/include/logger.hrl"). - --logger_header("[Bridge Connect]"). - -%% establish the connection to remote node/cluster -%% protal worker (the caller process) should be expecting -%% a message {disconnected, conn_ref()} when disconnected. --callback start(config()) -> {ok, connection()} | {error, any()}. - -%% send to remote node/cluster -%% bridge worker (the caller process) should be expecting -%% a message {batch_ack, reference()} when batch is acknowledged by remote node/cluster --callback send(connection(), batch()) -> {ok, ack_ref()} | {ok, integer()} | {error, any()}. - -%% called when owner is shutting down. --callback stop(connection()) -> ok. - --callback ensure_subscribed(connection(), topic(), qos()) -> ok. - --callback ensure_unsubscribed(connection(), topic()) -> ok. - -start(Module, Config) -> - case Module:start(Config) of - {ok, Conn} -> - {ok, Conn}; - {error, Reason} -> - Config1 = obfuscate(Config), - ?LOG(error, "Failed to connect with module=~p\n" - "config=~p\nreason:~p", [Module, Config1, Reason]), - {error, Reason} - end. - -obfuscate(Map) -> - maps:fold(fun(K, V, Acc) -> - case is_sensitive(K) of - true -> [{K, '***'} | Acc]; - false -> [{K, V} | Acc] - end - end, [], Map). - -is_sensitive(password) -> true; -is_sensitive(_) -> false. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src index 83ce7a759..385c89965 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_mqtt, [{description, "EMQ X Bridge to MQTT Broker"}, - {vsn, "4.3.1"}, % strict semver, bump manually! + {vsn, "5.0.0"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,replayq,emqtt]}, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src deleted file mode 100644 index 03e6119ae..000000000 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src +++ /dev/null @@ -1,16 +0,0 @@ -%% -*-: erlang -*- - -{VSN, - [ - {"4.3.0", [ - {load_module, emqx_bridge_worker, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ], - [ - {"4.3.0", [ - {load_module, emqx_bridge_worker, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ] -}. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl index d612af668..8d442463b 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl @@ -18,15 +18,11 @@ -module(emqx_bridge_mqtt). --behaviour(emqx_bridge_connect). - -%% behaviour callbacks -export([ start/1 , send/2 , stop/1 ]). -%% optional behaviour callbacks -export([ ensure_subscribed/3 , ensure_unsubscribed/2 ]). @@ -37,6 +33,9 @@ , handle_disconnected/2 ]). +-export([ check_subscriptions/1 + ]). + -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). @@ -49,26 +48,31 @@ %% emqx_bridge_connect callbacks %%-------------------------------------------------------------------- -start(Config = #{address := Address}) -> +start(Config) -> Parent = self(), + Address = maps:get(address, Config), Mountpoint = maps:get(receive_mountpoint, Config, undefined), + Subscriptions = maps:get(subscriptions, Config, []), + Subscriptions1 = check_subscriptions(Subscriptions), Handlers = make_hdlr(Parent, Mountpoint), {Host, Port} = case string:tokens(Address, ":") of [H] -> {H, 1883}; [H, P] -> {H, list_to_integer(P)} end, - ClientConfig = Config#{msg_handler => Handlers, - host => Host, - port => Port, - force_ping => true - }, - case emqtt:start_link(replvar(ClientConfig)) of + Config1 = Config#{ + msg_handler => Handlers, + host => Host, + port => Port, + force_ping => true, + proto_ver => maps:get(proto_ver, Config, v4) + }, + case emqtt:start_link(without_config(Config1)) of {ok, Pid} -> case emqtt:connect(Pid) of {ok, _} -> try - subscribe_remote_topics(Pid, maps:get(subscriptions, Config, [])), - {ok, #{client_pid => Pid}} + Subscriptions2 = subscribe_remote_topics(Pid, Subscriptions1), + {ok, #{client_pid => Pid, subscriptions => Subscriptions2}} catch throw : Reason -> ok = stop(#{client_pid => Pid}), @@ -86,25 +90,25 @@ stop(#{client_pid := Pid}) -> safe_stop(Pid, fun() -> emqtt:stop(Pid) end, 1000), ok. -ensure_subscribed(#{client_pid := Pid}, Topic, QoS) when is_pid(Pid) -> +ensure_subscribed(#{client_pid := Pid, subscriptions := Subs} = Conn, Topic, QoS) when is_pid(Pid) -> case emqtt:subscribe(Pid, Topic, QoS) of - {ok, _, _} -> ok; - Error -> Error + {ok, _, _} -> Conn#{subscriptions => [{Topic, QoS}|Subs]}; + Error -> {error, Error} end; ensure_subscribed(_Conn, _Topic, _QoS) -> %% return ok for now %% next re-connect should should call start with new topic added to config ok. -ensure_unsubscribed(#{client_pid := Pid}, Topic) when is_pid(Pid) -> +ensure_unsubscribed(#{client_pid := Pid, subscriptions := Subs} = Conn, Topic) when is_pid(Pid) -> case emqtt:unsubscribe(Pid, Topic) of - {ok, _, _} -> ok; - Error -> Error + {ok, _, _} -> Conn#{subscriptions => lists:keydelete(Topic, 1, Subs)}; + Error -> {error, Error} end; -ensure_unsubscribed(_, _) -> +ensure_unsubscribed(Conn, _) -> %% return ok for now %% next re-connect should should call start with this topic deleted from config - ok. + Conn. safe_stop(Pid, StopF, Timeout) -> MRef = monitor(process, Pid), @@ -169,36 +173,18 @@ make_hdlr(Parent, Mountpoint) -> }. subscribe_remote_topics(ClientPid, Subscriptions) -> - lists:foreach(fun({Topic, Qos}) -> - case emqtt:subscribe(ClientPid, Topic, Qos) of - {ok, _, _} -> ok; - Error -> throw(Error) - end - end, Subscriptions). + lists:map(fun({Topic, Qos}) -> + case emqtt:subscribe(ClientPid, Topic, Qos) of + {ok, _, _} -> {Topic, Qos}; + Error -> throw(Error) + end + end, Subscriptions). -%%-------------------------------------------------------------------- -%% Internal funcs -%%-------------------------------------------------------------------- +without_config(Config) -> + maps:without([conn_type, address, receive_mountpoint, subscriptions], Config). -replvar(Options) -> - replvar([clientid, max_inflight], Options). - -replvar([], Options) -> - Options; -replvar([Key|More], Options) -> - case maps:get(Key, Options, undefined) of - undefined -> - replvar(More, Options); - Val -> - replvar(More, maps:put(Key, feedvar(Key, Val, Options), Options)) - end. - -%% ${node} => node() -feedvar(clientid, ClientId, _) -> - iolist_to_binary(re:replace(ClientId, "\\${node}", atom_to_list(node()))); - -feedvar(max_inflight, 0, _) -> - infinity; - -feedvar(max_inflight, Size, _) -> - Size. +check_subscriptions(Subscriptions) -> + lists:map(fun(#{qos := QoS, topic := Topic}) -> + true = emqx_topic:validate({filter, Topic}), + {Topic, QoS} + end, Subscriptions). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_app.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_app.erl index 2382b3a42..a145009c9 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_app.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_app.erl @@ -16,8 +16,6 @@ -module(emqx_bridge_mqtt_app). --emqx_plugin(bridge). - -behaviour(application). -export([start/2, stop/1]). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl new file mode 100644 index 000000000..8cc87ef64 --- /dev/null +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl @@ -0,0 +1,89 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_mqtt_schema). + +-include_lib("typerefl/include/types.hrl"). + +-behaviour(hocon_schema). + +-export([ structs/0 + , fields/1]). + +structs() -> ["emqx_bridge_mqtt"]. + +fields("emqx_bridge_mqtt") -> + [ {bridges, hoconsc:array(hoconsc:ref(?MODULE, "bridges"))} + ]; + +fields("bridges") -> + [ {name, emqx_schema:t(string(), undefined, true)} + , {start_type, fun start_type/1} + , {forwards, fun forwards/1} + , {forward_mountpoint, emqx_schema:t(string())} + , {reconnect_interval, emqx_schema:t(emqx_schema:duration_ms(), undefined, "30s")} + , {batch_size, emqx_schema:t(integer(), undefined, 100)} + , {queue, emqx_schema:t(hoconsc:ref(?MODULE, "queue"))} + , {config, hoconsc:union([hoconsc:ref(?MODULE, "mqtt"), hoconsc:ref(?MODULE, "rpc")])} + ]; + +fields("mqtt") -> + [ {conn_type, fun conn_type/1} + , {address, emqx_schema:t(string(), undefined, "127.0.0.1:1883")} + , {proto_ver, fun proto_ver/1} + , {bridge_mode, emqx_schema:t(boolean(), undefined, true)} + , {clientid, emqx_schema:t(string())} + , {username, emqx_schema:t(string())} + , {password, emqx_schema:t(string())} + , {clean_start, emqx_schema:t(boolean(), undefined, true)} + , {keepalive, emqx_schema:t(integer(), undefined, 300)} + , {subscriptions, hoconsc:array("subscriptions")} + , {receive_mountpoint, emqx_schema:t(string())} + , {retry_interval, emqx_schema:t(emqx_schema:duration_ms(), undefined, "30s")} + , {max_inflight, emqx_schema:t(integer(), undefined, 32)} + ]; + +fields("rpc") -> + [ {conn_type, fun conn_type/1} + , {node, emqx_schema:t(atom(), undefined, 'emqx@127.0.0.1')} + ]; + +fields("subscriptions") -> + [ {topic, #{type => binary(), nullable => false}} + , {qos, emqx_schema:t(integer(), undefined, 1)} + ]; + +fields("queue") -> + [ {replayq_dir, hoconsc:union([boolean(), string()])} + , {replayq_seg_bytes, emqx_schema:t(emqx_schema:bytesize(), undefined, "100MB")} + , {replayq_offload_mode, emqx_schema:t(boolean(), undefined, false)} + , {replayq_max_total_bytes, emqx_schema:t(emqx_schema:bytesize(), undefined, "1024MB")} + ]. + +conn_type(type) -> hoconsc:enum([mqtt, rpc]); +conn_type(_) -> undefined. + +proto_ver(type) -> hoconsc:enum([v3, v4, v5]); +proto_ver(default) -> v4; +proto_ver(_) -> undefined. + +start_type(type) -> hoconsc:enum([auto, manual]); +start_type(default) -> auto; +start_type(_) -> undefined. + +forwards(type) -> hoconsc:array(binary()); +forwards(default) -> []; +forwards(_) -> undefined. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl index 80a11c1c0..0075b4a1d 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl @@ -24,37 +24,33 @@ %% APIs -export([ start_link/0 - , start_link/1 ]). --export([ create_bridge/2 +-export([ create_bridge/1 , drop_bridge/1 , bridges/0 - , is_bridge_exist/1 ]). %% supervisor callbacks -export([init/1]). --define(SUP, ?MODULE). -define(WORKER_SUP, emqx_bridge_worker_sup). -start_link() -> start_link(?SUP). +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). -start_link(Name) -> - supervisor:start_link({local, Name}, ?MODULE, Name). - -init(?SUP) -> - BridgesConf = application:get_env(?APP, bridges, []), +init([]) -> + BridgesConf = emqx_config:get([?APP, bridges], []), BridgeSpec = lists:map(fun bridge_spec/1, BridgesConf), SupFlag = #{strategy => one_for_one, intensity => 100, period => 10}, {ok, {SupFlag, BridgeSpec}}. -bridge_spec({Name, Config}) -> +bridge_spec(Config) -> + Name = list_to_atom(maps:get(name, Config)), #{id => Name, - start => {emqx_bridge_worker, start_link, [Name, Config]}, + start => {emqx_bridge_worker, start_link, [Config]}, restart => permanent, shutdown => 5000, type => worker, @@ -62,22 +58,15 @@ bridge_spec({Name, Config}) -> -spec(bridges() -> [{node(), map()}]). bridges() -> - [{Name, emqx_bridge_worker:status(Pid)} || {Name, Pid, _, _} <- supervisor:which_children(?SUP)]. + [{Name, emqx_bridge_worker:status(Name)} || {Name, _Pid, _, _} <- supervisor:which_children(?MODULE)]. --spec(is_bridge_exist(atom() | pid()) -> boolean()). -is_bridge_exist(Id) -> - case supervisor:get_childspec(?SUP, Id) of - {ok, _ChildSpec} -> true; - {error, _Error} -> false - end. +create_bridge(Config) -> + supervisor:start_child(?MODULE, bridge_spec(Config)). -create_bridge(Id, Config) -> - supervisor:start_child(?SUP, bridge_spec({Id, Config})). - -drop_bridge(Id) -> - case supervisor:terminate_child(?SUP, Id) of +drop_bridge(Name) -> + case supervisor:terminate_child(?MODULE, Name) of ok -> - supervisor:delete_child(?SUP, Id); + supervisor:delete_child(?MODULE, Name); {error, Error} -> ?LOG(error, "Delete bridge failed, error : ~p", [Error]), {error, Error} diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_rpc.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_rpc.erl index 0cf4b5bc5..33511cc03 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_rpc.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_rpc.erl @@ -18,9 +18,6 @@ -module(emqx_bridge_rpc). --behaviour(emqx_bridge_connect). - -%% behaviour callbacks -export([ start/1 , send/2 , stop/1 @@ -33,17 +30,15 @@ -type ack_ref() :: emqx_bridge_worker:ack_ref(). -type batch() :: emqx_bridge_worker:batch(). --type node_or_tuple() :: atom() | {atom(), term()}. - -define(HEARTBEAT_INTERVAL, timer:seconds(1)). -define(RPC, emqx_rpc). -start(#{address := Remote}) -> - case poke(Remote) of +start(#{node := RemoteNode}) -> + case poke(RemoteNode) of ok -> - Pid = proc_lib:spawn_link(?MODULE, heartbeat, [self(), Remote]), - {ok, #{client_pid => Pid, address => Remote}}; + Pid = proc_lib:spawn_link(?MODULE, heartbeat, [self(), RemoteNode]), + {ok, #{client_pid => Pid, remote_node => RemoteNode}}; Error -> Error end. @@ -62,9 +57,9 @@ stop(#{client_pid := Pid}) when is_pid(Pid) -> ok. %% @doc Callback for `emqx_bridge_connect' behaviour --spec send(#{address := node_or_tuple(), _ => _}, batch()) -> {ok, ack_ref()} | {error, any()}. -send(#{address := Remote}, Batch) -> - case ?RPC:call(Remote, ?MODULE, handle_send, [Batch]) of +-spec send(#{remote_node := atom(), _ => _}, batch()) -> {ok, ack_ref()} | {error, any()}. +send(#{remote_node := RemoteNode}, Batch) -> + case ?RPC:call(RemoteNode, ?MODULE, handle_send, [Batch]) of ok -> Ref = make_ref(), self() ! {batch_ack, Ref}, @@ -93,8 +88,8 @@ heartbeat(Parent, RemoteNode) -> end end. -poke(Node) -> - case ?RPC:call(Node, erlang, node, []) of - Node -> ok; +poke(RemoteNode) -> + case ?RPC:call(RemoteNode, erlang, node, []) of + RemoteNode -> ok; {badrpc, Reason} -> {error, Reason} end. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl index e7414683c..dfef6973e 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl @@ -66,7 +66,6 @@ %% APIs -export([ start_link/1 - , start_link/2 , register_metrics/0 , stop/1 ]). @@ -86,7 +85,6 @@ %% management APIs -export([ ensure_started/1 , ensure_stopped/1 - , ensure_stopped/2 , status/1 ]). @@ -125,14 +123,13 @@ -define(DEFAULT_RECONNECT_DELAY_MS, timer:seconds(5)). -define(DEFAULT_SEG_BYTES, (1 bsl 20)). -define(DEFAULT_MAX_TOTAL_SIZE, (1 bsl 31)). --define(NO_BRIDGE_HANDLER, undefined). %% @doc Start a bridge worker. Supported configs: %% start_type: 'manual' (default) or 'auto', when manual, bridge will stay %% at 'idle' state until a manual call to start it. %% connect_module: The module which implements emqx_bridge_connect behaviour %% and work as message batch transport layer -%% reconnect_delay_ms: Delay in milli-seconds for the bridge worker to retry +%% reconnect_interval: Delay in milli-seconds for the bridge worker to retry %% in case of transportation failure. %% max_inflight: Max number of batches allowed to send-ahead before receiving %% confirmation from remote node/cluster @@ -148,128 +145,98 @@ %% %% Find more connection specific configs in the callback modules %% of emqx_bridge_connect behaviour. -start_link(Config) when is_list(Config) -> - start_link(maps:from_list(Config)); -start_link(Config) -> - gen_statem:start_link(?MODULE, Config, []). - -start_link(Name, Config) when is_list(Config) -> - start_link(Name, maps:from_list(Config)); -start_link(Name, Config) -> - Name1 = name(Name), - gen_statem:start_link({local, Name1}, ?MODULE, Config#{name => Name1}, []). +start_link(Opts) when is_list(Opts) -> + start_link(maps:from_list(Opts)); +start_link(Opts) -> + case maps:get(name, Opts, undefined) of + undefined -> + gen_statem:start_link(?MODULE, Opts, []); + Name -> + Name1 = name(Name), + gen_statem:start_link({local, Name1}, ?MODULE, Opts#{name => Name1}, []) + end. ensure_started(Name) -> gen_statem:call(name(Name), ensure_started). %% @doc Manually stop bridge worker. State idempotency ensured. -ensure_stopped(Id) -> - ensure_stopped(Id, 1000). - -ensure_stopped(Id, Timeout) -> - Pid = case id(Id) of - P when is_pid(P) -> P; - N -> whereis(N) - end, - case Pid of - undefined -> - ok; - _ -> - MRef = monitor(process, Pid), - unlink(Pid), - _ = gen_statem:call(id(Id), ensure_stopped, Timeout), - receive - {'DOWN', MRef, _, _, _} -> - ok - after - Timeout -> - exit(Pid, kill) - end - end. +ensure_stopped(Name) -> + gen_statem:call(name(Name), ensure_stopped, 5000). stop(Pid) -> gen_statem:stop(Pid). status(Pid) when is_pid(Pid) -> gen_statem:call(Pid, status); -status(Id) -> - gen_statem:call(name(Id), status). +status(Name) -> + gen_statem:call(name(Name), status). %% @doc Return all forwards (local subscriptions). -spec get_forwards(id()) -> [topic()]. -get_forwards(Id) -> gen_statem:call(id(Id), get_forwards, timer:seconds(1000)). +get_forwards(Name) -> gen_statem:call(name(Name), get_forwards, timer:seconds(1000)). %% @doc Return all subscriptions (subscription over mqtt connection to remote broker). -spec get_subscriptions(id()) -> [{emqx_topic:topic(), qos()}]. -get_subscriptions(Id) -> gen_statem:call(id(Id), get_subscriptions). +get_subscriptions(Name) -> gen_statem:call(name(Name), get_subscriptions). %% @doc Add a new forward (local topic subscription). -spec ensure_forward_present(id(), topic()) -> ok. -ensure_forward_present(Id, Topic) -> - gen_statem:call(id(Id), {ensure_present, forwards, topic(Topic)}). +ensure_forward_present(Name, Topic) -> + gen_statem:call(name(Name), {ensure_forward_present, topic(Topic)}). %% @doc Ensure a forward topic is deleted. -spec ensure_forward_absent(id(), topic()) -> ok. -ensure_forward_absent(Id, Topic) -> - gen_statem:call(id(Id), {ensure_absent, forwards, topic(Topic)}). +ensure_forward_absent(Name, Topic) -> + gen_statem:call(name(Name), {ensure_forward_absent, topic(Topic)}). %% @doc Ensure subscribed to remote topic. %% NOTE: only applicable when connection module is emqx_bridge_mqtt %% return `{error, no_remote_subscription_support}' otherwise. -spec ensure_subscription_present(id(), topic(), qos()) -> ok | {error, any()}. -ensure_subscription_present(Id, Topic, QoS) -> - gen_statem:call(id(Id), {ensure_present, subscriptions, {topic(Topic), QoS}}). +ensure_subscription_present(Name, Topic, QoS) -> + gen_statem:call(name(Name), {ensure_subscription_present, topic(Topic), QoS}). %% @doc Ensure unsubscribed from remote topic. %% NOTE: only applicable when connection module is emqx_bridge_mqtt -spec ensure_subscription_absent(id(), topic()) -> ok. -ensure_subscription_absent(Id, Topic) -> - gen_statem:call(id(Id), {ensure_absent, subscriptions, topic(Topic)}). +ensure_subscription_absent(Name, Topic) -> + gen_statem:call(name(Name), {ensure_subscription_absent, topic(Topic)}). callback_mode() -> [state_functions]. %% @doc Config should be a map(). -init(Config) -> +init(Opts) -> erlang:process_flag(trap_exit, true), - ConnectModule = maps:get(connect_module, Config), - Subscriptions = maps:get(subscriptions, Config, []), - Forwards = maps:get(forwards, Config, []), - Queue = open_replayq(Config), - State = init_opts(Config), - Topics = [iolist_to_binary(T) || T <- Forwards], - Subs = check_subscriptions(Subscriptions), - ConnectCfg = get_conn_cfg(Config), + ConnectOpts = maps:get(config, Opts), + ConnectModule = conn_type(maps:get(conn_type, ConnectOpts)), + Forwards = maps:get(forwards, Opts, []), + Queue = open_replayq(maps:get(queue, Opts, #{})), + State = init_opts(Opts), self() ! idle, {ok, idle, State#{connect_module => ConnectModule, - connect_cfg => ConnectCfg, - forwards => Topics, - subscriptions => Subs, + connect_opts => ConnectOpts, + forwards => Forwards, replayq => Queue }}. -init_opts(Config) -> - IfRecordMetrics = maps:get(if_record_metrics, Config, true), - ReconnDelayMs = maps:get(reconnect_delay_ms, Config, ?DEFAULT_RECONNECT_DELAY_MS), - StartType = maps:get(start_type, Config, manual), - BridgeHandler = maps:get(bridge_handler, Config, ?NO_BRIDGE_HANDLER), - Mountpoint = maps:get(forward_mountpoint, Config, undefined), - ReceiveMountpoint = maps:get(receive_mountpoint, Config, undefined), - MaxInflightSize = maps:get(max_inflight, Config, ?DEFAULT_BATCH_SIZE), - BatchSize = maps:get(batch_size, Config, ?DEFAULT_BATCH_SIZE), - Name = maps:get(name, Config, undefined), +init_opts(Opts) -> + IfRecordMetrics = maps:get(if_record_metrics, Opts, true), + ReconnDelayMs = maps:get(reconnect_interval, Opts, ?DEFAULT_RECONNECT_DELAY_MS), + StartType = maps:get(start_type, Opts, manual), + Mountpoint = maps:get(forward_mountpoint, Opts, undefined), + MaxInflightSize = maps:get(max_inflight, Opts, ?DEFAULT_BATCH_SIZE), + BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE), + Name = maps:get(name, Opts, undefined), #{start_type => StartType, - reconnect_delay_ms => ReconnDelayMs, + reconnect_interval => ReconnDelayMs, batch_size => BatchSize, mountpoint => format_mountpoint(Mountpoint), - receive_mountpoint => ReceiveMountpoint, inflight => [], max_inflight => MaxInflightSize, connection => undefined, - bridge_handler => BridgeHandler, if_record_metrics => IfRecordMetrics, name => Name}. -open_replayq(Config) -> - QCfg = maps:get(queue, Config, #{}), +open_replayq(QCfg) -> Dir = maps:get(replayq_dir, QCfg, undefined), SegBytes = maps:get(replayq_seg_bytes, QCfg, ?DEFAULT_SEG_BYTES), MaxTotalSize = maps:get(max_total_size, QCfg, ?DEFAULT_MAX_TOTAL_SIZE), @@ -280,22 +247,6 @@ open_replayq(Config) -> replayq:open(QueueConfig#{sizer => fun emqx_bridge_msg:estimate_size/1, marshaller => fun ?MODULE:msg_marshaller/1}). -check_subscriptions(Subscriptions) -> - lists:map(fun({Topic, QoS}) -> - Topic1 = iolist_to_binary(Topic), - true = emqx_topic:validate({filter, Topic1}), - {Topic1, QoS} - end, Subscriptions). - -get_conn_cfg(Config) -> - maps:without([connect_module, - queue, - reconnect_delay_ms, - forwards, - mountpoint, - name - ], Config). - code_change(_Vsn, State, Data, _Extra) -> {ok, State, Data}. @@ -321,14 +272,10 @@ idle(info, idle, #{start_type := auto} = State) -> idle(state_timeout, reconnect, State) -> connecting(State); -idle(info, {batch_ack, Ref}, State) -> - NewState = handle_batch_ack(State, Ref), - {keep_state, NewState}; - idle(Type, Content, State) -> common(idle, Type, Content, State). -connecting(#{reconnect_delay_ms := ReconnectDelayMs} = State) -> +connecting(#{reconnect_interval := ReconnectDelayMs} = State) -> case do_connect(State) of {ok, State1} -> {next_state, connected, State1, {state_timeout, 0, connected}}; @@ -348,7 +295,7 @@ connected(internal, maybe_send, State) -> {keep_state, NewState}; connected(info, {disconnected, Conn, Reason}, - #{connection := Connection, name := Name, reconnect_delay_ms := ReconnectDelayMs} = State) -> + #{connection := Connection, name := Name, reconnect_interval := ReconnectDelayMs} = State) -> ?tp(info, disconnected, #{name => Name, reason => Reason}), case Conn =:= maps:get(client_pid, Connection, undefined) of true -> @@ -365,19 +312,27 @@ connected(Type, Content, State) -> %% Common handlers common(StateName, {call, From}, status, _State) -> {keep_state_and_data, [{reply, From, StateName}]}; -common(_StateName, {call, From}, ensure_started, _State) -> - {keep_state_and_data, [{reply, From, connected}]}; -common(_StateName, {call, From}, ensure_stopped, _State) -> - {stop_and_reply, {shutdown, manual}, [{reply, From, ok}]}; +common(_StateName, {call, From}, ensure_stopped, #{connection := undefined} = _State) -> + {keep_state_and_data, [{reply, From, ok}]}; +common(_StateName, {call, From}, ensure_stopped, #{connection := Conn, + connect_module := ConnectModule} = State) -> + Reply = ConnectModule:stop(Conn), + {next_state, idle, State#{connection => undefined}, [{reply, From, Reply}]}; common(_StateName, {call, From}, get_forwards, #{forwards := Forwards}) -> {keep_state_and_data, [{reply, From, Forwards}]}; -common(_StateName, {call, From}, get_subscriptions, #{subscriptions := Subs}) -> - {keep_state_and_data, [{reply, From, Subs}]}; -common(_StateName, {call, From}, {ensure_present, What, Topic}, State) -> - {Result, NewState} = ensure_present(What, Topic, State), +common(_StateName, {call, From}, get_subscriptions, #{connection := Connection}) -> + {keep_state_and_data, [{reply, From, maps:get(subscriptions, Connection, [])}]}; +common(_StateName, {call, From}, {ensure_forward_present, Topic}, State) -> + {Result, NewState} = do_ensure_forward_present(Topic, State), {keep_state, NewState, [{reply, From, Result}]}; -common(_StateName, {call, From}, {ensure_absent, What, Topic}, State) -> - {Result, NewState} = ensure_absent(What, Topic, State), +common(_StateName, {call, From}, {ensure_subscription_present, Topic, QoS}, State) -> + {Result, NewState} = do_ensure_subscription_present(Topic, QoS, State), + {keep_state, NewState, [{reply, From, Result}]}; +common(_StateName, {call, From}, {ensure_forward_absent, Topic}, State) -> + {Result, NewState} = do_ensure_forward_absent(Topic, State), + {keep_state, NewState, [{reply, From, Result}]}; +common(_StateName, {call, From}, {ensure_subscription_absent, Topic}, State) -> + {Result, NewState} = do_ensure_subscription_absent(Topic, State), {keep_state, NewState, [{reply, From, Result}]}; common(_StateName, info, {deliver, _, Msg}, State = #{replayq := Q, if_record_metrics := IfRecordMetric}) -> @@ -395,77 +350,79 @@ common(StateName, Type, Content, #{name := Name} = State) -> [Name, Type, StateName, Content]), {keep_state, State}. -eval_bridge_handler(State = #{bridge_handler := ?NO_BRIDGE_HANDLER}, _Msg) -> - State; -eval_bridge_handler(State = #{bridge_handler := Handler}, Msg) -> - Handler(Msg), - State. - -ensure_present(Key, Topic, State) -> - Topics = maps:get(Key, State), - case is_topic_present(Topic, Topics) of +do_ensure_forward_present(Topic, #{forwards := Forwards, name := Name} = State) -> + case is_topic_present(Topic, Forwards) of true -> {ok, State}; false -> - R = do_ensure_present(Key, Topic, State), - {R, State#{Key := lists:usort([Topic | Topics])}} + R = subscribe_local_topic(Topic, Name), + {R, State#{forwards => [Topic | Forwards]}} end. -ensure_absent(Key, Topic, State) -> - Topics = maps:get(Key, State), - case is_topic_present(Topic, Topics) of +do_ensure_subscription_present(_Topic, _QoS, #{connection := undefined} = State) -> + {{error, no_connection}, State}; +do_ensure_subscription_present(_Topic, _QoS, #{connect_module := emqx_bridge_rpc} = State) -> + {{error, no_remote_subscription_support}, State}; +do_ensure_subscription_present(Topic, QoS, #{connect_module := ConnectModule, + connection := Conn} = State) -> + case is_topic_present(Topic, maps:get(subscriptions, Conn, [])) of true -> - R = do_ensure_absent(Key, Topic, State), - {R, State#{Key := ensure_topic_absent(Topic, Topics)}}; + {ok, State}; + false -> + case ConnectModule:ensure_subscribed(Conn, Topic, QoS) of + {error, Error} -> + {{error, Error}, State}; + Conn1 -> + {ok, State#{connection => Conn1}} + end + end. + +do_ensure_forward_absent(Topic, #{forwards := Forwards} = State) -> + case is_topic_present(Topic, Forwards) of + true -> + R = do_unsubscribe(Topic), + {R, State#{forwards => lists:delete(Topic, Forwards)}}; + false -> + {ok, State} + end. +do_ensure_subscription_absent(_Topic, #{connection := undefined} = State) -> + {{error, no_connection}, State}; +do_ensure_subscription_absent(_Topic, #{connect_module := emqx_bridge_rpc} = State) -> + {{error, no_remote_subscription_support}, State}; +do_ensure_subscription_absent(Topic, #{connect_module := ConnectModule, + connection := Conn} = State) -> + case is_topic_present(Topic, maps:get(subscriptions, Conn, [])) of + true -> + case ConnectModule:ensure_unsubscribed(Conn, Topic) of + {error, Error} -> + {{error, Error}, State}; + Conn1 -> + {ok, State#{connection => Conn1}} + end; false -> {ok, State} end. -ensure_topic_absent(_Topic, []) -> []; -ensure_topic_absent(Topic, [{_, _} | _] = L) -> lists:keydelete(Topic, 1, L); -ensure_topic_absent(Topic, L) -> lists:delete(Topic, L). - -is_topic_present({Topic, _QoS}, Topics) -> - is_topic_present(Topic, Topics); is_topic_present(Topic, Topics) -> lists:member(Topic, Topics) orelse false =/= lists:keyfind(Topic, 1, Topics). do_connect(#{forwards := Forwards, - subscriptions := Subs, connect_module := ConnectModule, - connect_cfg := ConnectCfg, + connect_opts := ConnectOpts, inflight := Inflight, name := Name} = State) -> ok = subscribe_local_topics(Forwards, Name), - case emqx_bridge_connect:start(ConnectModule, ConnectCfg#{subscriptions => Subs}) of + case ConnectModule:start(ConnectOpts) of {ok, Conn} -> - Res = eval_bridge_handler(State#{connection => Conn}, connected), ?tp(info, connected, #{name => Name, inflight => length(Inflight)}), - {ok, Res}; + {ok, State#{connection => Conn}}; {error, Reason} -> + ConnectOpts1 = obfuscate(ConnectOpts), + ?LOG(error, "Failed to connect with module=~p\n" + "config=~p\nreason:~p", [ConnectModule, ConnectOpts1, Reason]), {error, Reason, State} end. -do_ensure_present(forwards, Topic, #{name := Name}) -> - subscribe_local_topic(Topic, Name); -do_ensure_present(subscriptions, _Topic, #{connection := undefined}) -> - {error, no_connection}; -do_ensure_present(subscriptions, _Topic, #{connect_module := emqx_bridge_rpc}) -> - {error, no_remote_subscription_support}; -do_ensure_present(subscriptions, {Topic, QoS}, #{connect_module := ConnectModule, - connection := Conn}) -> - ConnectModule:ensure_subscribed(Conn, Topic, QoS). - -do_ensure_absent(forwards, Topic, _) -> - do_unsubscribe(Topic); -do_ensure_absent(subscriptions, _Topic, #{connection := undefined}) -> - {error, no_connection}; -do_ensure_absent(subscriptions, _Topic, #{connect_module := emqx_bridge_rpc}) -> - {error, no_remote_subscription_support}; -do_ensure_absent(subscriptions, Topic, #{connect_module := ConnectModule, - connection := Conn}) -> - ConnectModule:ensure_unsubscribed(Conn, Topic). - collect(Acc) -> receive {deliver, _, Msg} -> @@ -605,10 +562,9 @@ disconnect(#{connection := Conn, connect_module := Module } = State) when Conn =/= undefined -> Module:stop(Conn), - State0 = State#{connection => undefined}, - eval_bridge_handler(State0, disconnected); + State#{connection => undefined}; disconnect(State) -> - eval_bridge_handler(State, disconnected). + State. %% Called only when replayq needs to dump it to disk. msg_marshaller(Bin) when is_binary(Bin) -> emqx_bridge_msg:from_binary(Bin); @@ -621,9 +577,6 @@ format_mountpoint(Prefix) -> name(Id) -> list_to_atom(lists:concat([?MODULE, "_", Id])). -id(Pid) when is_pid(Pid) -> Pid; -id(Name) -> name(Name). - register_metrics() -> lists:foreach(fun emqx_metrics:ensure/1, ['bridge.mqtt.message_sent', @@ -639,3 +592,21 @@ bridges_metrics_inc(true, Metric, Value) -> emqx_metrics:inc(Metric, Value); bridges_metrics_inc(_IsRecordMetric, _Metric, _Value) -> ok. + +obfuscate(Map) -> + maps:fold(fun(K, V, Acc) -> + case is_sensitive(K) of + true -> [{K, '***'} | Acc]; + false -> [{K, V} | Acc] + end + end, [], Map). + +is_sensitive(password) -> true; +is_sensitive(_) -> false. + +conn_type(rpc) -> + emqx_bridge_rpc; +conn_type(mqtt) -> + emqx_bridge_mqtt; +conn_type(Mod) when is_atom(Mod) -> + Mod. diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_tests.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_tests.erl index 830fb1fe0..5babe0ed9 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_tests.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_tests.erl @@ -44,4 +44,4 @@ send_and_ack_test() -> ok = emqx_bridge_mqtt:stop(Conn) after meck:unload(emqtt) - end. \ No newline at end of file + end. diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_rpc_tests.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_rpc_tests.erl index fdcc25d5f..cbd80ba3d 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_rpc_tests.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_rpc_tests.erl @@ -30,8 +30,8 @@ send_and_ack_test() -> end), meck:new(emqx_bridge_worker, [passthrough, no_history]), try - {ok, #{client_pid := Pid, address := Node}} = emqx_bridge_rpc:start(#{address => node()}), - {ok, Ref} = emqx_bridge_rpc:send(#{address => Node}, []), + {ok, #{client_pid := Pid, remote_node := Node}} = emqx_bridge_rpc:start(#{node => node()}), + {ok, Ref} = emqx_bridge_rpc:send(#{remote_node => Node}, []), receive {batch_ack, Ref} -> ok diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_stub_conn.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_stub_conn.erl index d38663fcd..4c2fde6dd 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_stub_conn.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_stub_conn.erl @@ -16,9 +16,6 @@ -module(emqx_bridge_stub_conn). --behaviour(emqx_bridge_connect). - -%% behaviour callbacks -export([ start/1 , send/2 , stop/1 diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_SUITE.erl index 680756742..f3f5d5ceb 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_SUITE.erl @@ -59,13 +59,12 @@ init_per_suite(Config) -> nonode@nohost -> net_kernel:start(['emqx@127.0.0.1', longnames]); _ -> ok end, - ok = application:set_env(gen_rpc, tcp_client_num, 1), - emqx_ct_helpers:start_apps([emqx_modules, emqx_bridge_mqtt]), + emqx_ct_helpers:start_apps([emqx_bridge_mqtt]), emqx_logger:set_log_level(error), [{log_level, error} | Config]. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_bridge_mqtt, emqx_modules]). + emqx_ct_helpers:stop_apps([emqx_bridge_mqtt]). init_per_testcase(_TestCase, Config) -> ok = snabbkaffe:start_trace(), @@ -74,260 +73,290 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, _Config) -> ok = snabbkaffe:stop(). -t_mngr(Config) when is_list(Config) -> - Subs = [{<<"a">>, 1}, {<<"b">>, 2}], - Cfg = #{address => node(), - forwards => [<<"mngr">>], - connect_module => emqx_bridge_rpc, - mountpoint => <<"forwarded">>, - subscriptions => Subs, - start_type => auto}, - Name = ?FUNCTION_NAME, - {ok, Pid} = emqx_bridge_worker:start_link(Name, Cfg), - try - ?assertEqual([<<"mngr">>], emqx_bridge_worker:get_forwards(Name)), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_present(Name, "mngr")), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_present(Name, "mngr2")), - ?assertEqual([<<"mngr">>, <<"mngr2">>], emqx_bridge_worker:get_forwards(Pid)), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_absent(Name, "mngr2")), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_absent(Name, "mngr3")), - ?assertEqual([<<"mngr">>], emqx_bridge_worker:get_forwards(Pid)), - ?assertEqual({error, no_remote_subscription_support}, - emqx_bridge_worker:ensure_subscription_present(Pid, <<"t">>, 0)), - ?assertEqual({error, no_remote_subscription_support}, - emqx_bridge_worker:ensure_subscription_absent(Pid, <<"t">>)), - ?assertEqual(Subs, emqx_bridge_worker:get_subscriptions(Pid)) - after - ok = emqx_bridge_worker:stop(Pid) - end. +t_rpc_mngr(_Config) -> + Name = "rpc_name", + Cfg = #{ + name => Name, + forwards => [<<"mngr">>], + forward_mountpoint => <<"forwarded">>, + start_type => auto, + config => #{ + conn_type => rpc, + node => node() + } + }, + {ok, Pid} = emqx_bridge_mqtt_sup:create_bridge(Cfg), + ?assertEqual([<<"mngr">>], emqx_bridge_worker:get_forwards(Name)), + ?assertEqual(ok, emqx_bridge_worker:ensure_forward_present(Name, "mngr")), + ?assertEqual(ok, emqx_bridge_worker:ensure_forward_present(Name, "mngr2")), + ?assertEqual([<<"mngr2">>, <<"mngr">>], emqx_bridge_worker:get_forwards(Name)), + ?assertEqual(ok, emqx_bridge_worker:ensure_forward_absent(Name, "mngr2")), + ?assertEqual(ok, emqx_bridge_worker:ensure_forward_absent(Name, "mngr3")), + ?assertEqual([<<"mngr">>], emqx_bridge_worker:get_forwards(Name)), + ?assertEqual({error, no_remote_subscription_support}, + emqx_bridge_worker:ensure_subscription_present(Name, <<"t">>, 0)), + ?assertEqual({error, no_remote_subscription_support}, + emqx_bridge_worker:ensure_subscription_absent(Name, <<"t">>)), + ok = emqx_bridge_worker:stop(Pid). + +t_mqtt_mngr(_Config) -> + Name = "mqtt_name", + Cfg = #{ + name => Name, + forwards => [<<"mngr">>], + forward_mountpoint => <<"forwarded">>, + start_type => auto, + config => #{ + address => "127.0.0.1:1883", + conn_type => mqtt, + clientid => <<"client1">>, + keepalive => 300, + subscriptions => [#{topic => <<"t/#">>, qos => 1}] + } + }, + {ok, Pid} = emqx_bridge_mqtt_sup:create_bridge(Cfg), + ?assertEqual([<<"mngr">>], emqx_bridge_worker:get_forwards(Name)), + ?assertEqual(ok, emqx_bridge_worker:ensure_forward_present(Name, "mngr")), + ?assertEqual(ok, emqx_bridge_worker:ensure_forward_present(Name, "mngr2")), + ?assertEqual([<<"mngr2">>, <<"mngr">>], emqx_bridge_worker:get_forwards(Name)), + ?assertEqual(ok, emqx_bridge_worker:ensure_forward_absent(Name, "mngr2")), + ?assertEqual(ok, emqx_bridge_worker:ensure_forward_absent(Name, "mngr3")), + ?assertEqual([<<"mngr">>], emqx_bridge_worker:get_forwards(Name)), + ?assertEqual(ok, emqx_bridge_worker:ensure_subscription_present(Name, <<"t">>, 0)), + ?assertEqual(ok, emqx_bridge_worker:ensure_subscription_absent(Name, <<"t">>)), + ?assertEqual([{<<"t/#">>,1}], emqx_bridge_worker:get_subscriptions(Name)), + ok = emqx_bridge_worker:stop(Pid). %% A loopback RPC to local node -t_rpc(Config) when is_list(Config) -> - Cfg = #{address => node(), - forwards => [<<"t_rpc/#">>], - connect_module => emqx_bridge_rpc, - forward_mountpoint => <<"forwarded">>, - start_type => auto}, - {ok, Pid} = emqx_bridge_worker:start_link(?FUNCTION_NAME, Cfg), - ClientId = <<"ClientId">>, - try - {ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]), - {ok, _Props} = emqtt:connect(ConnPid), - {ok, _Props, [1]} = emqtt:subscribe(ConnPid, {<<"forwarded/t_rpc/one">>, ?QOS_1}), - timer:sleep(100), - {ok, _PacketId} = emqtt:publish(ConnPid, <<"t_rpc/one">>, <<"hello">>, ?QOS_1), - timer:sleep(100), - ?assertEqual(1, length(receive_messages(1))), - emqtt:disconnect(ConnPid) - after - ok = emqx_bridge_worker:stop(Pid) - end. +t_rpc(_Config) -> + Name = "rpc", + Cfg = #{ + name => Name, + forwards => [<<"t_rpc/#">>], + forward_mountpoint => <<"forwarded">>, + start_type => auto, + config => #{ + conn_type => rpc, + node => node() + } + }, + {ok, Pid} = emqx_bridge_mqtt_sup:create_bridge(Cfg), + {ok, ConnPid} = emqtt:start_link([{clientid, <<"ClientId">>}]), + {ok, _Props} = emqtt:connect(ConnPid), + {ok, _Props, [1]} = emqtt:subscribe(ConnPid, {<<"forwarded/t_rpc/one">>, ?QOS_1}), + timer:sleep(100), + {ok, _PacketId} = emqtt:publish(ConnPid, <<"t_rpc/one">>, <<"hello">>, ?QOS_1), + timer:sleep(100), + ?assertEqual(1, length(receive_messages(1))), + emqtt:disconnect(ConnPid), + emqx_bridge_worker:stop(Pid). %% Full data loopback flow explained: %% mqtt-client ----> local-broker ---(local-subscription)---> %% bridge(export) --- (mqtt-connection)--> local-broker ---(remote-subscription) --> %% bridge(import) --> mqtt-client -t_mqtt(Config) when is_list(Config) -> +t_mqtt(_Config) -> SendToTopic = <<"t_mqtt/one">>, SendToTopic2 = <<"t_mqtt/two">>, SendToTopic3 = <<"t_mqtt/three">>, Mountpoint = <<"forwarded/${node}/">>, - Cfg = #{address => "127.0.0.1:1883", - forwards => [SendToTopic], - connect_module => emqx_bridge_mqtt, - forward_mountpoint => Mountpoint, - username => "user", - clean_start => true, - clientid => "bridge_aws", - keepalive => 60000, - password => "passwd", - proto_ver => mqttv4, - queue => #{replayq_dir => "data/t_mqtt/", - replayq_seg_bytes => 10000, - batch_bytes_limit => 1000, - batch_count_limit => 10 - }, - reconnect_delay_ms => 1000, - ssl => false, - %% Consume back to forwarded message for verification - %% NOTE: this is a indefenite loopback without mocking emqx_bridge_worker:import_batch/1 - subscriptions => [{SendToTopic2, _QoS = 1}], - receive_mountpoint => <<"receive/aws/">>, - start_type => auto}, - {ok, Pid} = emqx_bridge_worker:start_link(?FUNCTION_NAME, Cfg), - ClientId = <<"client-1">>, - try - ?assertEqual([{SendToTopic2, 1}], emqx_bridge_worker:get_subscriptions(Pid)), - ok = emqx_bridge_worker:ensure_subscription_present(Pid, SendToTopic3, _QoS = 1), - ?assertEqual([{SendToTopic3, 1},{SendToTopic2, 1}], - emqx_bridge_worker:get_subscriptions(Pid)), - {ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]), - {ok, _Props} = emqtt:connect(ConnPid), - emqtt:subscribe(ConnPid, <<"forwarded/+/t_mqtt/one">>, 1), - %% message from a different client, to avoid getting terminated by no-local - Max = 10, - Msgs = lists:seq(1, Max), - lists:foreach(fun(I) -> - {ok, _PacketId} = emqtt:publish(ConnPid, SendToTopic, integer_to_binary(I), ?QOS_1) - end, Msgs), - ?assertEqual(10, length(receive_messages(200))), + Name = "mqtt", + Cfg = #{ + name => Name, + forwards => [SendToTopic], + forward_mountpoint => Mountpoint, + start_type => auto, + config => #{ + address => "127.0.0.1:1883", + conn_type => mqtt, + clientid => <<"client1">>, + keepalive => 300, + subscriptions => [#{topic => SendToTopic2, qos => 1}], + receive_mountpoint => <<"receive/aws/">> + }, + queue => #{ + replayq_dir => "data/t_mqtt/", + replayq_seg_bytes => 10000, + batch_bytes_limit => 1000, + batch_count_limit => 10 + } + }, + {ok, Pid} = emqx_bridge_mqtt_sup:create_bridge(Cfg), + ?assertEqual([{SendToTopic2, 1}], emqx_bridge_worker:get_subscriptions(Name)), + ok = emqx_bridge_worker:ensure_subscription_present(Name, SendToTopic3, _QoS = 1), + ?assertEqual([{SendToTopic3, 1},{SendToTopic2, 1}], + emqx_bridge_worker:get_subscriptions(Name)), + {ok, ConnPid} = emqtt:start_link([{clientid, <<"client-1">>}]), + {ok, _Props} = emqtt:connect(ConnPid), + emqtt:subscribe(ConnPid, <<"forwarded/+/t_mqtt/one">>, 1), + %% message from a different client, to avoid getting terminated by no-local + Max = 10, + Msgs = lists:seq(1, Max), + lists:foreach(fun(I) -> + {ok, _PacketId} = emqtt:publish(ConnPid, SendToTopic, integer_to_binary(I), ?QOS_1) + end, Msgs), + ?assertEqual(10, length(receive_messages(200))), - emqtt:subscribe(ConnPid, <<"receive/aws/t_mqtt/two">>, 1), - %% message from a different client, to avoid getting terminated by no-local - Max = 10, - Msgs = lists:seq(1, Max), - lists:foreach(fun(I) -> - {ok, _PacketId} = emqtt:publish(ConnPid, SendToTopic2, integer_to_binary(I), ?QOS_1) - end, Msgs), - ?assertEqual(10, length(receive_messages(200))), + emqtt:subscribe(ConnPid, <<"receive/aws/t_mqtt/two">>, 1), + %% message from a different client, to avoid getting terminated by no-local + Max = 10, + Msgs = lists:seq(1, Max), + lists:foreach(fun(I) -> + {ok, _PacketId} = emqtt:publish(ConnPid, SendToTopic2, integer_to_binary(I), ?QOS_1) + end, Msgs), + ?assertEqual(10, length(receive_messages(200))), - emqtt:disconnect(ConnPid) - after - ok = emqx_bridge_worker:stop(Pid) - end. + emqtt:disconnect(ConnPid), + ok = emqx_bridge_worker:stop(Pid). t_stub_normal(Config) when is_list(Config) -> - Cfg = #{forwards => [<<"t_stub_normal/#">>], - connect_module => emqx_bridge_stub_conn, - forward_mountpoint => <<"forwarded">>, - start_type => auto, + Name = "stub_normal", + Cfg = #{ + name => Name, + forwards => [<<"t_stub_normal/#">>], + forward_mountpoint => <<"forwarded">>, + start_type => auto, + config => #{ + conn_type => emqx_bridge_stub_conn, client_pid => self() - }, - {ok, Pid} = emqx_bridge_worker:start_link(?FUNCTION_NAME, Cfg), + } + }, + {ok, Pid} = emqx_bridge_mqtt_sup:create_bridge(Cfg), receive {Pid, emqx_bridge_stub_conn, ready} -> ok after 5000 -> error(timeout) end, - ClientId = <<"ClientId">>, - try - {ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]), - {ok, _} = emqtt:connect(ConnPid), - {ok, _PacketId} = emqtt:publish(ConnPid, <<"t_stub_normal/one">>, <<"hello">>, ?QOS_1), - receive - {stub_message, WorkerPid, BatchRef, _Batch} -> - WorkerPid ! {batch_ack, BatchRef}, - ok - after - 5000 -> - error(timeout) - end, - ?SNK_WAIT(inflight_drained), - ?SNK_WAIT(replayq_drained), - emqtt:disconnect(ConnPid) + {ok, ConnPid} = emqtt:start_link([{clientid, <<"ClientId">>}]), + {ok, _} = emqtt:connect(ConnPid), + {ok, _PacketId} = emqtt:publish(ConnPid, <<"t_stub_normal/one">>, <<"hello">>, ?QOS_1), + receive + {stub_message, WorkerPid, BatchRef, _Batch} -> + WorkerPid ! {batch_ack, BatchRef}, + ok after - ok = emqx_bridge_worker:stop(Pid) - end. + 5000 -> + error(timeout) + end, + ?SNK_WAIT(inflight_drained), + ?SNK_WAIT(replayq_drained), + emqtt:disconnect(ConnPid), + ok = emqx_bridge_worker:stop(Pid). -t_stub_overflow(Config) when is_list(Config) -> +t_stub_overflow(_Config) -> Topic = <<"t_stub_overflow/one">>, MaxInflight = 20, - Cfg = #{forwards => [Topic], - connect_module => emqx_bridge_stub_conn, - forward_mountpoint => <<"forwarded">>, - start_type => auto, - client_pid => self(), - max_inflight => MaxInflight - }, - {ok, Worker} = emqx_bridge_worker:start_link(?FUNCTION_NAME, Cfg), - ClientId = <<"ClientId">>, - try - {ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]), - {ok, _} = emqtt:connect(ConnPid), - lists:foreach( - fun(I) -> - Data = integer_to_binary(I), - _ = emqtt:publish(ConnPid, Topic, Data, ?QOS_1) - end, lists:seq(1, MaxInflight * 2)), - ?SNK_WAIT(inflight_full), - Acks = stub_receive(MaxInflight), - lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, Acks), - Acks2 = stub_receive(MaxInflight), - lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, Acks2), - ?SNK_WAIT(inflight_drained), - ?SNK_WAIT(replayq_drained), - emqtt:disconnect(ConnPid) - after - ok = emqx_bridge_worker:stop(Worker) - end. + Name = "stub_overflow", + Cfg = #{ + name => Name, + forwards => [<<"t_stub_overflow/one">>], + forward_mountpoint => <<"forwarded">>, + start_type => auto, + max_inflight => MaxInflight, + config => #{ + conn_type => emqx_bridge_stub_conn, + client_pid => self() + } + }, + {ok, Worker} = emqx_bridge_mqtt_sup:create_bridge(Cfg), + {ok, ConnPid} = emqtt:start_link([{clientid, <<"ClientId">>}]), + {ok, _} = emqtt:connect(ConnPid), + lists:foreach( + fun(I) -> + Data = integer_to_binary(I), + _ = emqtt:publish(ConnPid, Topic, Data, ?QOS_1) + end, lists:seq(1, MaxInflight * 2)), + ?SNK_WAIT(inflight_full), + Acks = stub_receive(MaxInflight), + lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, Acks), + Acks2 = stub_receive(MaxInflight), + lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, Acks2), + ?SNK_WAIT(inflight_drained), + ?SNK_WAIT(replayq_drained), + emqtt:disconnect(ConnPid), + ok = emqx_bridge_worker:stop(Worker). -t_stub_random_order(Config) when is_list(Config) -> +t_stub_random_order(_Config) -> Topic = <<"t_stub_random_order/a">>, MaxInflight = 10, - Cfg = #{forwards => [Topic], - connect_module => emqx_bridge_stub_conn, - forward_mountpoint => <<"forwarded">>, - start_type => auto, - client_pid => self(), - max_inflight => MaxInflight - }, - {ok, Worker} = emqx_bridge_worker:start_link(?FUNCTION_NAME, Cfg), + Name = "stub_random_order", + Cfg = #{ + name => Name, + forwards => [Topic], + forward_mountpoint => <<"forwarded">>, + start_type => auto, + max_inflight => MaxInflight, + config => #{ + conn_type => emqx_bridge_stub_conn, + client_pid => self() + } + }, + {ok, Worker} = emqx_bridge_mqtt_sup:create_bridge(Cfg), ClientId = <<"ClientId">>, - try - {ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]), - {ok, _} = emqtt:connect(ConnPid), - lists:foreach( - fun(I) -> - Data = integer_to_binary(I), - _ = emqtt:publish(ConnPid, Topic, Data, ?QOS_1) - end, lists:seq(1, MaxInflight)), - Acks = stub_receive(MaxInflight), - lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, - lists:reverse(Acks)), - ?SNK_WAIT(inflight_drained), - ?SNK_WAIT(replayq_drained), - emqtt:disconnect(ConnPid) - after - ok = emqx_bridge_worker:stop(Worker) - end. + {ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]), + {ok, _} = emqtt:connect(ConnPid), + lists:foreach( + fun(I) -> + Data = integer_to_binary(I), + _ = emqtt:publish(ConnPid, Topic, Data, ?QOS_1) + end, lists:seq(1, MaxInflight)), + Acks = stub_receive(MaxInflight), + lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, + lists:reverse(Acks)), + ?SNK_WAIT(inflight_drained), + ?SNK_WAIT(replayq_drained), + emqtt:disconnect(ConnPid), + ok = emqx_bridge_worker:stop(Worker). -t_stub_retry_inflight(Config) when is_list(Config) -> +t_stub_retry_inflight(_Config) -> Topic = <<"to_stub_retry_inflight/a">>, MaxInflight = 10, - Cfg = #{forwards => [Topic], - connect_module => emqx_bridge_stub_conn, - forward_mountpoint => <<"forwarded">>, - reconnect_delay_ms => 10, - start_type => auto, - client_pid => self(), - max_inflight => MaxInflight - }, - {ok, Worker} = emqx_bridge_worker:start_link(?FUNCTION_NAME, Cfg), + Name = "stub_retry_inflight", + Cfg = #{ + name => Name, + forwards => [Topic], + forward_mountpoint => <<"forwarded">>, + reconnect_interval => 10, + start_type => auto, + max_inflight => MaxInflight, + config => #{ + conn_type => emqx_bridge_stub_conn, + client_pid => self() + } + }, + {ok, Worker} = emqx_bridge_mqtt_sup:create_bridge(Cfg), ClientId = <<"ClientId2">>, - try - case ?block_until(#{?snk_kind := connected, inflight := 0}, 2000, 1000) of - {ok, #{inflight := 0}} -> ok; - Other -> ct:fail("~p", [Other]) - end, - {ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]), - {ok, _} = emqtt:connect(ConnPid), - lists:foreach( - fun(I) -> - Data = integer_to_binary(I), - _ = emqtt:publish(ConnPid, Topic, Data, ?QOS_1) - end, lists:seq(1, MaxInflight)), - %% receive acks but do not ack - Acks1 = stub_receive(MaxInflight), - ?assertEqual(MaxInflight, length(Acks1)), - %% simulate a disconnect - Worker ! {disconnected, self(), test}, - ?SNK_WAIT(disconnected), - case ?block_until(#{?snk_kind := connected, inflight := MaxInflight}, 2000, 20) of - {ok, _} -> ok; - Error -> ct:fail("~p", [Error]) - end, - %% expect worker to retry inflight, so to receive acks again - Acks2 = stub_receive(MaxInflight), - ?assertEqual(MaxInflight, length(Acks2)), - lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, - lists:reverse(Acks2)), - ?SNK_WAIT(inflight_drained), - ?SNK_WAIT(replayq_drained), - emqtt:disconnect(ConnPid) - after - ok = emqx_bridge_worker:stop(Worker) - end. + case ?block_until(#{?snk_kind := connected, inflight := 0}, 2000, 1000) of + {ok, #{inflight := 0}} -> ok; + Other -> ct:fail("~p", [Other]) + end, + {ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]), + {ok, _} = emqtt:connect(ConnPid), + lists:foreach( + fun(I) -> + Data = integer_to_binary(I), + _ = emqtt:publish(ConnPid, Topic, Data, ?QOS_1) + end, lists:seq(1, MaxInflight)), + %% receive acks but do not ack + Acks1 = stub_receive(MaxInflight), + ?assertEqual(MaxInflight, length(Acks1)), + %% simulate a disconnect + Worker ! {disconnected, self(), test}, + ?SNK_WAIT(disconnected), + case ?block_until(#{?snk_kind := connected, inflight := MaxInflight}, 2000, 20) of + {ok, _} -> ok; + Error -> ct:fail("~p", [Error]) + end, + %% expect worker to retry inflight, so to receive acks again + Acks2 = stub_receive(MaxInflight), + ?assertEqual(MaxInflight, length(Acks2)), + lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, + lists:reverse(Acks2)), + ?SNK_WAIT(inflight_drained), + ?SNK_WAIT(replayq_drained), + emqtt:disconnect(ConnPid), + ok = emqx_bridge_worker:stop(Worker). stub_receive(N) -> stub_receive(N, []). diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_tests.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_tests.erl index 69ff87356..ffa2e9ee5 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_tests.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_tests.erl @@ -15,7 +15,6 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_worker_tests). --behaviour(emqx_bridge_connect). -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx/include/emqx.hrl"). @@ -69,14 +68,14 @@ disturbance_test() -> emqx_bridge_worker:register_metrics(), Ref = make_ref(), TestPid = self(), - Config = make_config(Ref, TestPid, {ok, #{client_pid => TestPid}}), - {ok, Pid} = emqx_bridge_worker:start_link(?BRIDGE_NAME, Config), - ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)), + Config = make_config(Ref, TestPid, {ok, #{client_pid => TestPid}}), + {ok, Pid} = emqx_bridge_worker:start_link(Config#{name => disturbance}), + ?assertEqual(Pid, whereis(emqx_bridge_worker_disturbance)), ?WAIT({connection_start_attempt, Ref}, 1000), Pid ! {disconnected, TestPid, test}, ?WAIT({connection_start_attempt, Ref}, 1000), emqx_metrics:stop(), - ok = emqx_bridge_worker:stop(?BRIDGE_REG_NAME). + ok = emqx_bridge_worker:stop(Pid). % % %% buffer should continue taking in messages when disconnected % buffer_when_disconnected_test_() -> @@ -113,22 +112,24 @@ manual_start_stop_test() -> emqx_bridge_worker:register_metrics(), Ref = make_ref(), TestPid = self(), + BridgeName = manual_start_stop, Config0 = make_config(Ref, TestPid, {ok, #{client_pid => TestPid}}), Config = Config0#{start_type := manual}, - {ok, Pid} = emqx_bridge_worker:start_link(?BRIDGE_NAME, Config), + {ok, Pid} = emqx_bridge_worker:start_link(Config#{name => BridgeName}), %% call ensure_started again should yeld the same result - ok = emqx_bridge_worker:ensure_started(?BRIDGE_NAME), - ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)), - emqx_bridge_worker:ensure_stopped(unknown), - emqx_bridge_worker:ensure_stopped(Pid), - emqx_bridge_worker:ensure_stopped(?BRIDGE_REG_NAME), - emqx_metrics:stop(). + ok = emqx_bridge_worker:ensure_started(BridgeName), + emqx_bridge_worker:ensure_stopped(BridgeName), + emqx_metrics:stop(), + ok = emqx_bridge_worker:stop(Pid). make_config(Ref, TestPid, Result) -> - #{test_pid => TestPid, - test_ref => Ref, - connect_module => ?MODULE, - reconnect_delay_ms => 50, - connect_result => Result, - start_type => auto - }. + #{ + start_type => auto, + reconnect_interval => 50, + config => #{ + test_pid => TestPid, + test_ref => Ref, + conn_type => ?MODULE, + connect_result => Result + } + }. diff --git a/apps/emqx_coap/.gitignore b/apps/emqx_coap/.gitignore deleted file mode 100644 index 67eaa0145..000000000 --- a/apps/emqx_coap/.gitignore +++ /dev/null @@ -1,25 +0,0 @@ -deps/ -ebin/ -_rel/ -.erlang.mk/ -*.d -*.o -*.exe -data/ -*.iml -.idea/ -logs/ -*.beam -emqx_coap.d -intergration_test/emqx-rel/ -intergration_test/libcoap/ -intergration_test/case*.txt -.DS_Store -_build/ -rebar.lock -rebar3.crashdump -*.swp -erlang.mk -.rebar3/ -etc/emqx_coap.conf.rendered -.tags* diff --git a/apps/emqx_coap/README.md b/apps/emqx_coap/README.md deleted file mode 100644 index 1a9ee802c..000000000 --- a/apps/emqx_coap/README.md +++ /dev/null @@ -1,256 +0,0 @@ - -# emqx-coap - -emqx-coap is a CoAP Gateway for EMQ X Broker. It translates CoAP messages into MQTT messages and make it possible to communiate between CoAP clients and MQTT clients. - -### Client Usage Example -libcoap is an excellent coap library which has a simple client tool. It is recommended to use libcoap as a coap client. - -To compile libcoap, do following steps: - -``` -git clone http://github.com/obgm/libcoap -cd libcoap -./autogen.sh -./configure --enable-documentation=no --enable-tests=no -make -``` - -### Publish example: -``` -libcoap/examples/coap-client -m put -e 1234 "coap://127.0.0.1/mqtt/topic1?c=client1&u=tom&p=secret" -``` -- topic name is "topic1", NOT "/topic1" -- client id is client1 -- username is tom -- password is secret -- payload is a text string "1234" - -A mqtt message with topic="topic1", payload="1234" has been published. Any mqtt client or coap client, who has subscribed this topic could receive this message immediately. - -### Subscribe example: - -``` -libcoap/examples/coap-client -m get -s 10 "coap://127.0.0.1/mqtt/topic1?c=client1&u=tom&p=secret" -``` -- topic name is "topic1", NOT "/topic1" -- client id is client1 -- username is tom -- password is secret -- subscribe time is 10 seconds - -And you will get following result if any mqtt client or coap client sent message with text "1234567" to "topic1": - -``` -v:1 t:CON c:GET i:31ae {} [ ] -1234567v:1 t:CON c:GET i:31af {} [ Observe:1, Uri-Path:mqtt, Uri-Path:topic1, Uri-Query:c=client1, Uri-Query:u=tom, Uri-Query:p=secret ] -``` -The output message is not well formatted which hide "1234567" at the head of the 2nd line. - -### Configure - -#### Common - -File: etc/emqx_coap.conf - -```properties - -## The UDP port that CoAP is listening on. -## -## Value: Port -coap.port = 5683 - -## Interval for keepalive, specified in seconds. -## -## Value: Duration -## -s: seconds -## -m: minutes -## -h: hours -coap.keepalive = 120s - -## Whether to enable statistics for CoAP clients. -## -## Value: on | off -coap.enable_stats = off - -``` - -#### DTLS - -emqx_coap enable one-way authentication by default. - -If you want to disable it, comment these lines. - -File: etc/emqx_coap.conf - -```properties - -## The DTLS port that CoAP is listening on. -## -## Value: Port -coap.dtls.port = 5684 - -## Private key file for DTLS -## -## Value: File -coap.dtls.keyfile = {{ platform_etc_dir }}/certs/key.pem - -## Server certificate for DTLS. -## -## Value: File -coap.dtls.certfile = {{ platform_etc_dir }}/certs/cert.pem - -``` - -##### Enable two-way autentication - -For two-way autentication: - -```properties - -## A server only does x509-path validation in mode verify_peer, -## as it then sends a certificate request to the client (this -## message is not sent if the verify option is verify_none). -## You can then also want to specify option fail_if_no_peer_cert. -## More information at: http://erlang.org/doc/man/ssl.html -## -## Value: verify_peer | verify_none -## coap.dtls.verify = verify_peer - -## PEM-encoded CA certificates for DTLS -## -## Value: File -## coap.dtls.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem - -## Used together with {verify, verify_peer} by an SSL server. If set to true, -## the server fails if the client does not have a certificate to send, that is, -## sends an empty certificate. -## -## Value: true | false -## coap.dtls.fail_if_no_peer_cert = false - -``` - -### Load emqx-coap - -```bash -./bin/emqx_ctl plugins load emqx_coap -``` - -CoAP Client Observe Operation (subscribe topic) ------------------------------------------------ -To subscribe any topic, issue following command: - -``` - GET coap://localhost/mqtt/{topicname}?c={clientid}&u={username}&p={password} with OBSERVE=0 -``` - -- "mqtt" in the path is mandatory. -- replace {topicname}, {clientid}, {username} and {password} with your true values. -- {topicname} and {clientid} is mandatory. -- if clientid is absent, a "bad_request" will be returned. -- {topicname} in URI should be percent-encoded to prevent special characters, such as + and #. -- {username} and {password} are optional. -- if {username} or {password} is incorrect, the error code `unauthorized` will be returned. -- topic is subscribed with qos1. -- if the subscription failed due to ACL deny, the error code `forbidden` will be returned. - -CoAP Client Unobserve Operation (unsubscribe topic) ---------------------------------------------------- -To cancel observation, issue following command: - -``` - GET coap://localhost/mqtt/{topicname}?c={clientid}&u={username}&p={password} with OBSERVE=1 -``` - -- "mqtt" in the path is mandatory. -- replace {topicname}, {clientid}, {username} and {password} with your true values. -- {topicname} and {clientid} is mandatory. -- if clientid is absent, a "bad_request" will be returned. -- {topicname} in URI should be percent-encoded to prevent special characters, such as + and #. -- {username} and {password} are optional. -- if {username} or {password} is incorrect, the error code `unauthorized` will be returned. - -CoAP Client Notification Operation (subscribed Message) -------------------------------------------------------- -Server will issue an observe-notification as a subscribed message. - -- Its payload is exactly the mqtt payload. -- payload data type is "application/octet-stream". - -CoAP Client Publish Operation ------------------------------ -Issue a coap put command to publish messages. For example: - -``` - PUT coap://localhost/mqtt/{topicname}?c={clientid}&u={username}&p={password} -``` - -- "mqtt" in the path is mandatory. -- replace {topicname}, {clientid}, {username} and {password} with your true values. -- {topicname} and {clientid} is mandatory. -- if clientid is absent, a "bad_request" will be returned. -- {topicname} in URI should be percent-encoded to prevent special characters, such as + and #. -- {username} and {password} are optional. -- if {username} or {password} is incorrect, the error code `unauthorized` will be returned. -- payload could be any binary data. -- payload data type is "application/octet-stream". -- publish message will be sent with qos0. -- if the publishing failed due to ACL deny, the error code `forbidden` will be returned. - -CoAP Client Keep Alive ----------------------- -Device should issue a get command periodically, serve as a ping to keep mqtt session online. - -``` - GET coap://localhost/mqtt/{any_topicname}?c={clientid}&u={username}&p={password} -``` - -- "mqtt" in the path is mandatory. -- replace {any_topicname}, {clientid}, {username} and {password} with your true values. -- {any_topicname} is optional, and should be percent-encoded to prevent special characters. -- {clientid} is mandatory. If clientid is absent, a "bad_request" will be returned. -- {username} and {password} are optional. -- if {username} or {password} is incorrect, the error code `unauthorized` will be returned. -- coap client should do keepalive work periodically to keep mqtt session online, especially those devices in a NAT network. - - -CoAP Client NOTES ------------------ -emqx-coap gateway does not accept POST and DELETE requests. - -Topics in URI should be percent-encoded, but corresponding uri_path option has percent-encoding converted. Please refer to RFC 7252 section 6.4, "Decomposing URIs into Options": - -> Note that these rules completely resolve any percent-encoding. - -That implies coap client is responsible to convert any percert-encoding into true character while assembling coap packet. - - -ClientId, Username, Password and Topic --------------------------------------- -ClientId/username/password/topic in the coap URI are the concepts in mqtt. That is to say, emqx-coap is trying to fit coap message into mqtt system, by borrowing the client/username/password/topic from mqtt. - -The Auth/ACL/Hook features in mqtt also applies on coap stuff. For example: -- If username/password is not authorized, coap client will get an unauthorized error. -- If username or clientid is not allowed to published specific topic, coap message will be dropped in fact, although coap client will get an acknoledgement from emqx-coap. -- If a coap message is published, a 'message.publish' hook is able to capture this message as well. - -well-known locations --------------------- -Discovery always return "," - -For example -``` -libcoap/examples/coap-client -m get "coap://127.0.0.1/.well-known/core" -``` - -License -------- - -Apache License Version 2.0 - -Author ------- - -EMQ X Team. - diff --git a/apps/emqx_coap/TODO b/apps/emqx_coap/TODO deleted file mode 100644 index a0a1c2aaf..000000000 --- a/apps/emqx_coap/TODO +++ /dev/null @@ -1,13 +0,0 @@ -1. Remove the test/test_mqtt_broker and use emqx-ct-helpers -> Done! - - Enhance all test case - -2. Remove the mqtt adaptor -3. Remove the emqx_coap_pubsub_topics.erl - - -### Problems - -1. The coap-client of libcoap does not support Fragment DTLS handshake frame - * So, the connection will be established failed, if the 'Server Hello' frame is too big - * Why is the 'Server Hello' too big when enable the 'coap.dtls.cacertfile' option? -2. diff --git a/apps/emqx_coap/docs/rfc7049.pdf b/apps/emqx_coap/docs/rfc7049.pdf deleted file mode 100644 index a16db36ef..000000000 Binary files a/apps/emqx_coap/docs/rfc7049.pdf and /dev/null differ diff --git a/apps/emqx_coap/docs/rfc7228.pdf b/apps/emqx_coap/docs/rfc7228.pdf deleted file mode 100644 index c9dc1b59f..000000000 Binary files a/apps/emqx_coap/docs/rfc7228.pdf and /dev/null differ diff --git a/apps/emqx_coap/docs/rfc7252.pdf b/apps/emqx_coap/docs/rfc7252.pdf deleted file mode 100644 index 6876fad3e..000000000 Binary files a/apps/emqx_coap/docs/rfc7252.pdf and /dev/null differ diff --git a/apps/emqx_coap/include/emqx_coap.hrl b/apps/emqx_coap/include/emqx_coap.hrl deleted file mode 100644 index 963feca6b..000000000 --- a/apps/emqx_coap/include/emqx_coap.hrl +++ /dev/null @@ -1,20 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --define(APP, emqx_coap). - --record(coap_mqtt_auth, {clientid, username, password}). - diff --git a/apps/emqx_coap/intergration_test/Makefile b/apps/emqx_coap/intergration_test/Makefile deleted file mode 100644 index 12a2081dd..000000000 --- a/apps/emqx_coap/intergration_test/Makefile +++ /dev/null @@ -1,129 +0,0 @@ -.PHONY: clean, clean_result, start_broker stop_broker case1 case2 case3 - -RELX_CONF = emqx-rel/relx.config -LIBCOAP_GIT = libcoap/README.md - -all: clean_result $(RELX_CONF) $(LIBCOAP_GIT) start_broker clean_result case1 case2 case3 case4 stop_broker - @echo " " - @echo " test complete" - @echo " " - -clean_result: - -rm -f case*.txt - - -start_broker: - -rm -f emqx-rel/_rel/emqx/log/* - -emqx-rel/_rel/emqx/bin/emqx stop - sleep 1 - emqx-rel/_rel/emqx/bin/emqx start - sleep 1 - emqx-rel/_rel/emqx/bin/emqx_ctl plugins load emqx_coap - -stop_broker: - -emqx-rel/_rel/emqx/bin/emqx stop - -case1: - libcoap/examples/coap-client -m get -s 5 "coap://127.0.0.1/mqtt/topic1?c=client1&u=tom&p=secret" > case1_output.txt & - sleep 1 - libcoap/examples/coap-client -m put -e w123G45 "coap://127.0.0.1/mqtt/topic1?c=client2&u=mike&p=pw12" - sleep 6 - python check_result.py case1 case1_output.txt==w123G45 - -case2: - # subscribe to topic="x/y" - libcoap/examples/coap-client -m get -s 5 "coap://127.0.0.1/mqtt/x%2Fy?c=client3&u=tom&p=secret" > case2_output1.txt & - # subscribe to topic="+/z" - libcoap/examples/coap-client -m get -s 5 "coap://127.0.0.1/mqtt/%2B%2Fz?c=client4&u=mike&p=pw12" > case2_output2.txt & - sleep 1 - # publish to topic="x/y" - libcoap/examples/coap-client -m put -e big9wolf "coap://127.0.0.1/mqtt/x%2Fy?c=client5&u=sun&p=pw3" - # publish to topic="p/z" - libcoap/examples/coap-client -m put -e black2ant "coap://127.0.0.1/mqtt/p%2Fz?c=client5&u=sun&p=pw3" - sleep 6 - python check_result.py case2 case2_output1.txt==big9wolf case2_output1.txt!=black2ant case2_output2.txt!=big9wolf case2_output2.txt==black2ant - -case3: - libcoap/examples/coap-client -m get -T tk12 -s 5 "coap://127.0.0.1/mqtt/a%2Fb?c=client3&u=tom&p=secret" > case3_output1.txt & - libcoap/examples/coap-client -m get -T tk34 -s 5 "coap://127.0.0.1/mqtt/c%2Fd?c=client3&u=tom&p=secret" > case3_output2.txt & - sleep 1 - libcoap/examples/coap-client -m put -e big9wolf "coap://127.0.0.1/mqtt/c%2Fd?c=client5&u=sun&p=pw3" - libcoap/examples/coap-client -m put -e black2ant "coap://127.0.0.1/mqtt/a%2Fb?c=client5&u=sun&p=pw3" - sleep 6 - python check_result.py case3 case3_output1.txt==black2ant case3_output2.txt==big9wolf case3_output2.txt!=black2ant - - - -case4: - # reload emqx_coap, does it work as expected? - sleep 1 - emqx-rel/_rel/emqx/bin/emqx_ctl plugins unload emqx_coap - sleep 1 - emqx-rel/_rel/emqx/bin/emqx_ctl plugins load emqx_coap - sleep 1 - libcoap/examples/coap-client -m get -s 5 "coap://127.0.0.1/mqtt/topic1?c=client1&u=tom&p=secret" > case4_output.txt & - sleep 1 - libcoap/examples/coap-client -m put -e w6J3G45 "coap://127.0.0.1/mqtt/topic1?c=client2&u=mike&p=pw12" - sleep 6 - python check_result.py case4 case4_output.txt==w6J3G45 - - - - -$(RELX_CONF): - git clone https://github.com/emqx/emqx-rel.git - git clone https://github.com/emqx/emq-coap.git - @echo "update emq-coap with this development code" - mv emq-coap emqx_coap - -rm -rf emqx_coap/etc - -rm -rf emqx_coap/include - -rm -rf emqx_coap/priv - -rm -rf emqx_coap/src - -rm -rf emqx_coap/Makefile - cp -rf ../etc emqx_coap/ - cp -rf ../include emqx_coap/ - cp -rf ../priv emqx_coap/ - cp -rf ../src emqx_coap/ - cp -rf ../Makefile emqx_coap/Makefile - -mkdir emqx-rel/deps - mv emqx_coap emqx-rel/deps/ - @echo "start building ..." - make -C emqx-rel -f Makefile - - -coap: $(LIBCOAP_GIT) - @echo "make coap" - -$(LIBCOAP_GIT): - git clone -b v4.1.2 http://github.com/obgm/libcoap - cd libcoap && ./autogen.sh && ./configure --enable-documentation=no --enable-tests=no - make -C libcoap -f Makefile - -r: rebuild_emq - # r short for rebuild_emq - @echo " rebuild complete " - -rebuild_emq: - -emqx-rel/_rel/emqx/bin/emqx stop - -rm -rf emqx-rel/deps/emqx_coap/etc - -rm -rf emqx-rel/deps/emqx_coap/include - -rm -rf emqx-rel/deps/emqx_coap/priv - -rm -rf emqx-rel/deps/emqx_coap/src - -rm -rf emqx-rel/deps/emqx_coap/Makefile - cp -rf ../etc emqx-rel/deps/emqx_coap/ - cp -rf ../include emqx-rel/deps/emqx_coap/ - cp -rf ../priv emqx-rel/deps/emqx_coap/ - cp -rf ../src emqx-rel/deps/emqx_coap/ - cp -rf ../Makefile emqx-rel/deps/emqx_coap/Makefile - make -C emqx-rel -f Makefile - -clean: clean_result - -rm -f client/*.exe - -rm -f client/*.o - -rm -rf emqx-rel - -rm -rf libcoap - -lazy: clean_result start_broker case2 stop_broker - # custom your command here - @echo "you are so lazy" - diff --git a/apps/emqx_coap/intergration_test/README.md b/apps/emqx_coap/intergration_test/README.md deleted file mode 100644 index eb3507923..000000000 --- a/apps/emqx_coap/intergration_test/README.md +++ /dev/null @@ -1,8 +0,0 @@ -Integration test for emq-coap -====== - -execute following command -``` -make -``` - diff --git a/apps/emqx_coap/intergration_test/check_result.py b/apps/emqx_coap/intergration_test/check_result.py deleted file mode 100644 index f9baaefae..000000000 --- a/apps/emqx_coap/intergration_test/check_result.py +++ /dev/null @@ -1,52 +0,0 @@ -import sys - - -def have_string(filename, text): - data = open(filename, "rb").read() - if data.find(text) > 0: - return True - else: - return False - - -def mark(case_number, result, description): - if result: - f = open(case_number+"_PASS.txt", "wb") - f.close() - print("\n\n"+case_number+" PASS\n\n") - else: - f = open(case_number+"_FAIL.txt", "wb") - f.write(description) - f.close() - print("\n\n"+case_number+" FAIL\n\n") - -def parse_condition(condition): - if condition.find("==") > 0: - r = condition.split("==") - return r[0], r[1], True - elif condition.find("!=") > 0: - r = condition.split("!=") - return r[0], r[1], False - else: - print("\ncondition syntax error\n\n\n") - sys.exit("condition syntax error") - - -def main(): - case_number = sys.argv[1] - description = "" - conclustion = True - for condition in sys.argv[2:]: - filename, text, result = parse_condition(condition) - if have_string(filename, text) == result: - pass - else: - conclustion = False - description = description + "\n" + condition + " failed\n" - - mark(case_number, conclustion, description) - - -if __name__ == "__main__": - main() - diff --git a/apps/emqx_coap/rebar.config b/apps/emqx_coap/rebar.config deleted file mode 100644 index 0f8759b8a..000000000 --- a/apps/emqx_coap/rebar.config +++ /dev/null @@ -1,4 +0,0 @@ -{deps, - [ - {gen_coap, {git, "https://github.com/emqx/gen_coap", {tag, "v0.3.2"}}} - ]}. diff --git a/apps/emqx_coap/src/emqx_coap.app.src b/apps/emqx_coap/src/emqx_coap.app.src deleted file mode 100644 index 2b5fcbb6a..000000000 --- a/apps/emqx_coap/src/emqx_coap.app.src +++ /dev/null @@ -1,14 +0,0 @@ -{application, emqx_coap, - [{description, "EMQ X CoAP Gateway"}, - {vsn, "4.3.0"}, % strict semver, bump manually! - {modules, []}, - {registered, []}, - {applications, [kernel,stdlib,gen_coap]}, - {mod, {emqx_coap_app, []}}, - {env, []}, - {licenses, ["Apache-2.0"]}, - {maintainers, ["EMQ X Team "]}, - {links, [{"Homepage", "https://emqx.io/"}, - {"Github", "https://github.com/emqx/emqx-coap"} - ]} - ]}. diff --git a/apps/emqx_coap/src/emqx_coap_app.erl b/apps/emqx_coap/src/emqx_coap_app.erl deleted file mode 100644 index 16f74faf7..000000000 --- a/apps/emqx_coap/src/emqx_coap_app.erl +++ /dev/null @@ -1,40 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_coap_app). - --behaviour(application). - --emqx_plugin(protocol). - --include("emqx_coap.hrl"). - --export([ start/2 - , stop/1 - ]). - -start(_Type, _Args) -> - {ok, Sup} = emqx_coap_sup:start_link(), - coap_server_registry:add_handler([<<"mqtt">>], emqx_coap_resource, undefined), - coap_server_registry:add_handler([<<"ps">>], emqx_coap_pubsub_resource, undefined), - _ = emqx_coap_pubsub_topics:start_link(), - emqx_coap_server:start(application:get_all_env(?APP)), - {ok,Sup}. - -stop(_State) -> - coap_server_registry:remove_handler([<<"mqtt">>], emqx_coap_resource, undefined), - coap_server_registry:remove_handler([<<"ps">>], emqx_coap_pubsub_resource, undefined), - emqx_coap_server:stop(application:get_all_env(?APP)). diff --git a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl deleted file mode 100644 index d465f9ca3..000000000 --- a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl +++ /dev/null @@ -1,387 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_coap_mqtt_adapter). - --behaviour(gen_server). - --include("emqx_coap.hrl"). - --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). - --logger_header("[CoAP-Adpter]"). - -%% API. --export([ subscribe/2 - , unsubscribe/2 - , publish/3 - ]). - --export([ client_pid/4 - , stop/1 - ]). - --export([ call/2 - , call/3 - ]). - -%% gen_server. --export([ init/1 - , handle_call/3 - , handle_cast/2 - , handle_info/2 - , terminate/2 - , code_change/3 - ]). - --record(state, {peername, clientid, username, password, sub_topics = [], connected_at}). - --define(ALIVE_INTERVAL, 20000). - --define(CONN_STATS, [recv_pkt, recv_msg, send_pkt, send_msg]). - --define(SUBOPTS, #{rh => 0, rap => 0, nl => 0, qos => ?QOS_0, is_new => false}). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - -client_pid(undefined, _Username, _Password, _Channel) -> - {error, bad_request}; -client_pid(ClientId, Username, Password, Channel) -> - % check authority - case start(ClientId, Username, Password, Channel) of - {ok, Pid1} -> {ok, Pid1}; - {error, {already_started, Pid2}} -> {ok, Pid2}; - {error, auth_failure} -> {error, auth_failure}; - Other -> {error, Other} - end. - -start(ClientId, Username, Password, Channel) -> - % DO NOT use start_link, since multiple coap_reponsder may have relation with one mqtt adapter, - % one coap_responder crashes should not make mqtt adapter crash too - % And coap_responder is not a system process - % it is dangerous to link mqtt adapter to coap_responder - gen_server:start({via, emqx_coap_registry, {ClientId, Username, Password}}, - ?MODULE, {ClientId, Username, Password, Channel}, []). - -stop(Pid) -> - gen_server:stop(Pid). - -subscribe(Pid, Topic) -> - gen_server:call(Pid, {subscribe, Topic, self()}). - -unsubscribe(Pid, Topic) -> - gen_server:call(Pid, {unsubscribe, Topic, self()}). - -publish(Pid, Topic, Payload) -> - gen_server:call(Pid, {publish, Topic, Payload}). - -%% For emqx_management plugin -call(Pid, Msg) -> - call(Pid, Msg, infinity). - -call(Pid, Msg, _) -> - Pid ! Msg, ok. - -%%-------------------------------------------------------------------- -%% gen_server Callbacks -%%-------------------------------------------------------------------- - -init({ClientId, Username, Password, Channel}) -> - ?LOG(debug, "try to start adapter ClientId=~p, Username=~p, Password=~p, " - "Channel=~0p", [ClientId, Username, Password, Channel]), - State0 = #state{peername = Channel, - clientid = ClientId, - username = Username, - password = Password}, - _ = run_hooks('client.connect', [conninfo(State0)], undefined), - case emqx_access_control:authenticate(clientinfo(State0)) of - {ok, _AuthResult} -> - ok = emqx_cm:discard_session(ClientId), - - _ = run_hooks('client.connack', [conninfo(State0), success], undefined), - - State = State0#state{connected_at = erlang:system_time(millisecond)}, - - run_hooks('client.connected', [clientinfo(State), conninfo(State)]), - - Self = self(), - erlang:send_after(?ALIVE_INTERVAL, Self, check_alive), - _ = emqx_cm_locker:trans(ClientId, fun(_) -> - emqx_cm:register_channel(ClientId, Self, conninfo(State)) - end), - emqx_cm:insert_channel_info(ClientId, info(State), stats(State)), - {ok, State}; - {error, Reason} -> - ?LOG(debug, "authentication faild: ~p", [Reason]), - _ = run_hooks('client.connack', [conninfo(State0), not_authorized], undefined), - {stop, {shutdown, Reason}} - end. - -handle_call({subscribe, Topic, CoapPid}, _From, State=#state{sub_topics = TopicList}) -> - NewTopics = proplists:delete(Topic, TopicList), - IsWild = emqx_topic:wildcard(Topic), - {reply, chann_subscribe(Topic, State), State#state{sub_topics = - [{Topic, {IsWild, CoapPid}}|NewTopics]}, hibernate}; - -handle_call({unsubscribe, Topic, _CoapPid}, _From, State=#state{sub_topics = TopicList}) -> - NewTopics = proplists:delete(Topic, TopicList), - chann_unsubscribe(Topic, State), - {reply, ok, State#state{sub_topics = NewTopics}, hibernate}; - -handle_call({publish, Topic, Payload}, _From, State) -> - {reply, chann_publish(Topic, Payload, State), State}; - -handle_call(info, _From, State) -> - {reply, info(State), State}; - -handle_call(stats, _From, State) -> - {reply, stats(State), State, hibernate}; - -handle_call(kick, _From, State) -> - {stop, {shutdown, kick}, ok, State}; - -handle_call({set_rate_limit, _Rl}, _From, State) -> - ?LOG(error, "set_rate_limit is not support", []), - {reply, ok, State}; - -handle_call(get_rate_limit, _From, State) -> - ?LOG(error, "get_rate_limit is not support", []), - {reply, ok, State}; - -handle_call(Request, _From, State) -> - ?LOG(error, "adapter unexpected call ~p", [Request]), - {reply, ignored, State, hibernate}. - -handle_cast(Msg, State) -> - ?LOG(error, "broker_api unexpected cast ~p", [Msg]), - {noreply, State, hibernate}. - -handle_info({deliver, _Topic, #message{topic = Topic, payload = Payload}}, - State = #state{sub_topics = Subscribers}) -> - deliver([{Topic, Payload}], Subscribers), - {noreply, State, hibernate}; - -handle_info(check_alive, State = #state{sub_topics = []}) -> - {stop, {shutdown, check_alive}, State}; -handle_info(check_alive, State) -> - erlang:send_after(?ALIVE_INTERVAL, self(), check_alive), - {noreply, State, hibernate}; - -handle_info({shutdown, Error}, State) -> - {stop, {shutdown, Error}, State}; - -handle_info({shutdown, conflict, {ClientId, NewPid}}, State) -> - ?LOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid]), - {stop, {shutdown, conflict}, State}; - -handle_info(discard, State) -> - ?LOG(warning, "the connection is discarded. " ++ - "possibly there is another client with the same clientid", []), - {stop, {shutdown, discarded}, State}; - -handle_info(kick, State) -> - ?LOG(info, "Kicked", []), - {stop, {shutdown, kick}, State}; - -handle_info(Info, State) -> - ?LOG(error, "adapter unexpected info ~p", [Info]), - {noreply, State, hibernate}. - -terminate(Reason, State = #state{clientid = ClientId, sub_topics = SubTopics}) -> - ?LOG(debug, "unsubscribe ~p while exiting for ~p", [SubTopics, Reason]), - [chann_unsubscribe(Topic, State) || {Topic, _} <- SubTopics], - emqx_cm:unregister_channel(ClientId), - - ConnInfo0 = conninfo(State), - ConnInfo = ConnInfo0#{disconnected_at => erlang:system_time(millisecond)}, - run_hooks('client.disconnected', [clientinfo(State), Reason, ConnInfo]). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Channel adapter functions - -chann_subscribe(Topic, State = #state{clientid = ClientId}) -> - ?LOG(debug, "subscribe Topic=~p", [Topic]), - case emqx_access_control:check_acl(clientinfo(State), subscribe, Topic) of - allow -> - emqx_broker:subscribe(Topic, ClientId, ?SUBOPTS), - emqx_hooks:run('session.subscribed', [clientinfo(State), Topic, ?SUBOPTS]), - ok; - deny -> - ?LOG(warning, "subscribe to ~p by clientid ~p failed due to acl check.", - [Topic, ClientId]), - {error, forbidden} - end. - -chann_unsubscribe(Topic, State) -> - ?LOG(debug, "unsubscribe Topic=~p", [Topic]), - Opts = #{rh => 0, rap => 0, nl => 0, qos => 0}, - emqx_broker:unsubscribe(Topic), - emqx_hooks:run('session.unsubscribed', [clientinfo(State), Topic, Opts]). - -chann_publish(Topic, Payload, State = #state{clientid = ClientId}) -> - ?LOG(debug, "publish Topic=~p, Payload=~p", [Topic, Payload]), - case emqx_access_control:check_acl(clientinfo(State), publish, Topic) of - allow -> - _ = emqx_broker:publish( - emqx_message:set_flag(retain, false, - emqx_message:make(ClientId, ?QOS_0, Topic, Payload))), - ok; - deny -> - ?LOG(warning, "publish to ~p by clientid ~p failed due to acl check.", - [Topic, ClientId]), - {error, forbidden} - end. - - -%%-------------------------------------------------------------------- -%% Deliver - -deliver([], _) -> ok; -deliver([Pub | More], Subscribers) -> - ok = do_deliver(Pub, Subscribers), - deliver(More, Subscribers). - -do_deliver({Topic, Payload}, Subscribers) -> - %% handle PUBLISH packet from broker - ?LOG(debug, "deliver message from broker Topic=~p, Payload=~p", [Topic, Payload]), - deliver_to_coap(Topic, Payload, Subscribers), - ok. - -deliver_to_coap(_TopicName, _Payload, []) -> - ok; -deliver_to_coap(TopicName, Payload, [{TopicFilter, {IsWild, CoapPid}}|T]) -> - Matched = case IsWild of - true -> emqx_topic:match(TopicName, TopicFilter); - false -> TopicName =:= TopicFilter - end, - %?LOG(debug, "deliver_to_coap Matched=~p, CoapPid=~p, TopicName=~p, Payload=~p, T=~p", - % [Matched, CoapPid, TopicName, Payload, T]), - Matched andalso (CoapPid ! {dispatch, TopicName, Payload}), - deliver_to_coap(TopicName, Payload, T). - -%%-------------------------------------------------------------------- -%% Helper funcs - --compile({inline, [run_hooks/2, run_hooks/3]}). -run_hooks(Name, Args) -> - ok = emqx_metrics:inc(Name), emqx_hooks:run(Name, Args). - -run_hooks(Name, Args, Acc) -> - ok = emqx_metrics:inc(Name), emqx_hooks:run_fold(Name, Args, Acc). - -%%-------------------------------------------------------------------- -%% Info & Stats - -info(State) -> - ChannInfo = chann_info(State), - ChannInfo#{sockinfo => sockinfo(State)}. - -%% copies from emqx_connection:info/1 -sockinfo(#state{peername = Peername}) -> - #{socktype => udp, - peername => Peername, - sockname => {{127, 0, 0, 1}, 5683}, %% FIXME: Sock? - sockstate => running, - active_n => 1 - }. - -%% copies from emqx_channel:info/1 -chann_info(State) -> - #{conninfo => conninfo(State), - conn_state => connected, - clientinfo => clientinfo(State), - session => maps:from_list(session_info(State)), - will_msg => undefined - }. - -conninfo(#state{peername = Peername, - clientid = ClientId, - connected_at = ConnectedAt}) -> - #{socktype => udp, - sockname => {{127, 0, 0, 1}, 5683}, - peername => Peername, - peercert => nossl, %% TODO: dtls - conn_mod => ?MODULE, - proto_name => <<"CoAP">>, - proto_ver => 1, - clean_start => true, - clientid => ClientId, - username => undefined, - conn_props => undefined, - connected => true, - connected_at => ConnectedAt, - keepalive => 0, - receive_maximum => 0, - expiry_interval => 0 - }. - -%% copies from emqx_session:info/1 -session_info(#state{sub_topics = SubTopics, connected_at = ConnectedAt}) -> - Subs = lists:foldl( - fun({Topic, _}, Acc) -> - Acc#{Topic => ?SUBOPTS} - end, #{}, SubTopics), - [{subscriptions, Subs}, - {upgrade_qos, false}, - {retry_interval, 0}, - {await_rel_timeout, 0}, - {created_at, ConnectedAt} - ]. - -%% The stats keys copied from emqx_connection:stats/1 -stats(#state{sub_topics = SubTopics}) -> - SockStats = [{recv_oct, 0}, {recv_cnt, 0}, {send_oct, 0}, {send_cnt, 0}, {send_pend, 0}], - ConnStats = emqx_pd:get_counters(?CONN_STATS), - ChanStats = [{subscriptions_cnt, length(SubTopics)}, - {subscriptions_max, length(SubTopics)}, - {inflight_cnt, 0}, - {inflight_max, 0}, - {mqueue_len, 0}, - {mqueue_max, 0}, - {mqueue_dropped, 0}, - {next_pkt_id, 0}, - {awaiting_rel_cnt, 0}, - {awaiting_rel_max, 0} - ], - ProcStats = emqx_misc:proc_stats(), - lists:append([SockStats, ConnStats, ChanStats, ProcStats]). - -clientinfo(#state{peername = {PeerHost, _}, - clientid = ClientId, - username = Username, - password = Password}) -> - #{zone => undefined, - protocol => coap, - peerhost => PeerHost, - sockport => 5683, %% FIXME: - clientid => ClientId, - username => Username, - password => Password, - peercert => nossl, - is_bridge => false, - is_superuser => false, - mountpoint => undefined, - ws_cookie => undefined - }. - diff --git a/apps/emqx_coap/src/emqx_coap_pubsub_resource.erl b/apps/emqx_coap/src/emqx_coap_pubsub_resource.erl deleted file mode 100644 index d87a26173..000000000 --- a/apps/emqx_coap/src/emqx_coap_pubsub_resource.erl +++ /dev/null @@ -1,322 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_coap_pubsub_resource). - --behaviour(coap_resource). - --include("emqx_coap.hrl"). --include_lib("gen_coap/include/coap.hrl"). --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("emqx/include/logger.hrl"). - --logger_header("[CoAP-PS-RES]"). - --export([ coap_discover/2 - , coap_get/5 - , coap_post/4 - , coap_put/4 - , coap_delete/3 - , coap_observe/5 - , coap_unobserve/1 - , handle_info/2 - , coap_ack/2 - ]). - --ifdef(TEST). --export([topic/1]). --endif. - --define(PS_PREFIX, [<<"ps">>]). - -%%-------------------------------------------------------------------- -%% Resource Callbacks -%%-------------------------------------------------------------------- -coap_discover(_Prefix, _Args) -> - [{absolute, [<<"ps">>], []}]. - -coap_get(ChId, ?PS_PREFIX, TopicPath, Query, Content=#coap_content{format = Format}) when TopicPath =/= [] -> - Topic = topic(TopicPath), - ?LOG(debug, "coap_get() Topic=~p, Query=~p~n", [Topic, Query]), - #coap_mqtt_auth{clientid = Clientid, username = Usr, password = Passwd} = get_auth(Query), - case emqx_coap_mqtt_adapter:client_pid(Clientid, Usr, Passwd, ChId) of - {ok, Pid} -> - put(mqtt_client_pid, Pid), - case Format of - <<"application/link-format">> -> - Content; - _Other -> - %% READ the topic info - read_last_publish_message(emqx_topic:wildcard(Topic), Topic, Content) - end; - {error, auth_failure} -> - put(mqtt_client_pid, undefined), - {error, unauthorized}; - {error, bad_request} -> - put(mqtt_client_pid, undefined), - {error, bad_request}; - {error, _Other} -> - put(mqtt_client_pid, undefined), - {error, internal_server_error} - end; -coap_get(ChId, Prefix, TopicPath, Query, _Content) -> - ?LOG(error, "ignore bad get request ChId=~p, Prefix=~p, TopicPath=~p, Query=~p", [ChId, Prefix, TopicPath, Query]), - {error, bad_request}. - -coap_post(_ChId, ?PS_PREFIX, TopicPath, #coap_content{format = Format, payload = Payload, max_age = MaxAge}) when TopicPath =/= [] -> - Topic = topic(TopicPath), - ?LOG(debug, "coap_post() Topic=~p, MaxAge=~p, Format=~p~n", [Topic, MaxAge, Format]), - case Format of - %% We treat ct of "application/link-format" as CREATE message - <<"application/link-format">> -> - handle_received_create(Topic, MaxAge, Payload); - %% We treat ct of other values as PUBLISH message - Other -> - ?LOG(debug, "coap_post() receive payload format=~p, will process as PUBLISH~n", [Format]), - handle_received_publish(Topic, MaxAge, Other, Payload) - end; - -coap_post(_ChId, _Prefix, _TopicPath, _Content) -> - {error, method_not_allowed}. - -coap_put(_ChId, ?PS_PREFIX, TopicPath, #coap_content{max_age = MaxAge, format = Format, payload = Payload}) when TopicPath =/= [] -> - Topic = topic(TopicPath), - ?LOG(debug, "put message, Topic=~p, Payload=~p~n", [Topic, Payload]), - handle_received_publish(Topic, MaxAge, Format, Payload); - -coap_put(_ChId, Prefix, TopicPath, Content) -> - ?LOG(error, "put has error, Prefix=~p, TopicPath=~p, Content=~p", [Prefix, TopicPath, Content]), - {error, bad_request}. - -coap_delete(_ChId, ?PS_PREFIX, TopicPath) -> - delete_topic_info(topic(TopicPath)); - -coap_delete(_ChId, _Prefix, _TopicPath) -> - {error, method_not_allowed}. - -coap_observe(ChId, ?PS_PREFIX, TopicPath, Ack, Content) when TopicPath =/= [] -> - Topic = topic(TopicPath), - ?LOG(debug, "observe Topic=~p, Ack=~p，Content=~p", [Topic, Ack, Content]), - Pid = get(mqtt_client_pid), - case emqx_coap_mqtt_adapter:subscribe(Pid, Topic) of - ok -> - Code = case emqx_coap_pubsub_topics:is_topic_timeout(Topic) of - true -> nocontent; - false-> content - end, - {ok, {state, ChId, ?PS_PREFIX, [Topic]}, Code, Content}; - {error, Code} -> - {error, Code} - end; - -coap_observe(ChId, Prefix, TopicPath, Ack, _Content) -> - ?LOG(error, "unknown observe request ChId=~p, Prefix=~p, TopicPath=~p, Ack=~p", [ChId, Prefix, TopicPath, Ack]), - {error, bad_request}. - -coap_unobserve({state, _ChId, ?PS_PREFIX, TopicPath}) when TopicPath =/= [] -> - Topic = topic(TopicPath), - ?LOG(debug, "unobserve ~p", [Topic]), - Pid = get(mqtt_client_pid), - emqx_coap_mqtt_adapter:unsubscribe(Pid, Topic), - ok; -coap_unobserve({state, ChId, Prefix, TopicPath}) -> - ?LOG(error, "ignore unknown unobserve request ChId=~p, Prefix=~p, TopicPath=~p", [ChId, Prefix, TopicPath]), - ok. - -handle_info({dispatch, Topic, Payload}, State) -> - ?LOG(debug, "dispatch Topic=~p, Payload=~p", [Topic, Payload]), - {ok, Ret} = emqx_coap_pubsub_topics:reset_topic_info(Topic, Payload), - ?LOG(debug, "Updated publish info of topic=~p, the Ret is ~p", [Topic, Ret]), - {notify, [], #coap_content{format = <<"application/octet-stream">>, payload = Payload}, State}; -handle_info(Message, State) -> - ?LOG(error, "Unknown Message ~p", [Message]), - {noreply, State}. - -coap_ack(_Ref, State) -> {ok, State}. - - -%%-------------------------------------------------------------------- -%% Internal Functions -%%-------------------------------------------------------------------- -get_auth(Query) -> - get_auth(Query, #coap_mqtt_auth{}). - -get_auth([], Auth=#coap_mqtt_auth{}) -> - Auth; -get_auth([<<$c, $=, Rest/binary>>|T], Auth=#coap_mqtt_auth{}) -> - get_auth(T, Auth#coap_mqtt_auth{clientid = Rest}); -get_auth([<<$u, $=, Rest/binary>>|T], Auth=#coap_mqtt_auth{}) -> - get_auth(T, Auth#coap_mqtt_auth{username = Rest}); -get_auth([<<$p, $=, Rest/binary>>|T], Auth=#coap_mqtt_auth{}) -> - get_auth(T, Auth#coap_mqtt_auth{password = Rest}); -get_auth([Param|T], Auth=#coap_mqtt_auth{}) -> - ?LOG(error, "ignore unknown parameter ~p", [Param]), - get_auth(T, Auth). - -add_topic_info(publish, Topic, MaxAge, Format, Payload) when is_binary(Topic), Topic =/= <<>> -> - case emqx_coap_pubsub_topics:lookup_topic_info(Topic) of - [{_, StoredMaxAge, StoredCT, _, _}] -> - ?LOG(debug, "publish topic=~p already exists, need reset the topic info", [Topic]), - %% check whether the ct value stored matches the ct option in this POST message - case Format =:= StoredCT of - true -> - {ok, Ret} = - case StoredMaxAge =:= MaxAge of - true -> - emqx_coap_pubsub_topics:reset_topic_info(Topic, Payload); - false -> - emqx_coap_pubsub_topics:reset_topic_info(Topic, MaxAge, Payload) - end, - {changed, Ret}; - false -> - ?LOG(debug, "ct values of topic=~p do not match, stored ct=~p, new ct=~p, ignore the PUBLISH", [Topic, StoredCT, Format]), - {changed, false} - end; - [] -> - ?LOG(debug, "publish topic=~p will be created", [Topic]), - {ok, Ret} = emqx_coap_pubsub_topics:add_topic_info(Topic, MaxAge, Format, Payload), - {created, Ret} - end; - -add_topic_info(create, Topic, MaxAge, Format, _Payload) when is_binary(Topic), Topic =/= <<>> -> - case emqx_coap_pubsub_topics:is_topic_existed(Topic) of - true -> - %% Whether we should support CREATE to an existed topic is TBD!! - ?LOG(debug, "create topic=~p already exists, need reset the topic info", [Topic]), - {ok, Ret} = emqx_coap_pubsub_topics:reset_topic_info(Topic, MaxAge, Format, <<>>); - false -> - ?LOG(debug, "create topic=~p will be created", [Topic]), - {ok, Ret} = emqx_coap_pubsub_topics:add_topic_info(Topic, MaxAge, Format, <<>>) - end, - {created, Ret}; - -add_topic_info(_, Topic, _MaxAge, _Format, _Payload) -> - ?LOG(debug, "create topic=~p info failed", [Topic]), - {badarg, false}. - -concatenate_location_path(List = [TopicPart1, TopicPart2, TopicPart3]) when is_binary(TopicPart1), is_binary(TopicPart2), is_binary(TopicPart3) -> - list_to_binary(lists:foldl( fun (Element, AccIn) when Element =/= <<>> -> - AccIn ++ "/" ++ binary_to_list(Element); - (_Element, AccIn) -> - AccIn - end, [], List)). - -format_string_to_int(<<"application/octet-stream">>) -> - <<"42">>; -format_string_to_int(<<"application/exi">>) -> - <<"47">>; -format_string_to_int(<<"application/json">>) -> - <<"50">>. - -handle_received_publish(Topic, MaxAge, Format, Payload) -> - case add_topic_info(publish, Topic, MaxAge, format_string_to_int(Format), Payload) of - {Ret, true} -> - Pid = get(mqtt_client_pid), - case emqx_coap_mqtt_adapter:publish(Pid, topic(Topic), Payload) of - ok -> - {ok, Ret, case Ret of - changed -> #coap_content{}; - created -> - #coap_content{location_path = [ - concatenate_location_path([<<"ps">>, Topic, <<>>])]} - end}; - {error, Code} -> - {error, Code} - end; - {_, false} -> - ?LOG(debug, "add_topic_info failed, will return bad_request", []), - {error, bad_request} - end. - -handle_received_create(TopicPrefix, MaxAge, Payload) -> - case core_link:decode(Payload) of - [{rootless, [Topic], [{ct, CT}]}] when is_binary(Topic), Topic =/= <<>> -> - TrueTopic = emqx_http_lib:uri_decode(Topic), - ?LOG(debug, "decoded link-format payload, the Topic=~p, CT=~p~n", [TrueTopic, CT]), - LocPath = concatenate_location_path([<<"ps">>, TopicPrefix, TrueTopic]), - FullTopic = binary:part(LocPath, 4, byte_size(LocPath)-4), - ?LOG(debug, "the location path is ~p, the full topic is ~p~n", [LocPath, FullTopic]), - case add_topic_info(create, FullTopic, MaxAge, CT, <<>>) of - {_, true} -> - ?LOG(debug, "create topic info successfully, will return LocPath=~p", [LocPath]), - {ok, created, #coap_content{location_path = [LocPath]}}; - {_, false} -> - ?LOG(debug, "create topic info failed, will return bad_request", []), - {error, bad_request} - end; - Other -> - ?LOG(debug, "post with bad payload of link-format ~p, will return bad_request", [Other]), - {error, bad_request} - end. - -%% When topic is timeout, server should return nocontent here, -%% but gen_coap only receive return value of #coap_content from coap_get, so temporarily we can't give the Code 2.07 {ok, nocontent} out.TBC!!! -return_resource(Topic, Payload, MaxAge, TimeStamp, Content) -> - TimeElapsed = trunc((erlang:system_time(millisecond) - TimeStamp) / 1000), - case TimeElapsed < MaxAge of - true -> - LeftTime = (MaxAge - TimeElapsed), - ?LOG(debug, "topic=~p has max age left time is ~p", [Topic, LeftTime]), - Content#coap_content{max_age = LeftTime, payload = Payload}; - false -> - ?LOG(debug, "topic=~p has been timeout, will return empty content", [Topic]), - #coap_content{} - end. - -read_last_publish_message(false, Topic, Content=#coap_content{format = QueryFormat}) when is_binary(QueryFormat)-> - ?LOG(debug, "the QueryFormat=~p", [QueryFormat]), - case emqx_coap_pubsub_topics:lookup_topic_info(Topic) of - [] -> - {error, not_found}; - [{_, MaxAge, CT, Payload, TimeStamp}] -> - case CT =:= format_string_to_int(QueryFormat) of - true -> - return_resource(Topic, Payload, MaxAge, TimeStamp, Content); - false -> - ?LOG(debug, "format value does not match, the queried format=~p, the stored format=~p", [QueryFormat, CT]), - {error, bad_request} - end - end; - -read_last_publish_message(false, Topic, Content) -> - case emqx_coap_pubsub_topics:lookup_topic_info(Topic) of - [] -> - {error, not_found}; - [{_, MaxAge, _, Payload, TimeStamp}] -> - return_resource(Topic, Payload, MaxAge, TimeStamp, Content) - end; - -read_last_publish_message(true, Topic, _Content) -> - ?LOG(debug, "the topic=~p is illegal wildcard topic", [Topic]), - {error, bad_request}. - -delete_topic_info(Topic) -> - case emqx_coap_pubsub_topics:lookup_topic_info(Topic) of - [] -> - {error, not_found}; - [{_, _, _, _, _}] -> - emqx_coap_pubsub_topics:delete_sub_topics(Topic) - end. - -topic(Topic) when is_binary(Topic) -> Topic; -topic([]) -> <<>>; -topic([Path | TopicPath]) -> - case topic(TopicPath) of - <<>> -> Path; - RemTopic -> - <> - end. diff --git a/apps/emqx_coap/src/emqx_coap_registry.erl b/apps/emqx_coap/src/emqx_coap_registry.erl deleted file mode 100644 index 066d37f1e..000000000 --- a/apps/emqx_coap/src/emqx_coap_registry.erl +++ /dev/null @@ -1,154 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_coap_registry). - --author("Feng Lee "). - --include("emqx_coap.hrl"). --include_lib("emqx/include/logger.hrl"). - --logger_header("[CoAP-Registry]"). - --behaviour(gen_server). - -%% API. --export([ start_link/0 - , register_name/2 - , unregister_name/1 - , whereis_name/1 - , send/2 - , stop/0 - ]). - -%% gen_server. --export([ init/1 - , handle_call/3 - , handle_cast/2 - , handle_info/2 - , terminate/2 - , code_change/3 - ]). - --record(state, {}). - --define(RESPONSE_TAB, coap_response_process). --define(RESPONSE_REF_TAB, coap_response_process_ref). - -%% ------------------------------------------------------------------ -%% API Function Definitions -%% ------------------------------------------------------------------ - - -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - -register_name(Name, Pid) -> - gen_server:call(?MODULE, {register_name, Name, Pid}). - -unregister_name(Name) -> - gen_server:call(?MODULE, {unregister_name, Name}). - -whereis_name(Name) -> - case ets:lookup(?RESPONSE_TAB, Name) of - [] -> undefined; - [{Name, Pid, _MRef}] -> Pid - end. - -send(Name, Msg) -> - case whereis_name(Name) of - undefined -> - exit({badarg, {Name, Msg}}); - Pid when is_pid(Pid) -> - Pid ! Msg, - Pid - end. - -stop() -> - gen_server:stop(?MODULE). - - -%% ------------------------------------------------------------------ -%% gen_server Function Definitions -%% ------------------------------------------------------------------ - -init([]) -> - _ = ets:new(?RESPONSE_TAB, [set, named_table, protected]), - _ = ets:new(?RESPONSE_REF_TAB, [set, named_table, protected]), - {ok, #state{}}. - -handle_call({register_name, Name, Pid}, _From, State) -> - case ets:member(?RESPONSE_TAB, Name) of - false -> - MRef = monitor_client(Pid), - ets:insert(?RESPONSE_TAB, {Name, Pid, MRef}), - ets:insert(?RESPONSE_REF_TAB, {MRef, Name, Pid}), - {reply, yes, State}; - true -> {reply, no, State} - end; - -handle_call({unregister_name, Name}, _From, State) -> - case ets:lookup(?RESPONSE_TAB, Name) of - [] -> - ok; - [{Name, _Pid, MRef}] -> - erase_monitor(MRef), - ets:delete(?RESPONSE_TAB, Name), - ets:delete(?RESPONSE_REF_TAB, MRef) - end, - {reply, ok, State}; - -handle_call(_Request, _From, State) -> - {reply, ignored, State}. - -handle_cast(_Msg, State) -> - {noreply, State}. - - -handle_info({'DOWN', MRef, process, DownPid, _Reason}, State) -> - case ets:lookup(?RESPONSE_REF_TAB, MRef) of - [{MRef, Name, _Pid}] -> - ets:delete(?RESPONSE_TAB, Name), - ets:delete(?RESPONSE_REF_TAB, MRef), - erase_monitor(MRef); - [] -> - ?LOG(error, "MRef of client ~p not found", [DownPid]) - end, - {noreply, State}; - - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, _State) -> - ets:delete(?RESPONSE_TAB), - ets:delete(?RESPONSE_REF_TAB), - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - - - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -monitor_client(Pid) -> - erlang:monitor(process, Pid). - -erase_monitor(MRef) -> - catch erlang:demonitor(MRef, [flush]). diff --git a/apps/emqx_coap/src/emqx_coap_resource.erl b/apps/emqx_coap/src/emqx_coap_resource.erl deleted file mode 100644 index 739037a42..000000000 --- a/apps/emqx_coap/src/emqx_coap_resource.erl +++ /dev/null @@ -1,137 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_coap_resource). - --behaviour(coap_resource). - --include("emqx_coap.hrl"). --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("gen_coap/include/coap.hrl"). - --logger_header("[CoAP-RES]"). - --export([ coap_discover/2 - , coap_get/5 - , coap_post/4 - , coap_put/4 - , coap_delete/3 - , coap_observe/5 - , coap_unobserve/1 - , handle_info/2 - , coap_ack/2 - ]). - --ifdef(TEST). --export([topic/1]). --endif. - --define(MQTT_PREFIX, [<<"mqtt">>]). - -% resource operations -coap_discover(_Prefix, _Args) -> - [{absolute, [<<"mqtt">>], []}]. - -coap_get(ChId, ?MQTT_PREFIX, Path, Query, _Content) -> - ?LOG(debug, "coap_get() Path=~p, Query=~p~n", [Path, Query]), - #coap_mqtt_auth{clientid = Clientid, username = Usr, password = Passwd} = get_auth(Query), - case emqx_coap_mqtt_adapter:client_pid(Clientid, Usr, Passwd, ChId) of - {ok, Pid} -> - put(mqtt_client_pid, Pid), - #coap_content{}; - {error, auth_failure} -> - put(mqtt_client_pid, undefined), - {error, unauthorized}; - {error, bad_request} -> - put(mqtt_client_pid, undefined), - {error, bad_request}; - {error, _Other} -> - put(mqtt_client_pid, undefined), - {error, internal_server_error} - end; -coap_get(ChId, Prefix, Path, Query, _Content) -> - ?LOG(error, "ignore bad get request ChId=~p, Prefix=~p, Path=~p, Query=~p", [ChId, Prefix, Path, Query]), - {error, bad_request}. - -coap_post(_ChId, _Prefix, _Topic, _Content) -> - {error, method_not_allowed}. - -coap_put(_ChId, ?MQTT_PREFIX, Topic, #coap_content{payload = Payload}) when Topic =/= [] -> - ?LOG(debug, "put message, Topic=~p, Payload=~p~n", [Topic, Payload]), - Pid = get(mqtt_client_pid), - emqx_coap_mqtt_adapter:publish(Pid, topic(Topic), Payload); -coap_put(_ChId, Prefix, Topic, Content) -> - ?LOG(error, "put has error, Prefix=~p, Topic=~p, Content=~p", [Prefix, Topic, Content]), - {error, bad_request}. - -coap_delete(_ChId, _Prefix, _Topic) -> - {error, method_not_allowed}. - -coap_observe(ChId, ?MQTT_PREFIX, Topic, Ack, Content) when Topic =/= [] -> - TrueTopic = topic(Topic), - ?LOG(debug, "observe Topic=~p, Ack=~p", [TrueTopic, Ack]), - Pid = get(mqtt_client_pid), - case emqx_coap_mqtt_adapter:subscribe(Pid, TrueTopic) of - ok -> {ok, {state, ChId, ?MQTT_PREFIX, [TrueTopic]}, content, Content}; - {error, Code} -> {error, Code} - end; -coap_observe(ChId, Prefix, Topic, Ack, _Content) -> - ?LOG(error, "unknown observe request ChId=~p, Prefix=~p, Topic=~p, Ack=~p", [ChId, Prefix, Topic, Ack]), - {error, bad_request}. - -coap_unobserve({state, _ChId, ?MQTT_PREFIX, Topic}) when Topic =/= [] -> - ?LOG(debug, "unobserve ~p", [Topic]), - Pid = get(mqtt_client_pid), - emqx_coap_mqtt_adapter:unsubscribe(Pid, topic(Topic)), - ok; -coap_unobserve({state, ChId, Prefix, Topic}) -> - ?LOG(error, "ignore unknown unobserve request ChId=~p, Prefix=~p, Topic=~p", [ChId, Prefix, Topic]), - ok. - -handle_info({dispatch, Topic, Payload}, State) -> - ?LOG(debug, "dispatch Topic=~p, Payload=~p", [Topic, Payload]), - {notify, [], #coap_content{format = <<"application/octet-stream">>, payload = Payload}, State}; -handle_info(Message, State) -> - emqx_coap_mqtt_adapter:handle_info(Message, State). - -coap_ack(_Ref, State) -> {ok, State}. - -get_auth(Query) -> - get_auth(Query, #coap_mqtt_auth{}). - -get_auth([], Auth=#coap_mqtt_auth{}) -> - Auth; -get_auth([<<$c, $=, Rest/binary>>|T], Auth=#coap_mqtt_auth{}) -> - get_auth(T, Auth#coap_mqtt_auth{clientid = Rest}); -get_auth([<<$u, $=, Rest/binary>>|T], Auth=#coap_mqtt_auth{}) -> - get_auth(T, Auth#coap_mqtt_auth{username = Rest}); -get_auth([<<$p, $=, Rest/binary>>|T], Auth=#coap_mqtt_auth{}) -> - get_auth(T, Auth#coap_mqtt_auth{password = Rest}); -get_auth([Param|T], Auth=#coap_mqtt_auth{}) -> - ?LOG(error, "ignore unknown parameter ~p", [Param]), - get_auth(T, Auth). - -topic(Topic) when is_binary(Topic) -> Topic; -topic([]) -> <<>>; -topic([Path | TopicPath]) -> - case topic(TopicPath) of - <<>> -> Path; - RemTopic -> - <> - end. - diff --git a/apps/emqx_coap/src/emqx_coap_server.erl b/apps/emqx_coap/src/emqx_coap_server.erl deleted file mode 100644 index ebdc1a0fe..000000000 --- a/apps/emqx_coap/src/emqx_coap_server.erl +++ /dev/null @@ -1,106 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_coap_server). - --include("emqx_coap.hrl"). - --export([ start/1 - , stop/1 - ]). - --export([ start_listener/1 - , start_listener/3 - , stop_listener/1 - , stop_listener/2 - ]). - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -start(Envs) -> - {ok, _} = application:ensure_all_started(gen_coap), - start_listeners(Envs). - -stop(Envs) -> - stop_listeners(Envs). - -%%-------------------------------------------------------------------- -%% Internal funcs -%%-------------------------------------------------------------------- - -start_listeners(Envs) -> - lists:foreach(fun start_listener/1, listeners_confs(Envs)). - -stop_listeners(Envs) -> - lists:foreach(fun stop_listener/1, listeners_confs(Envs)). - -start_listener({Proto, ListenOn, Opts}) -> - case start_listener(Proto, ListenOn, Opts) of - {ok, _Pid} -> - io:format("Start coap:~s listener on ~s successfully.~n", - [Proto, format(ListenOn)]); - {error, Reason} -> - io:format(standard_error, "Failed to start coap:~s listener on ~s: ~0p~n", - [Proto, format(ListenOn), Reason]), - error(Reason) - end. - -start_listener(udp, ListenOn, Opts) -> - coap_server:start_udp('coap:udp', ListenOn, Opts); -start_listener(dtls, ListenOn, Opts) -> - coap_server:start_dtls('coap:dtls', ListenOn, Opts). - -stop_listener({Proto, ListenOn, _Opts}) -> - Ret = stop_listener(Proto, ListenOn), - case Ret of - ok -> io:format("Stop coap:~s listener on ~s successfully.~n", - [Proto, format(ListenOn)]); - {error, Reason} -> - io:format(standard_error, "Failed to stop coap:~s listener on ~s: ~0p~n.", - [Proto, format(ListenOn), Reason]) - end, - Ret. - -stop_listener(udp, ListenOn) -> - coap_server:stop_udp('coap:udp', ListenOn); -stop_listener(dtls, ListenOn) -> - coap_server:stop_dtls('coap:dtls', ListenOn). - -%% XXX: It is a temporary func to convert conf format for esockd -listeners_confs(Envs) -> - listeners_confs(udp, Envs) ++ listeners_confs(dtls, Envs). - -listeners_confs(udp, Envs) -> - Udps = proplists:get_value(bind_udp, Envs, []), - [{udp, Port, [{udp_options, InetOpts}]} || {Port, InetOpts} <- Udps]; - -listeners_confs(dtls, Envs) -> - case proplists:get_value(dtls_opts, Envs, []) of - [] -> []; - DtlsOpts -> - BindDtls = proplists:get_value(bind_dtls, Envs, []), - [{dtls, Port, [{dtls_options, InetOpts ++ DtlsOpts}]} || {Port, InetOpts} <- BindDtls] - end. - -format(Port) when is_integer(Port) -> - io_lib:format("0.0.0.0:~w", [Port]); -format({Addr, Port}) when is_list(Addr) -> - io_lib:format("~s:~w", [Addr, Port]); -format({Addr, Port}) when is_tuple(Addr) -> - io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). - diff --git a/apps/emqx_coap/src/emqx_coap_sup.erl b/apps/emqx_coap/src/emqx_coap_sup.erl deleted file mode 100644 index 94e9a1c77..000000000 --- a/apps/emqx_coap/src/emqx_coap_sup.erl +++ /dev/null @@ -1,42 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_coap_sup). - --behaviour(supervisor). - --export([ start_link/0 - , init/1 - ]). - -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -init(_Args) -> - Registry = #{id => emqx_coap_registry, - start => {emqx_coap_registry, start_link, []}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_coap_registry]}, - PsTopics = #{id => emqx_coap_pubsub_topics, - start => {emqx_coap_pubsub_topics, start_link, []}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_coap_pubsub_topics]}, - {ok, {{one_for_all, 10, 3600}, [Registry, PsTopics]}}. - diff --git a/apps/emqx_coap/src/emqx_coap_timer.erl b/apps/emqx_coap/src/emqx_coap_timer.erl deleted file mode 100644 index 92b0ddb2f..000000000 --- a/apps/emqx_coap/src/emqx_coap_timer.erl +++ /dev/null @@ -1,59 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_coap_timer). - --include("emqx_coap.hrl"). - --export([ cancel_timer/1 - , start_timer/2 - , restart_timer/1 - , kick_timer/1 - , is_timeout/1 - , get_timer_length/1 - ]). - --record(timer_state, {interval, kickme, tref, message}). - --define(LOG(Level, Format, Args), - emqx_logger:Level("CoAP-Timer: " ++ Format, Args)). - -cancel_timer(#timer_state{tref = TRef}) when is_reference(TRef) -> - catch erlang:cancel_timer(TRef), - ok; -cancel_timer(_) -> - ok. - -kick_timer(State=#timer_state{kickme = false}) -> - State#timer_state{kickme = true}; -kick_timer(State=#timer_state{kickme = true}) -> - State. - -start_timer(Sec, Msg) -> - ?LOG(debug, "emqx_coap_timer:start_timer ~p", [Sec]), - TRef = erlang:send_after(timer:seconds(Sec), self(), Msg), - #timer_state{interval = Sec, kickme = false, tref = TRef, message = Msg}. - -restart_timer(State=#timer_state{interval = Sec, message = Msg}) -> - ?LOG(debug, "emqx_coap_timer:restart_timer ~p", [Sec]), - TRef = erlang:send_after(timer:seconds(Sec), self(), Msg), - State#timer_state{kickme = false, tref = TRef}. - -is_timeout(#timer_state{kickme = Bool}) -> - not Bool. - -get_timer_length(#timer_state{interval = Interval}) -> - Interval. diff --git a/apps/emqx_coap/test/emqx_coap_SUITE.erl b/apps/emqx_coap/test/emqx_coap_SUITE.erl deleted file mode 100644 index 73c9ef162..000000000 --- a/apps/emqx_coap/test/emqx_coap_SUITE.erl +++ /dev/null @@ -1,318 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_coap_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("gen_coap/include/coap.hrl"). --include_lib("eunit/include/eunit.hrl"). --include_lib("emqx/include/emqx.hrl"). - --define(LOGT(Format, Args), ct:pal(Format, Args)). - -all() -> emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_coap], fun set_special_cfg/1), - Config. - -set_special_cfg(emqx_coap) -> - Opts = application:get_env(emqx_coap, dtls_opts,[]), - Opts2 = [{keyfile, emqx_ct_helpers:deps_path(emqx, "etc/certs/key.pem")}, - {certfile, emqx_ct_helpers:deps_path(emqx, "etc/certs/cert.pem")}], - application:set_env(emqx_coap, dtls_opts, emqx_misc:merge_opts(Opts, Opts2)), - application:set_env(emqx_coap, enable_stats, true); -set_special_cfg(_) -> - ok. - -end_per_suite(Config) -> - emqx_ct_helpers:stop_apps([emqx_coap]), - Config. - -%%-------------------------------------------------------------------- -%% Test Cases -%%-------------------------------------------------------------------- - -t_publish(_Config) -> - Topic = <<"abc">>, Payload = <<"123">>, - TopicStr = binary_to_list(Topic), - URI = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", - - %% Sub topic first - emqx:subscribe(Topic), - - Reply = er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), - {ok, changed, _} = Reply, - - receive - {deliver, Topic, Msg} -> - ?assertEqual(Topic, Msg#message.topic), - ?assertEqual(Payload, Msg#message.payload) - after - 500 -> - ?assert(false) - end. - -t_publish_acl_deny(_Config) -> - Topic = <<"abc">>, Payload = <<"123">>, - TopicStr = binary_to_list(Topic), - URI = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", - - %% Sub topic first - emqx:subscribe(Topic), - - ok = meck:new(emqx_access_control, [non_strict, passthrough, no_history]), - ok = meck:expect(emqx_access_control, check_acl, 3, deny), - Reply = er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), - ?assertEqual({error,forbidden}, Reply), - ok = meck:unload(emqx_access_control), - receive - {deliver, Topic, Msg} -> ct:fail({unexpected, {Topic, Msg}}) - after - 500 -> ok - end. - -t_observe(_Config) -> - Topic = <<"abc">>, TopicStr = binary_to_list(Topic), - Payload = <<"123">>, - Uri = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", - {ok, Pid, N, Code, Content} = er_coap_observer:observe(Uri), - ?LOGT("observer Pid=~p, N=~p, Code=~p, Content=~p", [Pid, N, Code, Content]), - - [SubPid] = emqx:subscribers(Topic), - ?assert(is_pid(SubPid)), - - %% Publish a message - emqx:publish(emqx_message:make(Topic, Payload)), - - Notif = receive_notification(), - ?LOGT("observer get Notif=~p", [Notif]), - {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv}} = Notif, - ?assertEqual(Payload, PayloadRecv), - - er_coap_observer:stop(Pid), - timer:sleep(100), - - [] = emqx:subscribers(Topic). - -t_observe_acl_deny(_Config) -> - Topic = <<"abc">>, TopicStr = binary_to_list(Topic), - Uri = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", - ok = meck:new(emqx_access_control, [non_strict, passthrough, no_history]), - ok = meck:expect(emqx_access_control, check_acl, 3, deny), - ?assertEqual({error,forbidden}, er_coap_observer:observe(Uri)), - [] = emqx:subscribers(Topic), - ok = meck:unload(emqx_access_control). - -t_observe_wildcard(_Config) -> - Topic = <<"+/b">>, TopicStr = emqx_http_lib:uri_encode(binary_to_list(Topic)), - Payload = <<"123">>, - Uri = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", - {ok, Pid, N, Code, Content} = er_coap_observer:observe(Uri), - ?LOGT("observer Uri=~p, Pid=~p, N=~p, Code=~p, Content=~p", [Uri, Pid, N, Code, Content]), - - [SubPid] = emqx:subscribers(Topic), - ?assert(is_pid(SubPid)), - - %% Publish a message - emqx:publish(emqx_message:make(<<"a/b">>, Payload)), - - Notif = receive_notification(), - ?LOGT("observer get Notif=~p", [Notif]), - {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv}} = Notif, - ?assertEqual(Payload, PayloadRecv), - - er_coap_observer:stop(Pid), - timer:sleep(100), - - [] = emqx:subscribers(Topic). - -t_observe_pub(_Config) -> - Topic = <<"+/b">>, TopicStr = emqx_http_lib:uri_encode(binary_to_list(Topic)), - Uri = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", - {ok, Pid, N, Code, Content} = er_coap_observer:observe(Uri), - ?LOGT("observer Pid=~p, N=~p, Code=~p, Content=~p", [Pid, N, Code, Content]), - - [SubPid] = emqx:subscribers(Topic), - ?assert(is_pid(SubPid)), - - Topic2 = <<"a/b">>, Payload2 = <<"UFO">>, - TopicStr2 = emqx_http_lib:uri_encode(binary_to_list(Topic2)), - URI2 = "coap://127.0.0.1/mqtt/"++TopicStr2++"?c=client1&u=tom&p=secret", - - Reply2 = er_coap_client:request(put, URI2, #coap_content{format = <<"application/octet-stream">>, payload = Payload2}), - {ok,changed, _} = Reply2, - - Notif2 = receive_notification(), - ?LOGT("observer get Notif2=~p", [Notif2]), - {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv2}} = Notif2, - ?assertEqual(Payload2, PayloadRecv2), - - Topic3 = <<"j/b">>, Payload3 = <<"ET629">>, - TopicStr3 = emqx_http_lib:uri_encode(binary_to_list(Topic3)), - URI3 = "coap://127.0.0.1/mqtt/"++TopicStr3++"?c=client2&u=mike&p=guess", - Reply3 = er_coap_client:request(put, URI3, #coap_content{format = <<"application/octet-stream">>, payload = Payload3}), - {ok,changed, _} = Reply3, - - Notif3 = receive_notification(), - ?LOGT("observer get Notif3=~p", [Notif3]), - {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv3}} = Notif3, - ?assertEqual(Payload3, PayloadRecv3), - - er_coap_observer:stop(Pid). - -t_one_clientid_sub_2_topics(_Config) -> - Topic1 = <<"abc">>, TopicStr1 = binary_to_list(Topic1), - Payload1 = <<"123">>, - Uri1 = "coap://127.0.0.1/mqtt/"++TopicStr1++"?c=client1&u=tom&p=secret", - {ok, Pid1, N1, Code1, Content1} = er_coap_observer:observe(Uri1), - ?LOGT("observer 1 Pid=~p, N=~p, Code=~p, Content=~p", [Pid1, N1, Code1, Content1]), - - [SubPid] = emqx:subscribers(Topic1), - ?assert(is_pid(SubPid)), - - Topic2 = <<"x/y">>, TopicStr2 = emqx_http_lib:uri_encode(binary_to_list(Topic2)), - Payload2 = <<"456">>, - Uri2 = "coap://127.0.0.1/mqtt/"++TopicStr2++"?c=client1&u=tom&p=secret", - {ok, Pid2, N2, Code2, Content2} = er_coap_observer:observe(Uri2), - ?LOGT("observer 2 Pid=~p, N=~p, Code=~p, Content=~p", [Pid2, N2, Code2, Content2]), - - [SubPid] = emqx:subscribers(Topic2), - ?assert(is_pid(SubPid)), - - emqx:publish(emqx_message:make(Topic1, Payload1)), - - Notif1 = receive_notification(), - ?LOGT("observer 1 get Notif=~p", [Notif1]), - {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv1}} = Notif1, - ?assertEqual(Payload1, PayloadRecv1), - - emqx:publish(emqx_message:make(Topic2, Payload2)), - - Notif2 = receive_notification(), - ?LOGT("observer 2 get Notif=~p", [Notif2]), - {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv2}} = Notif2, - ?assertEqual(Payload2, PayloadRecv2), - - er_coap_observer:stop(Pid1), - er_coap_observer:stop(Pid2). - -t_invalid_parameter(_Config) -> - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% "cid=client2" is invaid - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - Topic3 = <<"a/b">>, Payload3 = <<"ET629">>, - TopicStr3 = emqx_http_lib:uri_encode(binary_to_list(Topic3)), - URI3 = "coap://127.0.0.1/mqtt/"++TopicStr3++"?cid=client2&u=tom&p=simple", - Reply3 = er_coap_client:request(put, URI3, #coap_content{format = <<"application/octet-stream">>, payload = Payload3}), - ?assertMatch({error,bad_request}, Reply3), - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% "what=hello" is invaid - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - URI4 = "coap://127.0.0.1/mqtt/"++TopicStr3++"?what=hello", - Reply4 = er_coap_client:request(put, URI4, #coap_content{format = <<"application/octet-stream">>, payload = Payload3}), - ?assertMatch({error, bad_request}, Reply4). - -t_invalid_topic(_Config) -> - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% "a/b" is a valid topic string - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - Topic3 = <<"a/b">>, Payload3 = <<"ET629">>, - TopicStr3 = binary_to_list(Topic3), - URI3 = "coap://127.0.0.1/mqtt/"++TopicStr3++"?c=client2&u=tom&p=simple", - Reply3 = er_coap_client:request(put, URI3, #coap_content{format = <<"application/octet-stream">>, payload = Payload3}), - ?assertMatch({ok,changed,_Content}, Reply3), - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% "+?#" is invaid topic string - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - URI4 = "coap://127.0.0.1/mqtt/"++"+?#"++"?what=hello", - Reply4 = er_coap_client:request(put, URI4, #coap_content{format = <<"application/octet-stream">>, payload = Payload3}), - ?assertMatch({error,bad_request}, Reply4). - -% mqtt connection kicked by coap with same client id -t_kick_1(_Config) -> - URI = "coap://127.0.0.1/mqtt/abc?c=clientid&u=tom&p=secret", - % workaround: emqx:subscribe does not kick same client id. - spawn_monitor(fun() -> - {ok, C} = emqtt:start_link([{host, "localhost"}, - {clientid, <<"clientid">>}, - {username, <<"plain">>}, - {password, <<"plain">>}]), - {ok, _} = emqtt:connect(C) end), - er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, - payload = <<"123">>}), - receive - {'DOWN', _, _, _, _} -> ok - after 2000 -> - ?assert(false) - end. - -% mqtt connection kicked by coap with same client id -t_acl(Config) -> - OldPath = emqx:get_env(plugins_etc_dir), - application:set_env(emqx, plugins_etc_dir, - emqx_ct_helpers:deps_path(emqx_authz, "test")), - Conf = #{<<"authz">> => - #{<<"rules">> => - [#{<<"principal">> =>#{<<"username">> => <<"coap">>}, - <<"permission">> => deny, - <<"topics">> => [<<"abc">>], - <<"action">> => <<"publish">>} - ]}}, - ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)), - application:ensure_all_started(emqx_authz), - - emqx:subscribe(<<"abc">>), - URI = "coap://127.0.0.1/mqtt/adbc?c=client1&u=coap&p=secret", - er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, - payload = <<"123">>}), - receive - _Something -> ?assert(false) - after 2000 -> - ok - end, - - file:delete(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf')), - application:set_env(emqx, plugins_etc_dir, OldPath), - application:stop(emqx_authz). - -t_stats(_) -> - ok. - -t_auth_failure(_) -> - ok. - -t_qos_supprot(_) -> - ok. - -%%-------------------------------------------------------------------- -%% Helpers - -receive_notification() -> - receive - {coap_notify, Pid, N2, Code2, Content2} -> - {coap_notify, Pid, N2, Code2, Content2} - after 2000 -> - receive_notification_timeout - end. - -testdir(DataPath) -> - Ls = filename:split(DataPath), - filename:join(lists:sublist(Ls, 1, length(Ls) - 1)). diff --git a/apps/emqx_coap/test/emqx_coap_pubsub_SUITE.erl b/apps/emqx_coap/test/emqx_coap_pubsub_SUITE.erl deleted file mode 100644 index c018b9165..000000000 --- a/apps/emqx_coap/test/emqx_coap_pubsub_SUITE.erl +++ /dev/null @@ -1,677 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_coap_pubsub_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("gen_coap/include/coap.hrl"). --include_lib("eunit/include/eunit.hrl"). --include_lib("emqx/include/emqx.hrl"). - --define(LOGT(Format, Args), ct:pal(Format, Args)). - -all() -> emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_coap], fun set_special_cfg/1), - Config. - -set_special_cfg(emqx_coap) -> - application:set_env(emqx_coap, enable_stats, true); -set_special_cfg(_) -> - ok. - -end_per_suite(Config) -> - emqx_ct_helpers:stop_apps([emqx_coap]), - Config. - -%%-------------------------------------------------------------------- -%% Test Cases -%%-------------------------------------------------------------------- - -t_update_max_age(_Config) -> - TopicInPayload = <<"topic1">>, - Payload = <<";ct=42">>, - Payload1 = <<";ct=50">>, - URI = "coap://127.0.0.1/ps/"++"?c=client1&u=tom&p=secret", - URI2 = "coap://127.0.0.1/ps/topic1"++"?c=client1&u=tom&p=secret", - Reply = er_coap_client:request(post, URI, #coap_content{format = <<"application/link-format">>, payload = Payload}), - ?LOGT("Reply =~p", [Reply]), - {ok,created, #coap_content{location_path = LocPath}} = Reply, - ?assertEqual([<<"/ps/topic1">>] ,LocPath), - TopicInfo = [{TopicInPayload, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(TopicInPayload), - ?LOGT("lookup topic info=~p", [TopicInfo]), - ?assertEqual(60, MaxAge1), - ?assertEqual(<<"42">>, CT1), - - timer:sleep(50), - - %% post to create the same topic but with different max age and ct value in payload - Reply1 = er_coap_client:request(post, URI, #coap_content{max_age = 70, format = <<"application/link-format">>, payload = Payload1}), - {ok,created, #coap_content{location_path = LocPath}} = Reply1, - ?assertEqual([<<"/ps/topic1">>] ,LocPath), - [{TopicInPayload, MaxAge2, CT2, _ResPayload, _TimeStamp1}] = emqx_coap_pubsub_topics:lookup_topic_info(TopicInPayload), - ?assertEqual(70, MaxAge2), - ?assertEqual(<<"50">>, CT2), - - {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI2). - -t_create_subtopic(_Config) -> - TopicInPayload = <<"topic1">>, - TopicInPayloadStr = "topic1", - Payload = <<";ct=42">>, - URI = "coap://127.0.0.1/ps/"++"?c=client1&u=tom&p=secret", - RealURI = "coap://127.0.0.1/ps/topic1"++"?c=client1&u=tom&p=secret", - - Reply = er_coap_client:request(post, URI, #coap_content{format = <<"application/link-format">>, payload = Payload}), - ?LOGT("Reply =~p", [Reply]), - {ok,created, #coap_content{location_path = LocPath}} = Reply, - ?assertEqual([<<"/ps/topic1">>] ,LocPath), - TopicInfo = [{TopicInPayload, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(TopicInPayload), - ?LOGT("lookup topic info=~p", [TopicInfo]), - ?assertEqual(60, MaxAge1), - ?assertEqual(<<"42">>, CT1), - - timer:sleep(50), - - %% post to create the a sub topic - SubPayload = <<";ct=42">>, - SubTopicInPayloadStr = "subtopic", - SubURI = "coap://127.0.0.1/ps/"++TopicInPayloadStr++"?c=client1&u=tom&p=secret", - SubRealURI = "coap://127.0.0.1/ps/"++TopicInPayloadStr++"/"++SubTopicInPayloadStr++"?c=client1&u=tom&p=secret", - FullTopic = list_to_binary(TopicInPayloadStr++"/"++SubTopicInPayloadStr), - Reply1 = er_coap_client:request(post, SubURI, #coap_content{format = <<"application/link-format">>, payload = SubPayload}), - ?LOGT("Reply =~p", [Reply1]), - {ok,created, #coap_content{location_path = LocPath1}} = Reply1, - ?assertEqual([<<"/ps/topic1/subtopic">>] ,LocPath1), - [{FullTopic, MaxAge2, CT2, _ResPayload, _}] = emqx_coap_pubsub_topics:lookup_topic_info(FullTopic), - ?assertEqual(60, MaxAge2), - ?assertEqual(<<"42">>, CT2), - - {ok, deleted, #coap_content{}} = er_coap_client:request(delete, SubRealURI), - {ok, deleted, #coap_content{}} = er_coap_client:request(delete, RealURI). - -t_over_max_age(_Config) -> - TopicInPayload = <<"topic1">>, - Payload = <<";ct=42">>, - URI = "coap://127.0.0.1/ps/"++"?c=client1&u=tom&p=secret", - Reply = er_coap_client:request(post, URI, #coap_content{max_age = 2, format = <<"application/link-format">>, payload = Payload}), - ?LOGT("Reply =~p", [Reply]), - {ok,created, #coap_content{location_path = LocPath}} = Reply, - ?assertEqual([<<"/ps/topic1">>] ,LocPath), - TopicInfo = [{TopicInPayload, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(TopicInPayload), - ?LOGT("lookup topic info=~p", [TopicInfo]), - ?assertEqual(2, MaxAge1), - ?assertEqual(<<"42">>, CT1), - - timer:sleep(3000), - ?assertEqual(true, emqx_coap_pubsub_topics:is_topic_timeout(TopicInPayload)). - -t_refreash_max_age(_Config) -> - TopicInPayload = <<"topic1">>, - Payload = <<";ct=42">>, - Payload1 = <<";ct=50">>, - URI = "coap://127.0.0.1/ps/"++"?c=client1&u=tom&p=secret", - RealURI = "coap://127.0.0.1/ps/topic1"++"?c=client1&u=tom&p=secret", - Reply = er_coap_client:request(post, URI, #coap_content{max_age = 5, format = <<"application/link-format">>, payload = Payload}), - ?LOGT("Reply =~p", [Reply]), - {ok,created, #coap_content{location_path = LocPath}} = Reply, - ?assertEqual([<<"/ps/topic1">>] ,LocPath), - TopicInfo = [{TopicInPayload, MaxAge1, CT1, _ResPayload, TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(TopicInPayload), - ?LOGT("lookup topic info=~p", [TopicInfo]), - ?LOGT("TimeStamp=~p", [TimeStamp]), - ?assertEqual(5, MaxAge1), - ?assertEqual(<<"42">>, CT1), - - timer:sleep(3000), - - %% post to create the same topic, the max age timer will be restarted with the new max age value - Reply1 = er_coap_client:request(post, URI, #coap_content{max_age = 5, format = <<"application/link-format">>, payload = Payload1}), - {ok,created, #coap_content{location_path = LocPath}} = Reply1, - ?assertEqual([<<"/ps/topic1">>] ,LocPath), - [{TopicInPayload, MaxAge2, CT2, _ResPayload, TimeStamp1}] = emqx_coap_pubsub_topics:lookup_topic_info(TopicInPayload), - ?LOGT("TimeStamp1=~p", [TimeStamp1]), - ?assertEqual(5, MaxAge2), - ?assertEqual(<<"50">>, CT2), - - timer:sleep(3000), - ?assertEqual(false, emqx_coap_pubsub_topics:is_topic_timeout(TopicInPayload)), - - {ok, deleted, #coap_content{}} = er_coap_client:request(delete, RealURI). - -t_case01_publish_post(_Config) -> - timer:sleep(100), - MainTopic = <<"maintopic">>, - TopicInPayload = <<"topic1">>, - Payload = <<";ct=42">>, - MainTopicStr = binary_to_list(MainTopic), - - %% post to create topic maintopic/topic1 - URI1 = "coap://127.0.0.1/ps/"++MainTopicStr++"?c=client1&u=tom&p=secret", - FullTopic = list_to_binary(MainTopicStr++"/"++binary_to_list(TopicInPayload)), - Reply1 = er_coap_client:request(post, URI1, #coap_content{format = <<"application/link-format">>, payload = Payload}), - ?LOGT("Reply =~p", [Reply1]), - {ok,created, #coap_content{location_path = LocPath1}} = Reply1, - ?assertEqual([<<"/ps/maintopic/topic1">>] ,LocPath1), - [{FullTopic, MaxAge, CT2, <<>>, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(FullTopic), - ?assertEqual(60, MaxAge), - ?assertEqual(<<"42">>, CT2), - - %% post to publish message to topic maintopic/topic1 - FullTopicStr = emqx_http_lib:uri_encode(binary_to_list(FullTopic)), - URI2 = "coap://127.0.0.1/ps/"++FullTopicStr++"?c=client1&u=tom&p=secret", - PubPayload = <<"PUBLISH">>, - - %% Sub topic first - emqx:subscribe(FullTopic), - - Reply2 = er_coap_client:request(post, URI2, #coap_content{format = <<"application/octet-stream">>, payload = PubPayload}), - ?LOGT("Reply =~p", [Reply2]), - {ok,changed, _} = Reply2, - TopicInfo = [{FullTopic, MaxAge, CT2, PubPayload, _TimeStamp1}] = emqx_coap_pubsub_topics:lookup_topic_info(FullTopic), - ?LOGT("the topic info =~p", [TopicInfo]), - - assert_recv(FullTopic, PubPayload), - {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI2). - -t_case02_publish_post(_Config) -> - Topic = <<"topic1">>, - TopicStr = binary_to_list(Topic), - Payload = <<"payload">>, - - %% Sub topic first - emqx:subscribe(Topic), - - %% post to publish a new topic "topic1", and the topic is created - URI = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", - Reply = er_coap_client:request(post, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), - ?LOGT("Reply =~p", [Reply]), - {ok,created, #coap_content{location_path = LocPath}} = Reply, - ?assertEqual([<<"/ps/topic1">>] ,LocPath), - [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), - ?assertEqual(60, MaxAge), - ?assertEqual(<<"42">>, CT), - - assert_recv(Topic, Payload), - - %% post to publish a new message to the same topic "topic1" with different payload - NewPayload = <<"newpayload">>, - Reply1 = er_coap_client:request(post, URI, #coap_content{format = <<"application/octet-stream">>, payload = NewPayload}), - ?LOGT("Reply =~p", [Reply1]), - {ok,changed, _} = Reply1, - [{Topic, MaxAge, CT, NewPayload, _TimeStamp1}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), - - assert_recv(Topic, NewPayload), - {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). - -t_case03_publish_post(_Config) -> - Topic = <<"topic1">>, - TopicStr = binary_to_list(Topic), - Payload = <<"payload">>, - - %% Sub topic first - emqx:subscribe(Topic), - - %% post to publish a new topic "topic1", and the topic is created - URI = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", - Reply = er_coap_client:request(post, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), - ?LOGT("Reply =~p", [Reply]), - {ok,created, #coap_content{location_path = LocPath}} = Reply, - ?assertEqual([<<"/ps/topic1">>] ,LocPath), - [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), - ?assertEqual(60, MaxAge), - ?assertEqual(<<"42">>, CT), - - assert_recv(Topic, Payload), - - %% post to publish a new message to the same topic "topic1", but the ct is not same as created - NewPayload = <<"newpayload">>, - Reply1 = er_coap_client:request(post, URI, #coap_content{format = <<"application/exi">>, payload = NewPayload}), - ?LOGT("Reply =~p", [Reply1]), - ?assertEqual({error,bad_request}, Reply1), - - {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). - -t_case04_publish_post(_Config) -> - Topic = <<"topic1">>, - TopicStr = binary_to_list(Topic), - Payload = <<"payload">>, - - %% post to publish a new topic "topic1", and the topic is created - URI = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", - Reply = er_coap_client:request(post, URI, #coap_content{max_age = 5, format = <<"application/octet-stream">>, payload = Payload}), - ?LOGT("Reply =~p", [Reply]), - {ok,created, #coap_content{location_path = LocPath}} = Reply, - ?assertEqual([<<"/ps/topic1">>] ,LocPath), - [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), - ?assertEqual(5, MaxAge), - ?assertEqual(<<"42">>, CT), - - %% after max age timeout, the topic still exists but the status is timeout - timer:sleep(6000), - ?assertEqual(true, emqx_coap_pubsub_topics:is_topic_timeout(Topic)), - - {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). - -t_case01_publish_put(_Config) -> - MainTopic = <<"maintopic">>, - TopicInPayload = <<"topic1">>, - Payload = <<";ct=42">>, - MainTopicStr = binary_to_list(MainTopic), - - %% post to create topic maintopic/topic1 - URI1 = "coap://127.0.0.1/ps/"++MainTopicStr++"?c=client1&u=tom&p=secret", - FullTopic = list_to_binary(MainTopicStr++"/"++binary_to_list(TopicInPayload)), - Reply1 = er_coap_client:request(post, URI1, #coap_content{format = <<"application/link-format">>, payload = Payload}), - ?LOGT("Reply =~p", [Reply1]), - {ok,created, #coap_content{location_path = LocPath1}} = Reply1, - ?assertEqual([<<"/ps/maintopic/topic1">>] ,LocPath1), - [{FullTopic, MaxAge, CT2, <<>>, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(FullTopic), - ?assertEqual(60, MaxAge), - ?assertEqual(<<"42">>, CT2), - - %% put to publish message to topic maintopic/topic1 - FullTopicStr = emqx_http_lib:uri_encode(binary_to_list(FullTopic)), - URI2 = "coap://127.0.0.1/ps/"++FullTopicStr++"?c=client1&u=tom&p=secret", - PubPayload = <<"PUBLISH">>, - - %% Sub topic first - emqx:subscribe(FullTopic), - - Reply2 = er_coap_client:request(put, URI2, #coap_content{format = <<"application/octet-stream">>, payload = PubPayload}), - ?LOGT("Reply =~p", [Reply2]), - {ok,changed, _} = Reply2, - [{FullTopic, MaxAge, CT2, PubPayload, _TimeStamp1}] = emqx_coap_pubsub_topics:lookup_topic_info(FullTopic), - - assert_recv(FullTopic, PubPayload), - - {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI2). - -t_case02_publish_put(_Config) -> - Topic = <<"topic1">>, - TopicStr = binary_to_list(Topic), - Payload = <<"payload">>, - - %% Sub topic first - emqx:subscribe(Topic), - - %% put to publish a new topic "topic1", and the topic is created - URI = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", - Reply = er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), - ?LOGT("Reply =~p", [Reply]), - {ok,created, #coap_content{location_path = LocPath}} = Reply, - ?assertEqual([<<"/ps/topic1">>] ,LocPath), - [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), - ?assertEqual(60, MaxAge), - ?assertEqual(<<"42">>, CT), - - assert_recv(Topic, Payload), - - %% put to publish a new message to the same topic "topic1" with different payload - NewPayload = <<"newpayload">>, - Reply1 = er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, payload = NewPayload}), - ?LOGT("Reply =~p", [Reply1]), - {ok,changed, _} = Reply1, - [{Topic, MaxAge, CT, NewPayload, _TimeStamp1}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), - - assert_recv(Topic, NewPayload), - - {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). - -t_case03_publish_put(_Config) -> - Topic = <<"topic1">>, - TopicStr = binary_to_list(Topic), - Payload = <<"payload">>, - - %% Sub topic first - emqx:subscribe(Topic), - - %% put to publish a new topic "topic1", and the topic is created - URI = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", - Reply = er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), - ?LOGT("Reply =~p", [Reply]), - {ok,created, #coap_content{location_path = LocPath}} = Reply, - ?assertEqual([<<"/ps/topic1">>] ,LocPath), - [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), - ?assertEqual(60, MaxAge), - ?assertEqual(<<"42">>, CT), - - assert_recv(Topic, Payload), - - %% put to publish a new message to the same topic "topic1", but the ct is not same as created - NewPayload = <<"newpayload">>, - Reply1 = er_coap_client:request(put, URI, #coap_content{format = <<"application/exi">>, payload = NewPayload}), - ?LOGT("Reply =~p", [Reply1]), - ?assertEqual({error,bad_request}, Reply1), - - {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). - -t_case04_publish_put(_Config) -> - Topic = <<"topic1">>, - TopicStr = binary_to_list(Topic), - Payload = <<"payload">>, - - %% put to publish a new topic "topic1", and the topic is created - URI = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", - Reply = er_coap_client:request(put, URI, #coap_content{max_age = 5, format = <<"application/octet-stream">>, payload = Payload}), - ?LOGT("Reply =~p", [Reply]), - {ok,created, #coap_content{location_path = LocPath}} = Reply, - ?assertEqual([<<"/ps/topic1">>] ,LocPath), - [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), - ?assertEqual(5, MaxAge), - ?assertEqual(<<"42">>, CT), - - %% after max age timeout, no publish message to the same topic, the topic info will be deleted - %%%%%%%%%%%%%%%%%%%%%%%%%% - % but there is one thing to do is we don't count in the publish message received from emqx(from other node).TBD!!!!!!!!!!!!! - %%%%%%%%%%%%%%%%%%%%%%%%%% - timer:sleep(6000), - ?assertEqual(true, emqx_coap_pubsub_topics:is_topic_timeout(Topic)), - - {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). - -t_case01_subscribe(_Config) -> - Topic = <<"topic1">>, - Payload1 = <<";ct=42">>, - timer:sleep(100), - - %% First post to create a topic "topic1" - Uri = "coap://127.0.0.1/ps/"++"?c=client1&u=tom&p=secret", - Reply = er_coap_client:request(post, Uri, #coap_content{format = <<"application/link-format">>, payload = Payload1}), - ?LOGT("Reply =~p", [Reply]), - {ok,created, #coap_content{location_path = [LocPath]}} = Reply, - ?assertEqual(<<"/ps/topic1">> ,LocPath), - TopicInfo = [{Topic, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), - ?LOGT("lookup topic info=~p", [TopicInfo]), - ?assertEqual(60, MaxAge1), - ?assertEqual(<<"42">>, CT1), - - %% Subscribe the topic - Uri1 = "coap://127.0.0.1"++binary_to_list(LocPath)++"?c=client1&u=tom&p=secret", - {ok, Pid, N, Code, Content} = er_coap_observer:observe(Uri1), - ?LOGT("observer Pid=~p, N=~p, Code=~p, Content=~p", [Pid, N, Code, Content]), - - [SubPid] = emqx:subscribers(Topic), - ?assert(is_pid(SubPid)), - - %% Publish a message - Payload = <<"123">>, - emqx:publish(emqx_message:make(Topic, Payload)), - - Notif = receive_notification(), - ?LOGT("observer get Notif=~p", [Notif]), - {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv}} = Notif, - - ?assertEqual(Payload, PayloadRecv), - - %% GET to read the publish message of the topic - Reply1 = er_coap_client:request(get, Uri1), - ?LOGT("Reply=~p", [Reply1]), - {ok,content, #coap_content{payload = <<"123">>}} = Reply1, - - er_coap_observer:stop(Pid), - {ok, deleted, #coap_content{}} = er_coap_client:request(delete, Uri1). - -t_case02_subscribe(_Config) -> - Topic = <<"a/b">>, - TopicStr = binary_to_list(Topic), - PercentEncodedTopic = emqx_http_lib:uri_encode(TopicStr), - Payload = <<"payload">>, - - %% post to publish a new topic "a/b", and the topic is created - URI = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", - Reply = er_coap_client:request(post, URI, #coap_content{max_age = 5, format = <<"application/octet-stream">>, payload = Payload}), - ?LOGT("Reply =~p", [Reply]), - {ok,created, #coap_content{location_path = LocPath}} = Reply, - ?assertEqual([<<"/ps/a/b">>] ,LocPath), - [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), - ?assertEqual(5, MaxAge), - ?assertEqual(<<"42">>, CT), - - %% Wait for the max age of the timer expires - timer:sleep(6000), - ?assertEqual(true, emqx_coap_pubsub_topics:is_topic_timeout(Topic)), - - %% Subscribe to the timeout topic "a/b", still successfully，got {ok, nocontent} Method - Uri = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", - Reply1 = {ok, Pid, _N, nocontent, _} = er_coap_observer:observe(Uri), - ?LOGT("Subscribe Reply=~p", [Reply1]), - - [SubPid] = emqx:subscribers(Topic), - ?assert(is_pid(SubPid)), - - %% put to publish to topic "a/b" - Reply2 = er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), - {ok,changed, #coap_content{}} = Reply2, - [{Topic, MaxAge1, CT, Payload, TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), - ?assertEqual(60, MaxAge1), - ?assertEqual(<<"42">>, CT), - ?assertEqual(false, TimeStamp =:= timeout), - - %% Publish a message - emqx:publish(emqx_message:make(Topic, Payload)), - - Notif = receive_notification(), - ?LOGT("observer get Notif=~p", [Notif]), - {coap_notify, _, _, {ok,content}, #coap_content{payload = Payload}} = Notif, - - er_coap_observer:stop(Pid), - {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). - -t_case03_subscribe(_Config) -> - %% Subscribe to the unexisted topic "a/b", got not_found - Topic = <<"a/b">>, - TopicStr = binary_to_list(Topic), - PercentEncodedTopic = emqx_http_lib:uri_encode(TopicStr), - Uri = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", - {error, not_found} = er_coap_observer:observe(Uri), - - [] = emqx:subscribers(Topic). - -t_case04_subscribe(_Config) -> - %% Subscribe to the wildcad topic "+/b", got bad_request - Topic = <<"+/b">>, - TopicStr = binary_to_list(Topic), - PercentEncodedTopic = emqx_http_lib:uri_encode(TopicStr), - Uri = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", - {error, bad_request} = er_coap_observer:observe(Uri), - - [] = emqx:subscribers(Topic). - -t_case01_read(_Config) -> - Topic = <<"topic1">>, - TopicStr = binary_to_list(Topic), - Payload = <<"PubPayload">>, - timer:sleep(100), - - %% First post to create a topic "topic1" - Uri = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", - Reply = er_coap_client:request(post, Uri, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), - ?LOGT("Reply =~p", [Reply]), - {ok,created, #coap_content{location_path = [LocPath]}} = Reply, - ?assertEqual(<<"/ps/topic1">> ,LocPath), - TopicInfo = [{Topic, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), - ?LOGT("lookup topic info=~p", [TopicInfo]), - ?assertEqual(60, MaxAge1), - ?assertEqual(<<"42">>, CT1), - - %% GET to read the publish message of the topic - timer:sleep(1000), - Reply1 = er_coap_client:request(get, Uri), - ?LOGT("Reply=~p", [Reply1]), - {ok,content, #coap_content{payload = Payload}} = Reply1, - - {ok, deleted, #coap_content{}} = er_coap_client:request(delete, Uri). - -t_case02_read(_Config) -> - Topic = <<"topic1">>, - TopicStr = binary_to_list(Topic), - Payload = <<"PubPayload">>, - timer:sleep(100), - - %% First post to publish a topic "topic1" - Uri = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", - Reply = er_coap_client:request(post, Uri, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), - ?LOGT("Reply =~p", [Reply]), - {ok,created, #coap_content{location_path = [LocPath]}} = Reply, - ?assertEqual(<<"/ps/topic1">> ,LocPath), - TopicInfo = [{Topic, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), - ?LOGT("lookup topic info=~p", [TopicInfo]), - ?assertEqual(60, MaxAge1), - ?assertEqual(<<"42">>, CT1), - - %% GET to read the publish message of unmatched format, got bad_request - Reply1 = er_coap_client:request(get, Uri, #coap_content{format = <<"application/json">>}), - ?LOGT("Reply=~p", [Reply1]), - {error, bad_request} = Reply1, - - {ok, deleted, #coap_content{}} = er_coap_client:request(delete, Uri). - -t_case03_read(_Config) -> - Topic = <<"topic1">>, - TopicStr = binary_to_list(Topic), - Uri = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", - timer:sleep(100), - - %% GET to read the nexisted topic "topic1", got not_found - Reply = er_coap_client:request(get, Uri), - ?LOGT("Reply=~p", [Reply]), - {error, not_found} = Reply. - -t_case04_read(_Config) -> - Topic = <<"topic1">>, - TopicStr = binary_to_list(Topic), - Payload = <<"PubPayload">>, - timer:sleep(100), - - %% First post to publish a topic "topic1" - Uri = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", - Reply = er_coap_client:request(post, Uri, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), - ?LOGT("Reply =~p", [Reply]), - {ok,created, #coap_content{location_path = [LocPath]}} = Reply, - ?assertEqual(<<"/ps/topic1">> ,LocPath), - TopicInfo = [{Topic, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), - ?LOGT("lookup topic info=~p", [TopicInfo]), - ?assertEqual(60, MaxAge1), - ?assertEqual(<<"42">>, CT1), - - %% GET to read the publish message of wildcard topic, got bad_request - WildTopic = binary_to_list(<<"+/topic1">>), - Uri1 = "coap://127.0.0.1/ps/"++WildTopic++"?c=client1&u=tom&p=secret", - Reply1 = er_coap_client:request(get, Uri1, #coap_content{format = <<"application/json">>}), - ?LOGT("Reply=~p", [Reply1]), - {error, bad_request} = Reply1, - - {ok, deleted, #coap_content{}} = er_coap_client:request(delete, Uri). - -t_case05_read(_Config) -> - Topic = <<"a/b">>, - TopicStr = binary_to_list(Topic), - PercentEncodedTopic = emqx_http_lib:uri_encode(TopicStr), - Payload = <<"payload">>, - - %% post to publish a new topic "a/b", and the topic is created - URI = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", - Reply = er_coap_client:request(post, URI, #coap_content{max_age = 5, format = <<"application/octet-stream">>, payload = Payload}), - ?LOGT("Reply =~p", [Reply]), - {ok,created, #coap_content{location_path = LocPath}} = Reply, - ?assertEqual([<<"/ps/a/b">>] ,LocPath), - [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), - ?assertEqual(5, MaxAge), - ?assertEqual(<<"42">>, CT), - - %% Wait for the max age of the timer expires - timer:sleep(6000), - ?assertEqual(true, emqx_coap_pubsub_topics:is_topic_timeout(Topic)), - - %% GET to read the expired publish message, supposed to get {ok, nocontent}, but now got {ok, content} - Reply1 = er_coap_client:request(get, URI), - ?LOGT("Reply=~p", [Reply1]), - {ok, content, #coap_content{payload = <<>>}}= Reply1, - - {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). - -t_case01_delete(_Config) -> - TopicInPayload = <<"a/b">>, - TopicStr = binary_to_list(TopicInPayload), - PercentEncodedTopic = emqx_http_lib:uri_encode(TopicStr), - Payload = list_to_binary("<"++PercentEncodedTopic++">;ct=42"), - URI = "coap://127.0.0.1/ps/"++"?c=client1&u=tom&p=secret", - - %% Client post to CREATE topic "a/b" - Reply = er_coap_client:request(post, URI, #coap_content{format = <<"application/link-format">>, payload = Payload}), - ?LOGT("Reply =~p", [Reply]), - {ok,created, #coap_content{location_path = LocPath}} = Reply, - ?assertEqual([<<"/ps/a/b">>] ,LocPath), - - %% Client post to CREATE topic "a/b/c" - TopicInPayload1 = <<"a/b/c">>, - PercentEncodedTopic1 = emqx_http_lib:uri_encode(binary_to_list(TopicInPayload1)), - Payload1 = list_to_binary("<"++PercentEncodedTopic1++">;ct=42"), - Reply1 = er_coap_client:request(post, URI, #coap_content{format = <<"application/link-format">>, payload = Payload1}), - ?LOGT("Reply =~p", [Reply1]), - {ok,created, #coap_content{location_path = LocPath1}} = Reply1, - ?assertEqual([<<"/ps/a/b/c">>] ,LocPath1), - - timer:sleep(50), - - %% DELETE the topic "a/b" - UriD = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", - ReplyD = er_coap_client:request(delete, UriD), - ?LOGT("Reply=~p", [ReplyD]), - {ok, deleted, #coap_content{}}= ReplyD, - - timer:sleep(300), %% Waiting gen_server:cast/2 for deleting operation - ?assertEqual(false, emqx_coap_pubsub_topics:is_topic_existed(TopicInPayload)), - ?assertEqual(false, emqx_coap_pubsub_topics:is_topic_existed(TopicInPayload1)). - -t_case02_delete(_Config) -> - TopicInPayload = <<"a/b">>, - TopicStr = binary_to_list(TopicInPayload), - PercentEncodedTopic = emqx_http_lib:uri_encode(TopicStr), - - %% DELETE the unexisted topic "a/b" - Uri1 = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", - Reply1 = er_coap_client:request(delete, Uri1), - ?LOGT("Reply=~p", [Reply1]), - {error, not_found} = Reply1. - -t_case13_emit_stats_test(_Config) -> - ok. - -%%-------------------------------------------------------------------- -%% Internal functions - -receive_notification() -> - receive - {coap_notify, Pid, N2, Code2, Content2} -> - {coap_notify, Pid, N2, Code2, Content2} - after 2000 -> - receive_notification_timeout - end. - -assert_recv(Topic, Payload) -> - receive - {deliver, _, Msg} -> - ?assertEqual(Topic, Msg#message.topic), - ?assertEqual(Payload, Msg#message.payload) - after - 500 -> - ?assert(false) - end. - diff --git a/apps/emqx_connector/include/emqx_connector.hrl b/apps/emqx_connector/include/emqx_connector.hrl index 143816402..fb299b19b 100644 --- a/apps/emqx_connector/include/emqx_connector.hrl +++ b/apps/emqx_connector/include/emqx_connector.hrl @@ -1,4 +1,4 @@ -define(VALID, emqx_resource_validator). --define(REQUIRED(MSG), ?VALID:required(MSG)). +-define(NOT_EMPTY(MSG), ?VALID:not_empty(MSG)). -define(MAX(MAXV), ?VALID:max(number, MAXV)). -define(MIN(MINV), ?VALID:min(number, MINV)). diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index b12bd3edb..cbeff37eb 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -4,12 +4,11 @@ ]}. {deps, [ - {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}}, {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}}, {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}}, {epgsql, {git, "https://github.com/epgsql/epgsql", {tag, "4.4.0"}}}, %% NOTE: mind poolboy version when updating mongodb-erlang version - {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.7"}}}, + {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.8"}}}, %% NOTE: mind poolboy version when updating eredis_cluster version {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.6.7"}}}, %% mongodb-erlang uses a special fork https://github.com/comtihon/poolboy.git diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index 5ff7d0828..0b8717a0f 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,14 +1,18 @@ {application, emqx_connector, [{description, "An OTP application"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [kernel, stdlib, + ecpool, emqx_resource, eredis_cluster, - ecpool + eredis, + epgsql, + mysql, + mongodb ]}, {env,[]}, {modules, []}, diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl new file mode 100644 index 000000000..11860f32d --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -0,0 +1,216 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connector_http). + +-include("emqx_connector.hrl"). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). + +%% callbacks of behaviour emqx_resource +-export([ on_start/2 + , on_stop/2 + , on_query/4 + , on_health_check/2 + ]). + +-type url() :: emqx_http_lib:uri_map(). +-reflect_type([url/0]). +-typerefl_from_string({url/0, emqx_http_lib, uri_parse}). + +-export([ structs/0 + , fields/1 + , validations/0]). + +-export([ check_ssl_opts/2 ]). + +-type connect_timeout() :: non_neg_integer() | infinity. +-type pool_type() :: random | hash. + +-reflect_type([ connect_timeout/0 + , pool_type/0 + ]). + +%%===================================================================== +%% Hocon schema +structs() -> [""]. + +fields("") -> + [{config, #{type => hoconsc:ref(?MODULE, config)}}]; + +fields(config) -> + [ {base_url, fun base_url/1} + , {connect_timeout, fun connect_timeout/1} + , {max_retries, fun max_retries/1} + , {retry_interval, fun retry_interval/1} + , {pool_type, fun pool_type/1} + , {pool_size, fun pool_size/1} + , {enable_pipelining, fun enable_pipelining/1} + , {ssl_opts, #{type => hoconsc:ref(?MODULE, ssl_opts), + default => #{}}} + ]; + +fields(ssl_opts) -> + [ {cacertfile, fun cacertfile/1} + , {keyfile, fun keyfile/1} + , {certfile, fun certfile/1} + , {verify, fun verify/1} + ]. + +validations() -> + [ {check_ssl_opts, fun check_ssl_opts/1} ]. + +base_url(type) -> url(); +base_url(nullable) -> false; +base_url(validate) -> fun (#{query := _Query}) -> + {error, "There must be no query in the base_url"}; + (_) -> ok + end; +base_url(_) -> undefined. + +connect_timeout(type) -> connect_timeout(); +connect_timeout(default) -> 5000; +connect_timeout(_) -> undefined. + +max_retries(type) -> non_neg_integer(); +max_retries(default) -> 5; +max_retries(_) -> undefined. + +retry_interval(type) -> non_neg_integer(); +retry_interval(default) -> 1000; +retry_interval(_) -> undefined. + +pool_type(type) -> pool_type(); +pool_type(default) -> random; +pool_type(_) -> undefined. + +pool_size(type) -> non_neg_integer(); +pool_size(default) -> 8; +pool_size(_) -> undefined. + +enable_pipelining(type) -> boolean(); +enable_pipelining(default) -> true; +enable_pipelining(_) -> undefined. + +cacertfile(type) -> string(); +cacertfile(nullable) -> true; +cacertfile(_) -> undefined. + +keyfile(type) -> string(); +keyfile(nullable) -> true; +keyfile(_) -> undefined. + +%% TODO: certfile is required +certfile(type) -> string(); +certfile(nullable) -> true; +certfile(_) -> undefined. + +verify(type) -> boolean(); +verify(default) -> false; +verify(_) -> undefined. + +%% =================================================================== +on_start(InstId, #{base_url := #{scheme := Scheme, + host := Host, + port := Port, + path := BasePath}, + connect_timeout := ConnectTimeout, + max_retries := MaxRetries, + retry_interval := RetryInterval, + pool_type := PoolType, + pool_size := PoolSize} = Config) -> + logger:info("starting http connector: ~p, config: ~p", [InstId, Config]), + {Transport, TransportOpts} = case Scheme of + http -> + {tcp, []}; + https -> + SSLOpts = emqx_plugin_libs_ssl:save_files_return_opts( + maps:get(ssl_opts, Config), "connectors", InstId), + {tls, SSLOpts} + end, + NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), + PoolOpts = [ {host, Host} + , {port, Port} + , {connect_timeout, ConnectTimeout} + , {retry, MaxRetries} + , {retry_timeout, RetryInterval} + , {keepalive, 5000} + , {pool_type, PoolType} + , {pool_size, PoolSize} + , {transport, Transport} + , {transport_opts, NTransportOpts}], + PoolName = emqx_plugin_libs_pool:pool_name(InstId), + {ok, _} = ehttpc_sup:start_pool(PoolName, PoolOpts), + {ok, #{pool_name => PoolName, + host => Host, + port => Port, + base_path => BasePath}}. + +on_stop(InstId, #{pool_name := PoolName}) -> + logger:info("stopping http connector: ~p", [InstId]), + ehttpc_sup:stop_pool(PoolName). + +on_query(InstId, {Method, Request}, AfterQuery, State) -> + on_query(InstId, {undefined, Method, Request, 5000}, AfterQuery, State); +on_query(InstId, {Method, Request, Timeout}, AfterQuery, State) -> + on_query(InstId, {undefined, Method, Request, Timeout}, AfterQuery, State); +on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery, #{pool_name := PoolName, + base_path := BasePath} = State) -> + logger:debug("http connector ~p received request: ~p, at state: ~p", [InstId, Request, State]), + NRequest = update_path(BasePath, Request), + case Result = ehttpc:request(case KeyOrNum of + undefined -> PoolName; + _ -> {PoolName, KeyOrNum} + end, Method, NRequest, Timeout) of + {error, Reason} -> + logger:debug("http connector ~p do reqeust failed, sql: ~p, reason: ~p", [InstId, NRequest, Reason]), + emqx_resource:query_failed(AfterQuery); + _ -> + emqx_resource:query_success(AfterQuery) + end, + Result. + +on_health_check(_InstId, #{host := Host, port := Port} = State) -> + case gen_tcp:connect(Host, Port, emqx_misc:ipv6_probe([]), 3000) of + {ok, Sock} -> + gen_tcp:close(Sock), + {ok, State}; + {error, _Reason} -> + {error, test_query_failed, State} + end. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +check_ssl_opts(Conf) -> + check_ssl_opts("base_url", Conf). + +check_ssl_opts(URLFrom, Conf) -> + #{schema := Scheme} = hocon_schema:get_value(URLFrom, Conf), + SSLOpts = hocon_schema:get_value("ssl_opts", Conf), + case {Scheme, maps:size(SSLOpts)} of + {http, 0} -> true; + {http, _} -> false; + {https, 0} -> false; + {https, _} -> true + end. + +update_path(BasePath, {Path, Headers}) -> + {filename:join(BasePath, Path), Headers}; +update_path(BasePath, {Path, Headers, Body}) -> + {filename:join(BasePath, Path), Headers, Body}. diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl index ca5ff2482..8c0504d53 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -38,7 +38,7 @@ structs() -> [""]. fields("") -> - redis_fields() ++ + ldap_fields() ++ emqx_connector_schema_lib:ssl_fields(). on_jsonify(Config) -> @@ -51,10 +51,17 @@ on_start(InstId, #{servers := Servers0, bind_password := BindPassword, timeout := Timeout, pool_size := PoolSize, - auto_reconnect := AutoReconn} = Config) -> - logger:info("starting redis connector: ~p, config: ~p", [InstId, Config]), + auto_reconnect := AutoReconn, + ssl := SSL} = Config) -> + logger:info("starting ldap connector: ~p, config: ~p", [InstId, Config]), Servers = [begin proplists:get_value(host, S) end || S <- Servers0], - SslOpts = init_ssl_opts(Config, InstId), + SslOpts = case maps:get(enable, SSL) of + true -> + [{ssl, true}, + {sslopts, emqx_plugin_libs_ssl:save_files_return_opts(SSL, "connectors", InstId)} + ]; + false -> [{ssl, false}] + end, Opts = [{servers, Servers}, {port, Port}, {bind_dn, BindDn}, @@ -68,14 +75,14 @@ on_start(InstId, #{servers := Servers0, {ok, #{poolname => PoolName}}. on_stop(InstId, #{poolname := PoolName}) -> - logger:info("stopping redis connector: ~p", [InstId]), + logger:info("stopping ldap connector: ~p", [InstId]), emqx_plugin_libs_pool:stop_pool(PoolName). on_query(InstId, {search, Base, Filter, Attributes}, AfterQuery, #{poolname := PoolName} = State) -> - logger:debug("redis connector ~p received request: ~p, at state: ~p", [InstId, {Base, Filter, Attributes}, State]), + logger:debug("ldap connector ~p received request: ~p, at state: ~p", [InstId, {Base, Filter, Attributes}, State]), case Result = ecpool:pick_and_do(PoolName, {?MODULE, search, [Base, Filter, Attributes]}, no_handover) of {error, Reason} -> - logger:debug("redis connector ~p do request failed, request: ~p, reason: ~p", [InstId, {Base, Filter, Attributes}, Reason]), + logger:debug("ldap connector ~p do request failed, request: ~p, reason: ~p", [InstId, {Base, Filter, Attributes}, Reason]), emqx_resource:query_failed(AfterQuery); _ -> emqx_resource:query_success(AfterQuery) @@ -116,14 +123,7 @@ connect(Opts) -> ok = eldap2:simple_bind(LDAP, BindDn, BindPassword), {ok, LDAP}. -init_ssl_opts(#{ssl := true} = Config, InstId) -> - [{ssl, true}, - {sslopts, emqx_plugin_libs_ssl:save_files_return_opts(Config, "connectors", InstId)} - ]; -init_ssl_opts(_Config, _InstId) -> - [{ssl, false}]. - -redis_fields() -> +ldap_fields() -> [ {servers, fun emqx_connector_schema_lib:servers/1} , {port, fun port/1} , {pool_size, fun emqx_connector_schema_lib:pool_size/1} diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index 841cc0a2a..4af339538 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -19,6 +19,9 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). +-type server() :: string(). +-reflect_type([server/0]). + %% callbacks of behaviour emqx_resource -export([ on_start/2 , on_stop/2 @@ -36,38 +39,78 @@ structs() -> [""]. fields("") -> - mongodb_fields() ++ - mongodb_topology_fields() ++ - mongodb_rs_set_name_fields() ++ + [ {config, #{type => hoconsc:union( + [ hoconsc:ref(?MODULE, single) + , hoconsc:ref(?MODULE, rs) + , hoconsc:ref(?MODULE, sharded) + ])}} + ]; +fields(single) -> + [ {mongo_type, #{type => single, + default => single}} + , {server, fun server/1} + ] ++ mongo_fields(); +fields(rs) -> + [ {mongo_type, #{type => rs, + default => rs}} + , {servers, fun servers/1} + , {replica_set_name, fun emqx_connector_schema_lib:database/1} + ] ++ mongo_fields(); +fields(sharded) -> + [ {mongo_type, #{type => sharded, + default => sharded}} + , {servers, fun servers/1} + ] ++ mongo_fields(); +fields(topology) -> + [ {max_overflow, fun emqx_connector_schema_lib:pool_size/1} + , {overflow_ttl, fun duration/1} + , {overflow_check_period, fun duration/1} + , {local_threshold_ms, fun duration/1} + , {connect_timeout_ms, fun duration/1} + , {socket_timeout_ms, fun duration/1} + , {server_selection_timeout_ms, fun duration/1} + , {wait_queue_timeout_ms, fun duration/1} + , {heartbeat_frequency_ms, fun duration/1} + , {min_heartbeat_frequency_ms, fun duration/1} + ]. + +mongo_fields() -> + [ {pool_size, fun emqx_connector_schema_lib:pool_size/1} + , {username, fun emqx_connector_schema_lib:username/1} + , {password, fun emqx_connector_schema_lib:password/1} + , {auth_source, #{type => binary(), + nullable => true}} + , {database, fun emqx_connector_schema_lib:database/1} + ] ++ emqx_connector_schema_lib:ssl_fields(). on_jsonify(Config) -> Config. %% =================================================================== -on_start(InstId, #{servers := Servers, - mongo_type := Type, - database := Database, - pool_size := PoolSize} = Config) -> +on_start(InstId, Config = #{server := Server, + mongo_type := single}) -> logger:info("starting mongodb connector: ~p, config: ~p", [InstId, Config]), - SslOpts = init_ssl_opts(Config, InstId), - Hosts = [string:trim(H) || H <- string:tokens(binary_to_list(Servers), ",")], - Opts = [{type, init_type(Type, Config)}, - {hosts, Hosts}, - {pool_size, PoolSize}, - {options, init_topology_options(maps:to_list(Config), [])}, - {worker_options, init_worker_options(maps:to_list(Config), SslOpts)}], + Opts = [{type, single}, + {hosts, [Server]} + ], + do_start(InstId, Opts, Config); - %% test the connection - TestOpts = [{database, Database}] ++ host_port(hd(Hosts)), - {ok, TestConn} = mc_worker_api:connect(TestOpts), +on_start(InstId, Config = #{servers := Servers, + mongo_type := rs, + replica_set_name := RsName}) -> + logger:info("starting mongodb connector: ~p, config: ~p", [InstId, Config]), + Opts = [{type, {rs, RsName}}, + {hosts, Servers}], + do_start(InstId, Opts, Config); - PoolName = emqx_plugin_libs_pool:pool_name(InstId), - _ = emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts ++ SslOpts), - {ok, #{pool => PoolName, - type => Type, - test_conn => TestConn, - test_opts => TestOpts}}. +on_start(InstId, Config = #{servers := Servers, + mongo_type := sharded}) -> + logger:info("starting mongodb connector: ~p, config: ~p", [InstId, Config]), + Opts = [{type, sharded}, + {hosts, Servers} + ], + do_start(InstId, Opts, Config). on_stop(InstId, #{poolname := PoolName}) -> logger:info("stopping mongodb connector: ~p", [InstId]), @@ -75,23 +118,27 @@ on_stop(InstId, #{poolname := PoolName}) -> on_query(InstId, {Action, Collection, Selector, Docs}, AfterQuery, #{poolname := PoolName} = State) -> logger:debug("mongodb connector ~p received request: ~p, at state: ~p", [InstId, {Action, Collection, Selector, Docs}, State]), - case Result = ecpool:pick_and_do(PoolName, {?MODULE, mongo_query, [Action, Collection, Selector, Docs]}, no_handover) of + case ecpool:pick_and_do(PoolName, {?MODULE, mongo_query, [Action, Collection, Selector, Docs]}, no_handover) of {error, Reason} -> logger:debug("mongodb connector ~p do sql query failed, request: ~p, reason: ~p", [InstId, {Action, Collection, Selector, Docs}, Reason]), - emqx_resource:query_failed(AfterQuery); - _ -> - emqx_resource:query_success(AfterQuery) - end, - Result. + emqx_resource:query_failed(AfterQuery), + {error, Reason}; + {ok, Cursor} when is_pid(Cursor) -> + emqx_resource:query_success(AfterQuery), + mc_cursor:foldl(fun(O, Acc2) -> [O|Acc2] end, [], Cursor, 1000); + Result -> + emqx_resource:query_success(AfterQuery), + Result + end. -dialyzer({nowarn_function, [on_health_check/2]}). -on_health_check(_InstId, #{test_opts := TestOpts}) -> +on_health_check(_InstId, #{test_opts := TestOpts} = State) -> case mc_worker_api:connect(TestOpts) of {ok, TestConn} -> mc_worker_api:disconnect(TestConn), - {ok, true}; + {ok, State}; {error, _} -> - {ok, false} + {error, health_check_failed, State} end. %% =================================================================== @@ -102,17 +149,48 @@ connect(Opts) -> WorkerOptions = proplists:get_value(worker_options, Opts, []), mongo_api:connect(Type, Hosts, Options, WorkerOptions). -mongo_query(Conn, find, Collection, Selector, Docs) -> - mongo_api:find(Conn, Collection, Selector, Docs); +mongo_query(Conn, find, Collection, Selector, Projector) -> + mongo_api:find(Conn, Collection, Selector, Projector); + +mongo_query(Conn, find_one, Collection, Selector, Projector) -> + mongo_api:find_one(Conn, Collection, Selector, Projector); %% Todo xxx -mongo_query(_Conn, _Action, _Collection, _Selector, _Docs) -> +mongo_query(_Conn, _Action, _Collection, _Selector, _Projector) -> ok. -init_type(rs, #{rs_set_name := Name}) -> - {rs, Name}; -init_type(Type, _Opts) -> - Type. +do_start(InstId, Opts0, Config = #{mongo_type := Type, + database := Database, + pool_size := PoolSize, + ssl := SSL}) -> + SslOpts = case maps:get(enable, SSL) of + true -> + [{ssl, true}, + {ssl_opts, emqx_plugin_libs_ssl:save_files_return_opts(SSL, "connectors", InstId)} + ]; + false -> [{ssl, false}] + end, + Opts = Opts0 ++ + [{pool_size, PoolSize}, + {options, init_topology_options(maps:to_list(Config), [])}, + {worker_options, init_worker_options(maps:to_list(Config), SslOpts)}], + %% test the connection + TestOpts = case maps:is_key(server, Config) of + true -> + Server = maps:get(server, Config), + host_port(Server); + false -> + Servers = maps:get(servers, Config), + host_port(erlang:hd(Servers)) + end ++ [{database, Database}], + {ok, TestConn} = mc_worker_api:connect(TestOpts), + + PoolName = emqx_plugin_libs_pool:pool_name(InstId), + _ = emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts ++ SslOpts), + {ok, #{poolname => PoolName, + type => Type, + test_conn => TestConn, + test_opts => TestOpts}}. init_topology_options([{pool_size, Val}| R], Acc) -> init_topology_options(R, [{pool_size, Val}| Acc]); @@ -145,7 +223,7 @@ init_worker_options([{database, V} | R], Acc) -> init_worker_options(R, [{database, V} | Acc]); init_worker_options([{auth_source, V} | R], Acc) -> init_worker_options(R, [{auth_source, V} | Acc]); -init_worker_options([{login, V} | R], Acc) -> +init_worker_options([{username, V} | R], Acc) -> init_worker_options(R, [{login, V} | Acc]); init_worker_options([{password, V} | R], Acc) -> init_worker_options(R, [{password, V} | Acc]); @@ -157,13 +235,6 @@ init_worker_options([_ | R], Acc) -> init_worker_options(R, Acc); init_worker_options([], Acc) -> Acc. -init_ssl_opts(#{ssl := true} = Config, InstId) -> - [{ssl, true}, - {ssl_opts, emqx_plugin_libs_ssl:save_files_return_opts(Config, "connectors", InstId)} - ]; -init_ssl_opts(_Config, _InstId) -> - [{ssl, false}]. - host_port(HostPort) -> case string:split(HostPort, ":") of [Host, Port] -> @@ -174,43 +245,14 @@ host_port(HostPort) -> [{host, Host1}] end. -mongodb_fields() -> - [ {mongo_type, fun mongo_type/1} - , {servers, fun servers/1} - , {pool_size, fun emqx_connector_schema_lib:pool_size/1} - , {login, fun emqx_connector_schema_lib:username/1} - , {password, fun emqx_connector_schema_lib:password/1} - , {auth_source, fun auth_source/1} - , {database, fun emqx_connector_schema_lib:database/1} - ]. +server(type) -> server(); +server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; +server(_) -> undefined. -mongodb_topology_fields() -> - [ {max_overflow, fun emqx_connector_schema_lib:pool_size/1} - , {overflow_ttl, fun duration/1} - , {overflow_check_period, fun duration/1} - , {local_threshold_ms, fun duration/1} - , {connect_timeout_ms, fun duration/1} - , {socket_timeout_ms, fun duration/1} - , {server_selection_timeout_ms, fun duration/1} - , {wait_queue_timeout_ms, fun duration/1} - , {heartbeat_frequency_ms, fun duration/1} - , {min_heartbeat_frequency_ms, fun duration/1} - ]. - -mongodb_rs_set_name_fields() -> - [ {rs_set_name, fun emqx_connector_schema_lib:database/1} - ]. - -auth_source(type) -> binary(); -auth_source(_) -> undefined. - -servers(type) -> binary(); -servers(validator) -> [?REQUIRED("the field 'servers' is required")]; +servers(type) -> hoconsc:array(server()); +servers(validator) -> [?NOT_EMPTY("the value of the field 'servers' cannot be empty")]; servers(_) -> undefined. -mongo_type(type) -> hoconsc:enum([single, unknown, shared, rs]); -mongo_type(default) -> single; -mongo_type(_) -> undefined. - duration(type) -> emqx_schema:duration_ms(); +duration(nullable) -> true; duration(_) -> undefined. diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index afe249682..6a5d93ca2 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -37,6 +37,9 @@ structs() -> [""]. fields("") -> + [{config, #{type => hoconsc:ref(?MODULE, config)}}]; + +fields(config) -> emqx_connector_schema_lib:relational_db_fields() ++ emqx_connector_schema_lib:ssl_fields(). @@ -51,14 +54,14 @@ on_start(InstId, #{server := {Host, Port}, username := User, password := Password, auto_reconnect := AutoReconn, - pool_size := PoolSize} = Config) -> + pool_size := PoolSize, + ssl := SSL } = Config) -> logger:info("starting mysql connector: ~p, config: ~p", [InstId, Config]), - SslOpts = case maps:get(ssl, Config) of + SslOpts = case maps:get(enable, SSL) of true -> [{ssl, [{server_name_indication, disable} | - emqx_plugin_libs_ssl:save_files_return_opts(Config, "connectors", InstId)]}]; - false -> - [] + emqx_plugin_libs_ssl:save_files_return_opts(SSL, "connectors", InstId)]}]; + false -> [] end, Options = [{host, Host}, {port, Port}, diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 827a9606c..e89ab7401 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -38,6 +38,9 @@ structs() -> [""]. fields("") -> + [{config, #{type => hoconsc:ref(?MODULE, config)}}]; + +fields(config) -> emqx_connector_schema_lib:relational_db_fields() ++ emqx_connector_schema_lib:ssl_fields(). @@ -50,14 +53,14 @@ on_start(InstId, #{server := {Host, Port}, username := User, password := Password, auto_reconnect := AutoReconn, - pool_size := PoolSize} = Config) -> + pool_size := PoolSize, + ssl := SSL } = Config) -> logger:info("starting postgresql connector: ~p, config: ~p", [InstId, Config]), - SslOpts = case maps:get(ssl, Config) of + SslOpts = case maps:get(enable, SSL) of true -> - [{ssl_opts, [{server_name_indication, disable} | - emqx_plugin_libs_ssl:save_files_return_opts(Config, "connectors", InstId)]}]; - false -> - [] + [{ssl, [{server_name_indication, disable} | + emqx_plugin_libs_ssl:save_files_return_opts(SSL, "connectors", InstId)]}]; + false -> [] end, Options = [{host, Host}, {port, Port}, diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 1aa6263b6..1ea31ced8 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -45,9 +45,9 @@ structs() -> [""]. fields("") -> [ {config, #{type => hoconsc:union( - [ hoconsc:ref(cluster) - , hoconsc:ref(single) - , hoconsc:ref(sentinel) + [ hoconsc:ref(?MODULE, cluster) + , hoconsc:ref(?MODULE, single) + , hoconsc:ref(?MODULE, sentinel) ])} } ]; @@ -78,10 +78,11 @@ on_jsonify(Config) -> Config. %% =================================================================== -on_start(InstId, #{config :=#{redis_type := Type, - database := Database, - pool_size := PoolSize, - auto_reconnect := AutoReconn} = Config}) -> +on_start(InstId, #{redis_type := Type, + database := Database, + pool_size := PoolSize, + auto_reconnect := AutoReconn, + ssl := SSL } = Config) -> logger:info("starting redis connector: ~p, config: ~p", [InstId, Config]), Servers = case Type of single -> [{servers, [maps:get(server, Config)]}]; @@ -92,8 +93,13 @@ on_start(InstId, #{config :=#{redis_type := Type, {password, maps:get(password, Config, "")}, {auto_reconnect, reconn_interval(AutoReconn)} ] ++ Servers, - Options = init_ssl_opts(Config, InstId) ++ - [{sentinel, maps:get(sentinel, Config, undefined)}], + Options = case maps:get(enable, SSL) of + true -> + [{ssl, true}, + {ssl_options, emqx_plugin_libs_ssl:save_files_return_opts(SSL, "connectors", InstId)} + ]; + false -> [{ssl, false}] + end ++ [{sentinel, maps:get(sentinel, Config, undefined)}], PoolName = emqx_plugin_libs_pool:pool_name(InstId), case Type of cluster -> @@ -134,7 +140,7 @@ on_health_check(_InstId, #{type := cluster, poolname := PoolName} = State) -> eredis_cluster_pool_worker:is_connected(Pid) =:= true end, Workers) of true -> {ok, State}; - false -> {error, test_query_failed, State} + false -> {error, health_check_failed, State} end; on_health_check(_InstId, #{poolname := PoolName} = State) -> emqx_plugin_libs_pool:health_check(PoolName, fun ?MODULE:do_health_check/1, State). @@ -157,13 +163,6 @@ cmd(Conn, _Type, Command) -> connect(Opts) -> eredis:start_link(Opts). -init_ssl_opts(#{ssl := true} = Config, InstId) -> - [{ssl, true}, - {ssl_opts, emqx_plugin_libs_ssl:save_files_return_opts(Config, "connectors", InstId)} - ]; -init_ssl_opts(_Config, _InstId) -> - [{ssl, false}]. - redis_fields() -> [ {pool_size, fun emqx_connector_schema_lib:pool_size/1} , {password, fun emqx_connector_schema_lib:password/1} diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl index 03572d91d..d0b314077 100644 --- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -51,6 +51,31 @@ , servers/0 ]). +-export([structs/0, fields/1]). + +structs() -> [ssl_on, ssl_off]. + +fields(ssl_on) -> + [ {enable, #{type => true}} + , {cacertfile, fun cacertfile/1} + , {keyfile, fun keyfile/1} + , {certfile, fun certfile/1} + , {verify, fun verify/1} + ]; + +fields(ssl_off) -> + [ {enable, #{type => false}} ]. + +ssl_fields() -> + [ {ssl, #{type => hoconsc:union( + [ hoconsc:ref(?MODULE, ssl_on) + , hoconsc:ref(?MODULE, ssl_off) + ]), + default => #{<<"enable">> => false} + } + } + ]. + relational_db_fields() -> [ {server, fun server/1} , {database, fun database/1} @@ -60,20 +85,14 @@ relational_db_fields() -> , {auto_reconnect, fun auto_reconnect/1} ]. -ssl_fields() -> - [ {ssl, fun ssl/1} - , {cacertfile, fun cacertfile/1} - , {keyfile, fun keyfile/1} - , {certfile, fun certfile/1} - , {verify, fun verify/1} - ]. - server(type) -> emqx_schema:ip_port(); -server(validator) -> [?REQUIRED("the field 'server' is required")]; +server(nullable) -> false; +server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; server(_) -> undefined. database(type) -> binary(); -database(validator) -> [?REQUIRED("the field 'database' is required")]; +database(nullable) -> false; +database(validator) -> [?NOT_EMPTY("the value of the field 'database' cannot be empty")]; database(_) -> undefined. pool_size(type) -> integer(); @@ -82,30 +101,26 @@ pool_size(validator) -> [?MIN(1), ?MAX(64)]; pool_size(_) -> undefined. username(type) -> binary(); -username(default) -> "root"; +username(nullable) -> true; username(_) -> undefined. password(type) -> binary(); -password(default) -> ""; +password(nullable) -> true; password(_) -> undefined. auto_reconnect(type) -> boolean(); auto_reconnect(default) -> true; auto_reconnect(_) -> undefined. -ssl(type) -> boolean(); -ssl(default) -> false; -ssl(_) -> undefined. - -cacertfile(type) -> binary(); +cacertfile(type) -> string(); cacertfile(default) -> ""; cacertfile(_) -> undefined. -keyfile(type) -> binary(); +keyfile(type) -> string(); keyfile(default) -> ""; keyfile(_) -> undefined. -certfile(type) -> binary(); +certfile(type) -> string(); certfile(default) -> ""; certfile(_) -> undefined. @@ -114,7 +129,7 @@ verify(default) -> false; verify(_) -> undefined. servers(type) -> servers(); -servers(validator) -> [?REQUIRED("the field 'servers' is required")]; +servers(validator) -> [?NOT_EMPTY("the value of the field 'servers' cannot be empty")]; servers(_) -> undefined. to_ip_port(Str) -> diff --git a/apps/emqx_dashboard/README.md b/apps/emqx_dashboard/README.md index e9e50a7c9..c86765c51 100644 --- a/apps/emqx_dashboard/README.md +++ b/apps/emqx_dashboard/README.md @@ -26,7 +26,7 @@ GET | /nodes/:node/sessions/ | A list of sessions on a node GET | /subscriptions/:clientid | A list of subscriptions of a client GET | /nodes/:node/subscriptions/:clientid | A list of subscriptions of a client on the node GET | /nodes/:node/subscriptions/ | A list of subscriptions on a node -PUT | /clients/:clientid/clean_acl_cache | Clean ACL cache of a client +PUT | /clients/:clientid/clean_authz_cache | Clean Authorization cache of a client GET | /configs/ | Get all configs GET | /nodes/:node/configs/ | Get all configs of a node GET | /nodes/:node/plugin_configs/:plugin | Get configurations of a plugin on the node diff --git a/apps/emqx_dashboard/etc/emqx_dashboard.conf b/apps/emqx_dashboard/etc/emqx_dashboard.conf index 4f28ec84e..de453c12c 100644 --- a/apps/emqx_dashboard/etc/emqx_dashboard.conf +++ b/apps/emqx_dashboard/etc/emqx_dashboard.conf @@ -2,129 +2,40 @@ ## EMQ X Dashboard ##-------------------------------------------------------------------- -## Default user's login name. -## -## Value: String -dashboard.default_user.login = admin - -## Default user's password. -## -## Value: String -dashboard.default_user.password = public - -##-------------------------------------------------------------------- -## HTTP Listener - -## The port that the Dashboard HTTP listener will bind. -## -## Value: Port -## -## Examples: 18083 -dashboard.listener.http.port = 18083 - -## The acceptor pool for external Dashboard HTTP listener. -## -## Value: Number -dashboard.listener.http.acceptors = 4 - -## Maximum number of concurrent Dashboard HTTP connections. -## -## Value: Number -dashboard.listener.http.max_clients = 512 - -## Set up the socket for IPv6. -## -## Value: false | true -dashboard.listener.http.inet6 = false - -## Listen on IPv4 and IPv6 (false) or only on IPv6 (true). Use with inet6. -## -## Value: false | true -dashboard.listener.http.ipv6_v6only = false - -##-------------------------------------------------------------------- -## HTTPS Listener - -## The port that the Dashboard HTTPS listener will bind. -## -## Value: Port -## -## Examples: 18084 -## dashboard.listener.https.port = 18084 - -## The acceptor pool for external Dashboard HTTPS listener. -## -## Value: Number -## dashboard.listener.https.acceptors = 2 - -## Maximum number of concurrent Dashboard HTTPS connections. -## -## Value: Number -## dashboard.listener.https.max_clients = 512 - -## Set up the socket for IPv6. -## -## Value: false | true -## dashboard.listener.https.inet6 = false - -## Listen on IPv4 and IPv6 (false) or only on IPv6 (true). Use with inet6. -## -## Value: false | true -## dashboard.listener.https.ipv6_v6only = false - -## Path to the file containing the user's private PEM-encoded key. -## -## Value: File -## dashboard.listener.https.keyfile = "etc/certs/key.pem" - -## Path to a file containing the user certificate. -## -## Value: File -## dashboard.listener.https.certfile = "etc/certs/cert.pem" - -## Path to the file containing PEM-encoded CA certificates. -## -## Value: File -## dashboard.listener.https.cacertfile = "etc/certs/cacert.pem" - -## See: 'listener.ssl..dhfile' in emq.conf -## -## Value: File -## dashboard.listener.https.dhfile = "{{ platform_etc_dir }}/certs/dh-params.pem" - -## See: 'listener.ssl..verify' in emq.conf -## -## Value: verify_peer | verify_none -## dashboard.listener.https.verify = verify_peer - -## See: 'listener.ssl..fail_if_no_peer_cert' in emq.conf -## -## Value: false | true -## dashboard.listener.https.fail_if_no_peer_cert = true - -## TLS versions only to protect from POODLE attack. -## -## Value: String, seperated by ',' -## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier -## dashboard.listener.https.tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" - -## See: 'listener.ssl..ciphers' in emq.conf -## -## Value: Ciphers -## dashboard.listener.https.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" - -## See: 'listener.ssl..secure_renegotiate' in emq.conf -## -## Value: on | off -## dashboard.listener.https.secure_renegotiate = off - -## See: 'listener.ssl..reuse_sessions' in emq.conf -## -## Value: on | off -## dashboard.listener.https.reuse_sessions = on - -## See: 'listener.ssl..honor_cipher_order' in emq.conf -## -## Value: on | off -## dashboard.listener.https.honor_cipher_order = on - +emqx_dashboard:{ + default_username: "admin" + default_password: "public" + listeners: [ + { + num_acceptors: 4 + max_connections: 512 + protocol: http + port: 18083 + backlog: 512 + send_timeout: 15s + send_timeout_close: true + inet6: false + ipv6_v6only: false + } +## , +## { +## protocol: https +## port: 18084 +## acceptors: 2 +## backlog: 512 +## send_timeout: 15s +## send_timeout_close: true +## inet6: false +## ipv6_v6only: false +## certfile = "etc/certs/cert.pem" +## keyfile = "etc/certs/key.pem" +## cacertfile = "etc/certs/cacert.pem" +## verify = verify_peer +## tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" +## ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" +## fail_if_no_peer_cert = true +## inet6 = false +## ipv6_v6only = false +## } + ] +} diff --git a/apps/emqx_dashboard/include/emqx_dashboard.hrl b/apps/emqx_dashboard/include/emqx_dashboard.hrl index 891a723f7..cc3d9b3d6 100644 --- a/apps/emqx_dashboard/include/emqx_dashboard.hrl +++ b/apps/emqx_dashboard/include/emqx_dashboard.hrl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --record(mqtt_admin, {username, password, tags}). +-record(mqtt_admin, {username, password, tags, role = undefined}). -type(mqtt_admin() :: #mqtt_admin{}). diff --git a/apps/emqx_dashboard/priv/emqx_dashboard.schema b/apps/emqx_dashboard/priv/emqx_dashboard.schema deleted file mode 100644 index 203bb7358..000000000 --- a/apps/emqx_dashboard/priv/emqx_dashboard.schema +++ /dev/null @@ -1,152 +0,0 @@ -%%-*- mode: erlang -*- -%% emqx_dashboard config mapping - -{mapping, "dashboard.default_user.login", "emqx_dashboard.default_user_username", [ - {datatype, string} -]}. - -{mapping, "dashboard.default_user.password", "emqx_dashboard.default_user_passwd", [ - {datatype, string}, - {override_env, "EMQX_ADMIN_PASSWORD"} -]}. - -{mapping, "dashboard.listener.http.port", "emqx_dashboard.listeners", [ - {datatype, integer} -]}. - -{mapping, "dashboard.listener.http.acceptors", "emqx_dashboard.listeners", [ - {default, 4}, - {datatype, integer} -]}. - -{mapping, "dashboard.listener.http.max_clients", "emqx_dashboard.listeners", [ - {default, 512}, - {datatype, integer} -]}. - -{mapping, "dashboard.listener.http.access.$id", "emqx_dashboard.listeners", [ - {datatype, string} -]}. - -{mapping, "dashboard.listener.http.inet6", "emqx_dashboard.listeners", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "dashboard.listener.http.ipv6_v6only", "emqx_dashboard.listeners", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "dashboard.listener.https.port", "emqx_dashboard.listeners", [ - {datatype, integer} -]}. - -{mapping, "dashboard.listener.https.acceptors", "emqx_dashboard.listeners", [ - {default, 8}, - {datatype, integer} -]}. - -{mapping, "dashboard.listener.https.max_clients", "emqx_dashboard.listeners", [ - {default, 64}, - {datatype, integer} -]}. - -{mapping, "dashboard.listener.https.inet6", "emqx_dashboard.listeners", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "dashboard.listener.https.ipv6_v6only", "emqx_dashboard.listeners", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "dashboard.listener.https.tls_versions", "emqx_dashboard.listeners", [ - {datatype, string} -]}. - -{mapping, "dashboard.listener.https.dhfile", "emqx_dashboard.listeners", [ - {datatype, string} -]}. - -{mapping, "dashboard.listener.https.keyfile", "emqx_dashboard.listeners", [ - {datatype, string} -]}. - -{mapping, "dashboard.listener.https.certfile", "emqx_dashboard.listeners", [ - {datatype, string} -]}. - -{mapping, "dashboard.listener.https.cacertfile", "emqx_dashboard.listeners", [ - {datatype, string} -]}. - -{mapping, "dashboard.listener.https.verify", "emqx_dashboard.listeners", [ - {datatype, string} -]}. - -{mapping, "dashboard.listener.https.fail_if_no_peer_cert", "emqx_dashboard.listeners", [ - {datatype, {enum, [true, false]}} -]}. - -{mapping, "dashboard.listener.https.ciphers", "emqx_dashboard.listeners", [ - {datatype, string} -]}. - -{mapping, "dashboard.listener.https.secure_renegotiate", "emqx_dashboard.listeners", [ - {datatype, flag} -]}. - -{mapping, "dashboard.listener.https.reuse_sessions", "emqx_dashboard.listeners", [ - {default, on}, - {datatype, flag} -]}. - -{mapping, "dashboard.listener.https.honor_cipher_order", "emqx_dashboard.listeners", [ - {datatype, flag} -]}. - -{translation, "emqx_dashboard.listeners", fun(Conf) -> - Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, - LisOpts = fun(Prefix) -> - Filter([{num_acceptors, cuttlefish:conf_get(Prefix ++ ".acceptors", Conf)}, - {max_connections, cuttlefish:conf_get(Prefix ++ ".max_clients", Conf)}, - {inet6, cuttlefish:conf_get(Prefix ++ ".inet6", Conf)}, - {ipv6_v6only, cuttlefish:conf_get(Prefix ++ ".ipv6_v6only", Conf)}]) - end, - - SplitFun = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end, - - SslOpts = fun(Prefix) -> - Versions = case SplitFun(cuttlefish:conf_get(Prefix ++ ".tls_versions", Conf, undefined)) of - undefined -> undefined; - L -> [list_to_atom(V) || V <- L] - end, - Filter([{versions, Versions}, - {ciphers, SplitFun(cuttlefish:conf_get(Prefix ++ ".ciphers", Conf, undefined))}, - {dhfile, cuttlefish:conf_get(Prefix ++ ".dhfile", Conf, undefined)}, - {keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, - {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, - {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)}, - {verify, cuttlefish:conf_get(Prefix ++ ".verify", Conf, undefined)}, - {fail_if_no_peer_cert, cuttlefish:conf_get(Prefix ++ ".fail_if_no_peer_cert", Conf, undefined)}, - {secure_renegotiate, cuttlefish:conf_get(Prefix ++ ".secure_renegotiate", Conf, undefined)}, - {reuse_sessions, cuttlefish:conf_get(Prefix ++ ".reuse_sessions", Conf, undefined)}, - {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)}]) - end, - lists:append( - lists:map( - fun(Proto) -> - Prefix = "dashboard.listener." ++ atom_to_list(Proto), - case cuttlefish:conf_get(Prefix ++ ".port", Conf, undefined) of - undefined -> []; - Port -> - [{Proto, Port, case Proto of - http -> LisOpts(Prefix); - https -> LisOpts(Prefix) ++ SslOpts(Prefix) - end}] - end - end, [http, https])) -end}. - diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index 1604198dc..788677a33 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.4.0"}, % strict semver, bump manually! + {vsn, "5.0.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard.appup.src b/apps/emqx_dashboard/src/emqx_dashboard.appup.src deleted file mode 100644 index 678bd3b22..000000000 --- a/apps/emqx_dashboard/src/emqx_dashboard.appup.src +++ /dev/null @@ -1,16 +0,0 @@ -%% -*- mode: erlang -*- -{VSN, - [ {"4.3.0", - %% load all plugins - %% NOTE: this depends on the fact that emqx_dashboard is always - %% the last application gets upgraded - [ {apply, {emqx_plugins, load, []}} - ]}, - {<<".*">>, []} - ], - [ {"4.3.0", - [ {apply, {emqx_plugins, load, []}} - ]}, - {<<".*">>, []} - ] -}. diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 0390339d3..9eb1b06ac 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -16,109 +16,112 @@ -module(emqx_dashboard). --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/logger.hrl"). +-define(APP, ?MODULE). --import(proplists, [get_value/3]). -export([ start_listeners/0 , stop_listeners/0 , start_listener/1 - , stop_listener/1 - ]). + , stop_listener/1]). -%% for minirest --export([ filter/1 - , is_authorized/1 - ]). +%% Authorization +-export([authorize_appid/1]). --define(APP, ?MODULE). + +-define(BASE_PATH, "/api/v5"). %%-------------------------------------------------------------------- -%% Start/Stop listeners. +%% Start/Stop Listeners %%-------------------------------------------------------------------- start_listeners() -> - lists:foreach(fun(Listener) -> start_listener(Listener) end, listeners()). - -%% Start HTTP Listener -start_listener({Proto, Port, Options}) when Proto == http -> - Dispatch = [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, - {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, - {"/api/v4/[...]", minirest, http_handlers()}], - minirest:start_http(listener_name(Proto), ranch_opts(Port, Options), Dispatch); - -start_listener({Proto, Port, Options}) when Proto == https -> - Dispatch = [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, - {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, - {"/api/v4/[...]", minirest, http_handlers()}], - minirest:start_https(listener_name(Proto), ranch_opts(Port, Options), Dispatch). - -ranch_opts(Port, Options0) -> - NumAcceptors = get_value(num_acceptors, Options0, 4), - MaxConnections = get_value(max_connections, Options0, 512), - Options = lists:foldl(fun({K, _V}, Acc) when K =:= max_connections orelse K =:= num_acceptors -> - Acc; - ({inet6, true}, Acc) -> [inet6 | Acc]; - ({inet6, false}, Acc) -> Acc; - ({ipv6_v6only, true}, Acc) -> [{ipv6_v6only, true} | Acc]; - ({ipv6_v6only, false}, Acc) -> Acc; - ({K, V}, Acc)-> - [{K, V} | Acc] - end, [], Options0), - #{num_acceptors => NumAcceptors, - max_connections => MaxConnections, - socket_opts => [{port, Port} | Options]}. + lists:foreach(fun start_listener/1, listeners()). stop_listeners() -> - lists:foreach(fun(Listener) -> stop_listener(Listener) end, listeners()). + lists:foreach(fun stop_listener/1, listeners()). -stop_listener({Proto, _Port, _}) -> - minirest:stop_http(listener_name(Proto)). +start_listener({Proto, Port, Options}) -> + {ok, _} = application:ensure_all_started(minirest), + Authorization = {?MODULE, authorize_appid}, + RanchOptions = ranch_opts(Port, Options), + GlobalSpec = #{ + openapi => "3.0.0", + info => #{title => "EMQ X Dashboard API", version => "5.0.0"}, + servers => [#{url => ?BASE_PATH}], + components => #{ + schemas => #{}, + securitySchemes => #{ + application => #{ + type => apiKey, + name => "authorization", + in => header}}}}, + Dispatch = [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, + {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}], + Minirest = #{ + protocol => Proto, + base_path => ?BASE_PATH, + modules => minirest_api:find_api_modules(apps()), + authorization => Authorization, + security => [#{application => []}], + swagger_global_spec => GlobalSpec, + dispatch => Dispatch}, + MinirestOptions = maps:merge(Minirest, RanchOptions), + {ok, _} = minirest:start(listener_name(Proto), MinirestOptions), + io:format("Start ~p listener on ~p successfully.~n", [listener_name(Proto), Port]). + +apps() -> + [App || {App, _, _} <- application:loaded_applications(), + case re:run(atom_to_list(App), "^emqx") of + {match,[{0,4}]} -> true; + _ -> false + end]. + +ranch_opts(Port, Options0) -> + Options = lists:foldl( + fun + ({K, _V}, Acc) when K =:= max_connections orelse K =:= num_acceptors -> Acc; + ({inet6, true}, Acc) -> [inet6 | Acc]; + ({inet6, false}, Acc) -> Acc; + ({ipv6_v6only, true}, Acc) -> [{ipv6_v6only, true} | Acc]; + ({ipv6_v6only, false}, Acc) -> Acc; + ({K, V}, Acc)-> + [{K, V} | Acc] + end, [], Options0), + maps:from_list([{port, Port} | Options]). + +stop_listener({Proto, Port, _}) -> + io:format("Stop dashboard listener on ~s successfully.~n",[format(Port)]), + minirest:stop(listener_name(Proto)). listeners() -> - application:get_env(?APP, listeners, []). + [{Protocol, Port, maps:to_list(maps:without([protocol, port], Map))} + || Map = #{protocol := Protocol,port := Port} + <- emqx_config:get([emqx_dashboard, listeners], [])]. listener_name(Proto) -> list_to_atom(atom_to_list(Proto) ++ ":dashboard"). -%%-------------------------------------------------------------------- -%% HTTP Handlers and Dispatcher -%%-------------------------------------------------------------------- - -http_handlers() -> - Plugins = lists:map(fun(Plugin) -> Plugin#plugin.name end, emqx_plugins:list()), - [{"/api/v4/", - minirest:handler(#{apps => Plugins ++ [emqx_modules], - filter => fun ?MODULE:filter/1}), - [{authorization, fun ?MODULE:is_authorized/1}]}]. - -%%-------------------------------------------------------------------- -%% Basic Authorization -%%-------------------------------------------------------------------- - -is_authorized(Req) -> - is_authorized(binary_to_list(cowboy_req:path(Req)), Req). - -is_authorized("/api/v4/auth", _Req) -> - true; -is_authorized(_Path, Req) -> +authorize_appid(Req) -> case cowboy_req:parse_header(<<"authorization">>, Req) of {basic, Username, Password} -> case emqx_dashboard_admin:check(iolist_to_binary(Username), iolist_to_binary(Password)) of - ok -> true; - {error, Reason} -> - ?LOG(error, "[Dashboard] Authorization Failure: username=~s, reason=~p", - [Username, Reason]), - false + ok -> + ok; + {error, _} -> + {401, #{<<"WWW-Authenticate">> => + <<"Basic Realm=\"minirest-server\"">>}, + <<"UNAUTHORIZED">>} end; - _ -> false + _ -> + {401, #{<<"WWW-Authenticate">> => + <<"Basic Realm=\"minirest-server\"">>}, + <<"UNAUTHORIZED">>} end. -filter(#{app := emqx_modules}) -> true; -filter(#{app := App}) -> - case emqx_plugins:find_plugin(App) of - false -> false; - Plugin -> Plugin#plugin.active - end. +format(Port) when is_integer(Port) -> + io_lib:format("0.0.0.0:~w", [Port]); +format({Addr, Port}) when is_list(Addr) -> + io_lib:format("~s:~w", [Addr, Port]); +format({Addr, Port}) when is_tuple(Addr) -> + io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl index 202914982..fdec41b2b 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -182,7 +182,7 @@ check(Username, Password) -> init([]) -> %% Add default admin user - _ = add_default_user(binenv(default_user_username), binenv(default_user_passwd)), + _ = add_default_user(binenv(default_username), binenv(default_password)), {ok, state}. handle_call(_Req, _From, State) -> @@ -217,7 +217,7 @@ salt() -> <>. binenv(Key) -> - iolist_to_binary(application:get_env(emqx_dashboard, Key, "")). + iolist_to_binary(emqx_config:get([emqx_dashboard, Key], "")). add_default_user(Username, Password) when ?EMPTY_KEY(Username) orelse ?EMPTY_KEY(Password) -> igonre; diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index e1c89efbb..f11d6588b 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -16,94 +16,226 @@ -module(emqx_dashboard_api). +-behaviour(minirest_api). + -include("emqx_dashboard.hrl"). --import(minirest, [return/1]). +-import(emqx_mgmt_util, [ response_schema/1 + , response_schema/2 + , request_body_schema/1 + , response_array_schema/2 + ]). --rest_api(#{name => auth_user, - method => 'POST', - path => "/auth", - func => auth, - descr => "Authenticate an user" - }). +-export([api_spec/0]). --rest_api(#{name => create_user, - method => 'POST', - path => "/users/", - func => create, - descr => "Create an user" - }). - --rest_api(#{name => list_users, - method => 'GET', - path => "/users/", - func => list, - descr => "List users" - }). - --rest_api(#{name => update_user, - method => 'PUT', - path => "/users/:bin:name", - func => update, - descr => "Update an user" - }). - --rest_api(#{name => delete_user, - method => 'DELETE', - path => "/users/:bin:name", - func => delete, - descr => "Delete an user" - }). - --rest_api(#{name => change_pwd, - method => 'PUT', - path => "/change_pwd/:bin:username", - func => change_pwd, - descr => "Change password for an user" - }). - --export([ list/2 - , create/2 - , update/2 - , delete/2 - , auth/2 +-export([ auth/2 + , users/2 + , user/2 , change_pwd/2 ]). +api_spec() -> + {[auth_api(), users_api(), user_api(), change_pwd_api()], schemas()}. + +schemas() -> + [#{auth => #{ + type => object, + properties => #{ + username => #{ + type => string, + description => <<"Username">>}, + password => #{ + type => string, + description => <<"password">>} + } + }}, + #{show_user => #{ + type => object, + properties => #{ + username => #{ + type => string, + description => <<"Username">>}, + tag => #{ + type => string, + description => <<"Tag">>} + } + }}, + #{create_user => #{ + type => object, + properties => #{ + username => #{ + type => string, + description => <<"Username">>}, + password => #{ + type => string, + description => <<"Password">>}, + tag => #{ + type => string, + description => <<"Tag">>} + } + }}]. + +auth_api() -> + Metadata = #{ + post => #{ + description => <<"Dashboard Auth">>, + 'requestBody' => request_body_schema(auth), + responses => #{ + <<"200">> => + response_schema(<<"Dashboard Auth successfully">>), + <<"400">> => bad_request() + }, + security => [] + } + }, + {"/auth", Metadata, auth}. + +users_api() -> + Metadata = #{ + get => #{ + description => <<"Get dashboard users">>, + responses => #{ + <<"200">> => response_array_schema(<<"">>, show_user) + } + } + }, + {"/users", Metadata, users}. + +user_api() -> + Metadata = #{ + delete => #{ + description => <<"Delete dashboard users">>, + responses => #{ + <<"200">> => response_schema(<<"Delete User successfully">>), + <<"400">> => bad_request() + } + }, + put => #{ + description => <<"Update dashboard users">>, + 'requestBody' => request_body_schema(#{ + type => object, + properties => #{ + tags => #{ + type => string + } + } + }), + responses => #{ + <<"200">> => response_schema(<<"Update Users successfully">>), + <<"400">> => bad_request() + } + }, + post => #{ + description => <<"Create dashboard users">>, + 'requestBody' => request_body_schema(create_user), + responses => #{ + <<"200">> => response_schema(<<"Create Users successfully">>), + <<"400">> => bad_request() + } + } + }, + {"/users/:username", Metadata, user}. + +change_pwd_api() -> + Metadata = #{ + put => #{ + description => <<"Update dashboard users password">>, + 'requestBody' => request_body_schema(#{ + type => object, + properties => #{ + old_pwd => #{ + type => string + }, + new_pwd => #{ + type => string + } + } + }), + responses => #{ + <<"200">> => response_schema(<<"Update Users password successfully">>), + <<"400">> => bad_request() + } + } + }, + {"/change_pwd/:username", Metadata, change_pwd}. + -define(EMPTY(V), (V == undefined orelse V == <<>>)). -auth(_Bindings, Params) -> - Username = proplists:get_value(<<"username">>, Params), - Password = proplists:get_value(<<"password">>, Params), - return(emqx_dashboard_admin:check(Username, Password)). +auth(post, Request) -> + {ok, Body, _} = cowboy_req:read_body(Request), + Params = emqx_json:decode(Body, [return_maps]), + Username = maps:get(<<"username">>, Params), + Password = maps:get(<<"password">>, Params), + case emqx_dashboard_admin:check(Username, Password) of + ok -> + {200}; + {error, Reason} -> + {400, #{code => <<"AUTH_FAIL">>, message => Reason}} + end. -change_pwd(#{username := Username}, Params) -> - OldPwd = proplists:get_value(<<"old_pwd">>, Params), - NewPwd = proplists:get_value(<<"new_pwd">>, Params), - return(emqx_dashboard_admin:change_password(Username, OldPwd, NewPwd)). +users(get, _Request) -> + {200, [row(User) || User <- emqx_dashboard_admin:all_users()]}. -create(_Bindings, Params) -> - Username = proplists:get_value(<<"username">>, Params), - Password = proplists:get_value(<<"password">>, Params), - Tags = proplists:get_value(<<"tags">>, Params), - return(case ?EMPTY(Username) orelse ?EMPTY(Password) of - true -> {error, <<"Username or password undefined">>}; - false -> emqx_dashboard_admin:add_user(Username, Password, Tags) - end). +user(put, Request) -> + Username = cowboy_req:binding(username, Request), + {ok, Body, _} = cowboy_req:read_body(Request), + Params = emqx_json:decode(Body, [return_maps]), + Tags = maps:get(<<"tags">>, Params), + case emqx_dashboard_admin:update_user(Username, Tags) of + ok -> {200}; + {error, Reason} -> + {400, #{code => <<"UPDATE_FAIL">>, message => Reason}} + end; -list(_Bindings, _Params) -> - return({ok, [row(User) || User <- emqx_dashboard_admin:all_users()]}). +user(delete, Request) -> + Username = cowboy_req:binding(username, Request), + case Username == <<"admin">> of + true -> {400, #{code => <<"CONNOT_DELETE_ADMIN">>, + message => <<"Cannot delete admin">>}}; + false -> + _ = emqx_dashboard_admin:remove_user(Username), + {200} + end; -update(#{name := Username}, Params) -> - Tags = proplists:get_value(<<"tags">>, Params), - return(emqx_dashboard_admin:update_user(Username, Tags)). +user(post, Request) -> + {ok, Body, _} = cowboy_req:read_body(Request), + Params = emqx_json:decode(Body, [return_maps]), + Tags = maps:get(<<"tags">>, Params), + Username = maps:get(<<"username">>, Params), + Password = maps:get(<<"password">>, Params), + case ?EMPTY(Username) orelse ?EMPTY(Password) of + true -> + {400, #{code => <<"CREATE_USER_FAIL">>, + message => <<"Username or password undefined">>}}; + false -> + case emqx_dashboard_admin:add_user(Username, Password, Tags) of + ok -> {200}; + {error, Reason} -> + {400, #{code => <<"CREATE_USER_FAIL">>, message => Reason}} + end + end. -delete(#{name := <<"admin">>}, _Params) -> - return({error, <<"Cannot delete admin">>}); - -delete(#{name := Username}, _Params) -> - return(emqx_dashboard_admin:remove_user(Username)). +change_pwd(put, Request) -> + Username = cowboy_req:binding(username, Request), + {ok, Body, _} = cowboy_req:read_body(Request), + Params = emqx_json:decode(Body, [return_maps]), + OldPwd = maps:get(<<"old_pwd">>, Params), + NewPwd = maps:get(<<"new_pwd">>, Params), + case emqx_dashboard_admin:change_password(Username, OldPwd, NewPwd) of + ok -> {200}; + {error, Reason} -> + {400, #{code => <<"CHANGE_PWD_FAIL">>, message => Reason}} + end. row(#mqtt_admin{username = Username, tags = Tags}) -> #{username => Username, tags => Tags}. +bad_request() -> + response_schema(<<"Bad Request">>, + #{ + type => object, + properties => #{ + message => #{type => string}, + code => #{type => string} + } + }). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_app.erl b/apps/emqx_dashboard/src/emqx_dashboard_app.erl index c966ef5fe..54202d806 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_app.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_app.erl @@ -18,8 +18,6 @@ -behaviour(application). --emqx_plugin(?MODULE). - -export([ start/2 , stop/1 ]). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl new file mode 100644 index 000000000..45ad345fb --- /dev/null +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -0,0 +1,55 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_dashboard_schema). + +-include_lib("typerefl/include/types.hrl"). + +-export([ structs/0 + , fields/1]). + +structs() -> ["emqx_dashboard"]. + +fields("emqx_dashboard") -> + [ {listeners, hoconsc:array(hoconsc:union([hoconsc:ref(?MODULE, "http"), + hoconsc:ref(?MODULE, "https")]))} + , {default_username, fun default_username/1} + , {default_password, fun default_password/1} + ]; + +fields("http") -> + [ {"protocol", hoconsc:enum([http, https])} + , {"port", emqx_schema:t(integer(), undefined, 8081)} + , {"num_acceptors", emqx_schema:t(integer(), undefined, 4)} + , {"max_connections", emqx_schema:t(integer(), undefined, 512)} + , {"backlog", emqx_schema:t(integer(), undefined, 1024)} + , {"send_timeout", emqx_schema:t(emqx_schema:duration(), undefined, "15s")} + , {"send_timeout_close", emqx_schema:t(boolean(), undefined, true)} + , {"inet6", emqx_schema:t(boolean(), undefined, false)} + , {"ipv6_v6only", emqx_schema:t(boolean(), undefined, false)} + ]; + +fields("https") -> + emqx_schema:ssl(#{enable => true}) ++ fields("http"). + +default_username(type) -> string(); +default_username(default) -> "admin"; +default_username(nullable) -> false; +default_username(_) -> undefined. + +default_password(type) -> string(); +default_password(default) -> "public"; +default_password(nullable) -> false; +default_password(_) -> undefined. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index be77d474b..1ffb6786e 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -40,38 +40,39 @@ -define(OVERVIEWS, ['alarms/activated', 'alarms/deactivated', banned, brokers, stats, metrics, listeners, clients, subscriptions, routes, plugins]). all() -> - [{group, overview}, - {group, admins}, - {group, rest}, - {group, cli} - ]. - -groups() -> - [{overview, [sequence], [t_overview]}, - {admins, [sequence], [t_admins_add_delete]}, - {rest, [sequence], [t_rest_api]}, - {cli, [sequence], [t_cli]} - ]. +%% TODO: V5 API +%% emqx_ct:all(?MODULE). + []. init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_dashboard]), + emqx_ct_helpers:start_apps([emqx_management, emqx_dashboard],fun set_special_configs/1), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_dashboard, emqx_management, emqx_modules]), + emqx_ct_helpers:stop_apps([emqx_dashboard, emqx_management]), ekka_mnesia:ensure_stopped(). +set_special_configs(emqx_management) -> + emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + applications =>[#{id => "admin", secret => "public"}]}), + ok; +set_special_configs(_) -> + ok. + t_overview(_) -> + mnesia:clear_table(mqtt_admin), + emqx_dashboard_admin:add_user(<<"admin">>, <<"public">>, <<"tag">>), [?assert(request_dashboard(get, api_path(erlang:atom_to_list(Overview)), auth_header_()))|| Overview <- ?OVERVIEWS]. t_admins_add_delete(_) -> + mnesia:clear_table(mqtt_admin), ok = emqx_dashboard_admin:add_user(<<"username">>, <<"password">>, <<"tag">>), ok = emqx_dashboard_admin:add_user(<<"username1">>, <<"password1">>, <<"tag1">>), Admins = emqx_dashboard_admin:all_users(), - ?assertEqual(3, length(Admins)), + ?assertEqual(2, length(Admins)), ok = emqx_dashboard_admin:remove_user(<<"username1">>), Users = emqx_dashboard_admin:all_users(), - ?assertEqual(2, length(Users)), + ?assertEqual(1, length(Users)), ok = emqx_dashboard_admin:change_password(<<"username">>, <<"password">>, <<"pwd">>), timer:sleep(10), ?assert(request_dashboard(get, api_path("brokers"), auth_header_("username", "pwd"))), @@ -80,6 +81,8 @@ t_admins_add_delete(_) -> ?assertNotEqual(true, request_dashboard(get, api_path("brokers"), auth_header_("username", "pwd"))). t_rest_api(_Config) -> + mnesia:clear_table(mqtt_admin), + emqx_dashboard_admin:add_user(<<"admin">>, <<"public">>, <<"administrator">>), {ok, Res0} = http_get("users"), ?assertEqual([#{<<"username">> => <<"admin">>, diff --git a/apps/emqx_data_bridge/etc/emqx_data_bridge.conf b/apps/emqx_data_bridge/etc/emqx_data_bridge.conf index 696cadf05..c299b97a1 100644 --- a/apps/emqx_data_bridge/etc/emqx_data_bridge.conf +++ b/apps/emqx_data_bridge/etc/emqx_data_bridge.conf @@ -2,125 +2,128 @@ ## EMQ X Bridge Plugin ##-------------------------------------------------------------------- -emqx_data_bridge.bridges: [ -# {name: "mysql_bridge_1" -# type: mysql -# config: { -# server: "192.168.0.172:3306" -# database: mqtt -# pool_size: 1 -# username: root -# password: public -# auto_reconnect: true -# ssl: false -# } -# } -# , {name: "pgsql_bridge_1" -# type: pgsql -# config: { -# server: "192.168.0.172:5432" -# database: mqtt -# pool_size: 1 -# username: root -# password: public -# auto_reconnect: true -# ssl: false -# } -# } -# , {name: "mongodb_bridge_single" -# type: mongo -# config: { -# servers: "192.168.0.172:27017" -# mongo_type: single -# pool_size: 1 -# login: root -# password: public -# auth_source: mqtt -# database: mqtt -# ssl: false -# } -# } -# ,{name: "mongodb_bridge_rs" -# type: mongo -# config: { -# servers: "127.0.0.1:27017" -# mongo_type: rs -# rs_set_name: rs_name -# pool_size: 1 -# login: root -# password: public -# auth_source: mqtt -# database: mqtt -# ssl: false -# } -# } -# ,{name: "mongodb_bridge_shared" -# type: mongo -# config: { -# servers: "127.0.0.1:27017" -# mongo_type: shared -# pool_size: 1 -# login: root -# password: public -# auth_source: mqtt -# database: mqtt -# ssl: false -# max_overflow: 1 -# overflow_ttl: -# overflow_check_period: 10s -# local_threshold_ms: 10s -# connect_timeout_ms: 10s -# socket_timeout_ms: 10s -# server_selection_timeout_ms: 10s -# wait_queue_timeout_ms: 10s -# heartbeat_frequency_ms: 10s -# min_heartbeat_frequency_ms: 10s -# } -# } -# , {name: "redis_bridge_single" -# type: redis -# config: { -# servers: "192.168.0.172:6379" -# redis_type: single -# pool_size: 1 -# database: 0 -# password: public -# auto_reconnect: true -# ssl: false -# } -# } -# ,{name: "redis_bridge_sentinel" -# type: redis -# config: { -# servers: "127.0.0.1:6379, 127.0.0.2:6379, 127.0.0.3:6379" -# redis_type: sentinel -# sentinel_name: mymaster -# pool_size: 1 -# database: 0 -# ssl: false -# } -# } -# ,{name: "redis_bridge_cluster" -# type: redis -# config: { -# servers: "127.0.0.1:6379, 127.0.0.2:6379, 127.0.0.3:6379" -# redis_type: cluster -# pool_size: 1 -# database: 0 -# password: "public" -# ssl: false -# } -# } -# , {name: "ldap_bridge_1" -# type: ldap -# config: { -# servers: "192.168.0.172" -# port: 389 -# bind_dn: "cn=root,dc=emqx,dc=io" -# bind_password: "public" -# timeout: 30s -# pool_size: 1 -# ssl: false -# } -# } -] +emqx_data_bridge:{ + bridges:[ + # {name: "mysql_bridge_1" + # type: mysql + # config: { + # server: "192.168.0.172:3306" + # database: mqtt + # pool_size: 1 + # username: root + # password: public + # auto_reconnect: true + # ssl: false + # } + # } + # , {name: "pgsql_bridge_1" + # type: pgsql + # config: { + # server: "192.168.0.172:5432" + # database: mqtt + # pool_size: 1 + # username: root + # password: public + # auto_reconnect: true + # ssl: false + # } + # } + # , {name: "mongodb_bridge_single" + # type: mongo + # config: { + # servers: "192.168.0.172:27017" + # mongo_type: single + # pool_size: 1 + # login: root + # password: public + # auth_source: mqtt + # database: mqtt + # ssl: false + # } + # } + # ,{name: "mongodb_bridge_rs" + # type: mongo + # config: { + # servers: "127.0.0.1:27017" + # mongo_type: rs + # rs_set_name: rs_name + # pool_size: 1 + # login: root + # password: public + # auth_source: mqtt + # database: mqtt + # ssl: false + # } + # } + # ,{name: "mongodb_bridge_shared" + # type: mongo + # config: { + # servers: "127.0.0.1:27017" + # mongo_type: shared + # pool_size: 1 + # login: root + # password: public + # auth_source: mqtt + # database: mqtt + # ssl: false + # max_overflow: 1 + # overflow_ttl: + # overflow_check_period: 10s + # local_threshold_ms: 10s + # connect_timeout_ms: 10s + # socket_timeout_ms: 10s + # server_selection_timeout_ms: 10s + # wait_queue_timeout_ms: 10s + # heartbeat_frequency_ms: 10s + # min_heartbeat_frequency_ms: 10s + # } + # } + # , {name: "redis_bridge_single" + # type: redis + # config: { + # servers: "192.168.0.172:6379" + # redis_type: single + # pool_size: 1 + # database: 0 + # password: public + # auto_reconnect: true + # ssl: false + # } + # } + # ,{name: "redis_bridge_sentinel" + # type: redis + # config: { + # servers: "127.0.0.1:6379, 127.0.0.2:6379, 127.0.0.3:6379" + # redis_type: sentinel + # sentinel_name: mymaster + # pool_size: 1 + # database: 0 + # ssl: false + # } + # } + # ,{name: "redis_bridge_cluster" + # type: redis + # config: { + # servers: "127.0.0.1:6379, 127.0.0.2:6379, 127.0.0.3:6379" + # redis_type: cluster + # pool_size: 1 + # database: 0 + # password: "public" + # ssl: false + # } + # } + # , {name: "ldap_bridge_1" + # type: ldap + # config: { + # servers: "192.168.0.172" + # port: 389 + # bind_dn: "cn=root,dc=emqx,dc=io" + # bind_password: "public" + # timeout: 30s + # pool_size: 1 + # ssl: false + # } + # } + + ] +} diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.erl b/apps/emqx_data_bridge/src/emqx_data_bridge.erl index e37b2f94b..9c27ff8d5 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.erl @@ -60,4 +60,4 @@ config_key_path() -> [emqx_data_bridge, bridges]. update_config(ConfigReq) -> - emqx_config:update_config(config_key_path(), ConfigReq). + emqx_config:update(config_key_path(), ConfigReq). diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_app.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_app.erl index 967791643..26128841b 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_app.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_app.erl @@ -19,7 +19,7 @@ -behaviour(emqx_config_handler). --export([start/2, stop/1, handle_update_config/2]). +-export([start/2, stop/1, pre_config_update/2]). start(_StartType, _StartArgs) -> {ok, Sup} = emqx_data_bridge_sup:start_link(), @@ -31,10 +31,13 @@ stop(_State) -> ok. %% internal functions -handle_update_config({update, Bridge = #{<<"name">> := Name}}, OldConf) -> +pre_config_update({update, Bridge = #{<<"name">> := Name}}, OldConf) -> [Bridge | remove_bridge(Name, OldConf)]; -handle_update_config({delete, Name}, OldConf) -> - remove_bridge(Name, OldConf). +pre_config_update({delete, Name}, OldConf) -> + remove_bridge(Name, OldConf); +pre_config_update(NewConf, _OldConf) when is_list(NewConf) -> + %% overwrite the entire config! + NewConf. remove_bridge(_Name, undefined) -> []; diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl index 1ba7f2fc5..066d72096 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl @@ -5,11 +5,6 @@ %%====================================================================================== %% Hocon Schema Definitions --define(BRIDGE_FIELDS(T), - [{name, hoconsc:t(typerefl:binary())}, - {type, hoconsc:t(typerefl:atom(T))}, - {config, hoconsc:t(hoconsc:ref(list_to_atom("emqx_connector_"++atom_to_list(T)), ""))}]). - -define(TYPES, [mysql, pgsql, mongo, redis, ldap]). -define(BRIDGES, [hoconsc:ref(?MODULE, T) || T <- ?TYPES]). @@ -19,8 +14,13 @@ fields("emqx_data_bridge") -> [{bridges, #{type => hoconsc:array(hoconsc:union(?BRIDGES)), default => []}}]; -fields(mysql) -> ?BRIDGE_FIELDS(mysql); -fields(pgsql) -> ?BRIDGE_FIELDS(pgsql); -fields(mongo) -> ?BRIDGE_FIELDS(mongo); -fields(redis) -> ?BRIDGE_FIELDS(redis); -fields(ldap) -> ?BRIDGE_FIELDS(ldap). \ No newline at end of file +fields(mysql) -> connector_fields(mysql); +fields(pgsql) -> connector_fields(pgsql); +fields(mongo) -> connector_fields(mongo); +fields(redis) -> connector_fields(redis); +fields(ldap) -> connector_fields(ldap). + +connector_fields(DB) -> + Mod = list_to_existing_atom(io_lib:format("~s_~s",[emqx_connector, DB])), + [{name, hoconsc:t(typerefl:binary())}, + {type, #{type => DB}}] ++ Mod:fields(""). diff --git a/apps/emqx_exhook/.gitignore b/apps/emqx_exhook/.gitignore deleted file mode 100644 index da1f0db23..000000000 --- a/apps/emqx_exhook/.gitignore +++ /dev/null @@ -1,29 +0,0 @@ -.rebar3 -_* -.eunit -*.o -*.beam -*.plt -*.swp -*.swo -.erlang.cookie -ebin -log -erl_crash.dump -.rebar -logs -_build -.idea -*.iml -rebar3.crashdump -*~ -rebar.lock -data/ -*.conf.rendered -*.pyc -.DS_Store -*.class -Mnesia.nonode@nohost/ -src/emqx_exhook_pb.erl -src/emqx_exhook_v_1_hook_provider_client.erl -src/emqx_exhook_v_1_hook_provider_bhvr.erl diff --git a/apps/emqx_exhook/docs/design-cn.md b/apps/emqx_exhook/docs/design-cn.md deleted file mode 100644 index 6686e96e3..000000000 --- a/apps/emqx_exhook/docs/design-cn.md +++ /dev/null @@ -1,116 +0,0 @@ -# 设计 - -## 动机 - -在 EMQ X Broker v4.1-v4.2 中，我们发布了 2 个插件来扩展 emqx 的编程能力： - -1. `emqx-extension-hook` 提供了使用 Java, Python 向 Broker 挂载钩子的功能 -2. `emqx-exproto` 提供了使用 Java，Python 编写用户自定义协议接入插件的功能 - -但在后续的支持中发现许多难以处理的问题： - -1. 有大量的编程语言需要支持，需要编写和维护如 Go, JavaScript, Lua.. 等语言的驱动。 -2. `erlport` 使用的操作系统的管道进行通信，这让用户代码只能部署在和 emqx 同一个操作系统上。部署方式受到了极大的限制。 -3. 用户程序的启动参数直接打包到 Broker 中，导致用户开发无法实时的进行调试，单步跟踪等。 -4. `erlport` 会占用 `stdin` `stdout`。 - -因此，我们计划重构这部分的实现，其中主要的内容是： -1. 使用 `gRPC` 替换 `erlport`。 -2. 将 `emqx-extension-hook` 重命名为 `emqx-exhook` - - -旧版本的设计：[emqx-extension-hook design in v4.2.0](https://github.com/emqx/emqx-exhook/blob/v4.2.0/docs/design.md) - -## 设计 - -架构如下： - -``` - EMQ X -+========================+ +========+==========+ -| ExHook | | | | -| +----------------+ | gRPC | gRPC | User's | -| | gRPC Client | ------------------> | Server | Codes | -| +----------------+ | (HTTP/2) | | | -| | | | | -+========================+ +========+==========+ -``` - -`emqx-exhook` 通过 gRPC 的方式向用户部署的 gRPC 服务发送钩子的请求，并处理其返回的值。 - - -和 emqx 原生的钩子一致，emqx-exhook 也按照链式的方式执行： - - - -### gRPC 服务示例 - -用户需要实现的方法，和数据类型的定义在 `priv/protos/exhook.proto` 文件中： - -```protobuff -syntax = "proto3"; - -package emqx.exhook.v1; - -service HookProvider { - - rpc OnProviderLoaded(ProviderLoadedRequest) returns (LoadedResponse) {}; - - rpc OnProviderUnloaded(ProviderUnloadedRequest) returns (EmptySuccess) {}; - - rpc OnClientConnect(ClientConnectRequest) returns (EmptySuccess) {}; - - rpc OnClientConnack(ClientConnackRequest) returns (EmptySuccess) {}; - - rpc OnClientConnected(ClientConnectedRequest) returns (EmptySuccess) {}; - - rpc OnClientDisconnected(ClientDisconnectedRequest) returns (EmptySuccess) {}; - - rpc OnClientAuthenticate(ClientAuthenticateRequest) returns (ValuedResponse) {}; - - rpc OnClientCheckAcl(ClientCheckAclRequest) returns (ValuedResponse) {}; - - rpc OnClientSubscribe(ClientSubscribeRequest) returns (EmptySuccess) {}; - - rpc OnClientUnsubscribe(ClientUnsubscribeRequest) returns (EmptySuccess) {}; - - rpc OnSessionCreated(SessionCreatedRequest) returns (EmptySuccess) {}; - - rpc OnSessionSubscribed(SessionSubscribedRequest) returns (EmptySuccess) {}; - - rpc OnSessionUnsubscribed(SessionUnsubscribedRequest) returns (EmptySuccess) {}; - - rpc OnSessionResumed(SessionResumedRequest) returns (EmptySuccess) {}; - - rpc OnSessionDiscarded(SessionDiscardedRequest) returns (EmptySuccess) {}; - - rpc OnSessionTakeovered(SessionTakeoveredRequest) returns (EmptySuccess) {}; - - rpc OnSessionTerminated(SessionTerminatedRequest) returns (EmptySuccess) {}; - - rpc OnMessagePublish(MessagePublishRequest) returns (ValuedResponse) {}; - - rpc OnMessageDelivered(MessageDeliveredRequest) returns (EmptySuccess) {}; - - rpc OnMessageDropped(MessageDroppedRequest) returns (EmptySuccess) {}; - - rpc OnMessageAcked(MessageAckedRequest) returns (EmptySuccess) {}; -} -``` - -### 配置文件示例 - -``` -## 配置 gRPC 服务地址 (HTTP) -## -## s1 为服务器的名称 -exhook.server.s1.url = http://127.0.0.1:9001 - -## 配置 gRPC 服务地址 (HTTPS) -## -## s2 为服务器名称 -exhook.server.s2.url = https://127.0.0.1:9002 -exhook.server.s2.cacertfile = ca.pem -exhook.server.s2.certfile = cert.pem -exhook.server.s2.keyfile = key.pem -``` diff --git a/apps/emqx_exhook/mix.exs b/apps/emqx_exhook/mix.exs deleted file mode 100644 index 03f03fce6..000000000 --- a/apps/emqx_exhook/mix.exs +++ /dev/null @@ -1,31 +0,0 @@ -defmodule EMQXExhook.MixProject do - use Mix.Project - - def project do - [ - app: :emqx_exhook, - version: "4.3.1", - build_path: "../../_build", - config_path: "../../config/config.exs", - deps_path: "../../deps", - lockfile: "../../mix.lock", - elixir: "~> 1.12", - start_permanent: Mix.env() == :prod, - deps: deps(), - description: "EMQ X Extension for Hook" - ] - end - - def application do - [ - mod: {:emqx_exhook_app, []}, - extra_applications: [:logger] - ] - end - - defp deps do - [ - {:grpc, github: "emqx/grpc-erl", tag: "0.6.2"} - ] - end -end diff --git a/apps/emqx_exhook/rebar.config b/apps/emqx_exhook/rebar.config deleted file mode 100644 index eafa20d85..000000000 --- a/apps/emqx_exhook/rebar.config +++ /dev/null @@ -1,48 +0,0 @@ -%%-*- mode: erlang -*- -{plugins, - [rebar3_proper, - {grpc_plugin, {git, "https://github.com/HJianBo/grpc_plugin", {tag, "v0.10.2"}}} -]}. - -{deps, - [{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.2"}}} -]}. - -{grpc, - [{protos, ["priv/protos"]}, - {gpb_opts, [{module_name_prefix, "emqx_"}, - {module_name_suffix, "_pb"}]} -]}. - -{provider_hooks, - [{pre, [{compile, {grpc, gen}}, - {clean, {grpc, clean}}]} -]}. - -{edoc_opts, [{preprocess, true}]}. - -{erl_opts, [warn_unused_vars, - warn_shadow_vars, - warn_unused_import, - warn_obsolete_guard, - debug_info, - {parse_transform}]}. - -{xref_checks, [undefined_function_calls, undefined_functions, - locals_not_used, deprecated_function_calls, - warnings_as_errors, deprecated_functions]}. -{xref_ignores, [emqx_exhook_pb]}. - -{cover_enabled, true}. -{cover_opts, [verbose]}. -{cover_export_enabled, true}. -{cover_excl_mods, [emqx_exhook_pb, - emqx_exhook_v_1_hook_provider_bhvr, - emqx_exhook_v_1_hook_provider_client]}. - -{profiles, - [{test, - [{deps, - []} - ]} -]}. diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src deleted file mode 100644 index 26e84d88f..000000000 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ /dev/null @@ -1,23 +0,0 @@ -%% -*-: erlang -*- -{VSN, - [ - {"4.3.1", [ - {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} - ]}, - {"4.3.0", [ - {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ], - [ - {"4.3.1", [ - {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} - ]}, - {"4.3.0", [ - {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ] -}. diff --git a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl deleted file mode 100644 index 5d5a396a5..000000000 --- a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl +++ /dev/null @@ -1,96 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_exhook_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - -%%-------------------------------------------------------------------- -%% Setups -%%-------------------------------------------------------------------- - -all() -> emqx_ct:all(?MODULE). - -init_per_suite(Cfg) -> - _ = emqx_exhook_demo_svr:start(), - emqx_ct_helpers:start_apps([emqx_exhook], fun set_special_cfgs/1), - Cfg. - -end_per_suite(_Cfg) -> - emqx_ct_helpers:stop_apps([emqx_exhook]), - emqx_exhook_demo_svr:stop(). - -set_special_cfgs(emqx) -> - application:set_env(emqx, allow_anonymous, false), - application:set_env(emqx, enable_acl_cache, false), - application:set_env(emqx, plugins_loaded_file, undefined), - application:set_env(emqx, modules_loaded_file, undefined); -set_special_cfgs(emqx_exhook) -> - ok. - -%%-------------------------------------------------------------------- -%% Test cases -%%-------------------------------------------------------------------- - -t_noserver_nohook(_) -> - emqx_exhook:disable(default), - ?assertEqual([], ets:tab2list(emqx_hooks)), - - Opts = proplists:get_value( - default, - application:get_env(emqx_exhook, servers, []) - ), - ok = emqx_exhook:enable(default, Opts), - ?assertNotEqual([], ets:tab2list(emqx_hooks)). - -t_cli_list(_) -> - meck_print(), - ?assertEqual( [[emqx_exhook_server:format(Svr) || Svr <- emqx_exhook:list()]] - , emqx_exhook_cli:cli(["server", "list"]) - ), - unmeck_print(). - -t_cli_enable_disable(_) -> - meck_print(), - ?assertEqual([already_started], emqx_exhook_cli:cli(["server", "enable", "default"])), - ?assertEqual(ok, emqx_exhook_cli:cli(["server", "disable", "default"])), - ?assertEqual([], emqx_exhook_cli:cli(["server", "list"])), - - ?assertEqual([not_running], emqx_exhook_cli:cli(["server", "disable", "default"])), - ?assertEqual(ok, emqx_exhook_cli:cli(["server", "enable", "default"])), - unmeck_print(). - -t_cli_stats(_) -> - meck_print(), - _ = emqx_exhook_cli:cli(["server", "stats"]), - _ = emqx_exhook_cli:cli(x), - unmeck_print(). - -%%-------------------------------------------------------------------- -%% Utils -%%-------------------------------------------------------------------- - -meck_print() -> - meck:new(emqx_ctl, [passthrough, no_history, no_link]), - meck:expect(emqx_ctl, print, fun(_) -> ok end), - meck:expect(emqx_ctl, print, fun(_, Args) -> Args end). - -unmeck_print() -> - meck:unload(emqx_ctl). diff --git a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl deleted file mode 100644 index c2db04dd4..000000000 --- a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl +++ /dev/null @@ -1,339 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_exhook_demo_svr). - --behavior(emqx_exhook_v_1_hook_provider_bhvr). - -%% --export([ start/0 - , stop/0 - , take/0 - , in/1 - ]). - -%% gRPC server HookProvider callbacks --export([ on_provider_loaded/2 - , on_provider_unloaded/2 - , on_client_connect/2 - , on_client_connack/2 - , on_client_connected/2 - , on_client_disconnected/2 - , on_client_authenticate/2 - , on_client_check_acl/2 - , on_client_subscribe/2 - , on_client_unsubscribe/2 - , on_session_created/2 - , on_session_subscribed/2 - , on_session_unsubscribed/2 - , on_session_resumed/2 - , on_session_discarded/2 - , on_session_takeovered/2 - , on_session_terminated/2 - , on_message_publish/2 - , on_message_delivered/2 - , on_message_dropped/2 - , on_message_acked/2 - ]). - --define(PORT, 9000). --define(NAME, ?MODULE). - -%%-------------------------------------------------------------------- -%% Server APIs -%%-------------------------------------------------------------------- - -start() -> - Pid = spawn(fun mngr_main/0), - register(?MODULE, Pid), - {ok, Pid}. - -stop() -> - grpc:stop_server(?NAME), - ?MODULE ! stop. - -take() -> - ?MODULE ! {take, self()}, - receive {value, V} -> V - after 5000 -> error(timeout) end. - -in({FunName, Req}) -> - ?MODULE ! {in, FunName, Req}. - -mngr_main() -> - application:ensure_all_started(grpc), - Services = #{protos => [emqx_exhook_pb], - services => #{'emqx.exhook.v1.HookProvider' => emqx_exhook_demo_svr} - }, - Options = [], - Svr = grpc:start_server(?NAME, ?PORT, Services, Options), - mngr_loop([Svr, queue:new(), queue:new()]). - -mngr_loop([Svr, Q, Takes]) -> - receive - {in, FunName, Req} -> - {NQ1, NQ2} = reply(queue:in({FunName, Req}, Q), Takes), - mngr_loop([Svr, NQ1, NQ2]); - {take, From} -> - {NQ1, NQ2} = reply(Q, queue:in(From, Takes)), - mngr_loop([Svr, NQ1, NQ2]); - stop -> - exit(normal) - end. - -reply(Q1, Q2) -> - case queue:len(Q1) =:= 0 orelse - queue:len(Q2) =:= 0 of - true -> {Q1, Q2}; - _ -> - {{value, {Name, V}}, NQ1} = queue:out(Q1), - {{value, From}, NQ2} = queue:out(Q2), - From ! {value, {Name, V}}, - {NQ1, NQ2} - end. - -%%-------------------------------------------------------------------- -%% callbacks -%%-------------------------------------------------------------------- - --spec on_provider_loaded(emqx_exhook_pb:provider_loaded_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:loaded_response(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. - -on_provider_loaded(Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{hooks => [ - #{name => <<"client.connect">>}, - #{name => <<"client.connack">>}, - #{name => <<"client.connected">>}, - #{name => <<"client.disconnected">>}, - #{name => <<"client.authenticate">>}, - #{name => <<"client.check_acl">>}, - #{name => <<"client.subscribe">>}, - #{name => <<"client.unsubscribe">>}, - #{name => <<"session.created">>}, - #{name => <<"session.subscribed">>}, - #{name => <<"session.unsubscribed">>}, - #{name => <<"session.resumed">>}, - #{name => <<"session.discarded">>}, - #{name => <<"session.takeovered">>}, - #{name => <<"session.terminated">>}, - #{name => <<"message.publish">>}, - #{name => <<"message.delivered">>}, - #{name => <<"message.acked">>}, - #{name => <<"message.dropped">>}]}, Md}. --spec on_provider_unloaded(emqx_exhook_pb:provider_unloaded_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_provider_unloaded(Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. - --spec on_client_connect(emqx_exhook_pb:client_connect_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_client_connect(Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. - --spec on_client_connack(emqx_exhook_pb:client_connack_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_client_connack(Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. - --spec on_client_connected(emqx_exhook_pb:client_connected_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_client_connected(Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. - --spec on_client_disconnected(emqx_exhook_pb:client_disconnected_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_client_disconnected(Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. - --spec on_client_authenticate(emqx_exhook_pb:client_authenticate_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_client_authenticate(#{clientinfo := #{username := Username}} = Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - %% some cases for testing - case Username of - <<"baduser">> -> - {ok, #{type => 'STOP_AND_RETURN', - value => {bool_result, false}}, Md}; - <<"gooduser">> -> - {ok, #{type => 'STOP_AND_RETURN', - value => {bool_result, true}}, Md}; - <<"normaluser">> -> - {ok, #{type => 'CONTINUE', - value => {bool_result, true}}, Md}; - _ -> - {ok, #{type => 'IGNORE'}, Md} - end. - --spec on_client_check_acl(emqx_exhook_pb:client_check_acl_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_client_check_acl(#{clientinfo := #{username := Username}} = Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - %% some cases for testing - case Username of - <<"baduser">> -> - {ok, #{type => 'STOP_AND_RETURN', - value => {bool_result, false}}, Md}; - <<"gooduser">> -> - {ok, #{type => 'STOP_AND_RETURN', - value => {bool_result, true}}, Md}; - <<"normaluser">> -> - {ok, #{type => 'CONTINUE', - value => {bool_result, true}}, Md}; - _ -> - {ok, #{type => 'IGNORE'}, Md} - end. - --spec on_client_subscribe(emqx_exhook_pb:client_subscribe_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_client_subscribe(Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. - --spec on_client_unsubscribe(emqx_exhook_pb:client_unsubscribe_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_client_unsubscribe(Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. - --spec on_session_created(emqx_exhook_pb:session_created_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_session_created(Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. - --spec on_session_subscribed(emqx_exhook_pb:session_subscribed_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_session_subscribed(Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. - --spec on_session_unsubscribed(emqx_exhook_pb:session_unsubscribed_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_session_unsubscribed(Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. - --spec on_session_resumed(emqx_exhook_pb:session_resumed_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_session_resumed(Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. - --spec on_session_discarded(emqx_exhook_pb:session_discarded_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_session_discarded(Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. - --spec on_session_takeovered(emqx_exhook_pb:session_takeovered_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_session_takeovered(Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. - --spec on_session_terminated(emqx_exhook_pb:session_terminated_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_session_terminated(Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. - --spec on_message_publish(emqx_exhook_pb:message_publish_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_message_publish(#{message := #{from := From} = Msg} = Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - %% some cases for testing - case From of - <<"baduser">> -> - NMsg = Msg#{qos => 0, - topic => <<"">>, - payload => <<"">> - }, - {ok, #{type => 'STOP_AND_RETURN', - value => {message, NMsg}}, Md}; - <<"gooduser">> -> - NMsg = Msg#{topic => From, - payload => From}, - {ok, #{type => 'STOP_AND_RETURN', - value => {message, NMsg}}, Md}; - _ -> - {ok, #{type => 'IGNORE'}, Md} - end. - --spec on_message_delivered(emqx_exhook_pb:message_delivered_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_message_delivered(Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. - --spec on_message_dropped(emqx_exhook_pb:message_dropped_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_message_dropped(Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. - --spec on_message_acked(emqx_exhook_pb:message_acked_request(), grpc:metadata()) - -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_message_acked(Req, Md) -> - ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. diff --git a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl deleted file mode 100644 index 24f45c8b0..000000000 --- a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl +++ /dev/null @@ -1,531 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(prop_exhook_hooks). - --include_lib("proper/include/proper.hrl"). --include_lib("eunit/include/eunit.hrl"). - --import(emqx_ct_proper_types, - [ conninfo/0 - , clientinfo/0 - , sessioninfo/0 - , message/0 - , connack_return_code/0 - , topictab/0 - , topic/0 - , subopts/0 - ]). - --define(ALL(Vars, Types, Exprs), - ?SETUP(fun() -> - State = do_setup(), - fun() -> do_teardown(State) end - end, ?FORALL(Vars, Types, Exprs))). - -%%-------------------------------------------------------------------- -%% Properties -%%-------------------------------------------------------------------- - -prop_client_connect() -> - ?ALL({ConnInfo, ConnProps}, - {conninfo(), conn_properties()}, - begin - ok = emqx_hooks:run('client.connect', [ConnInfo, ConnProps]), - {'on_client_connect', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{props => properties(ConnProps), - conninfo => from_conninfo(ConnInfo) - }, - ?assertEqual(Expected, Resp), - true - end). - -prop_client_connack() -> - ?ALL({ConnInfo, Rc, AckProps}, - {conninfo(), connack_return_code(), ack_properties()}, - begin - ok = emqx_hooks:run('client.connack', [ConnInfo, Rc, AckProps]), - {'on_client_connack', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{props => properties(AckProps), - result_code => atom_to_binary(Rc, utf8), - conninfo => from_conninfo(ConnInfo) - }, - ?assertEqual(Expected, Resp), - true - end). - -prop_client_authenticate() -> - ?ALL({ClientInfo0, AuthResult}, - {clientinfo(), authresult()}, - begin - ClientInfo = inject_magic_into(username, ClientInfo0), - OutAuthResult = emqx_hooks:run_fold('client.authenticate', [ClientInfo], AuthResult), - ExpectedAuthResult = case maps:get(username, ClientInfo) of - <<"baduser">> -> - AuthResult#{ - auth_result => not_authorized, - anonymous => false}; - <<"gooduser">> -> - AuthResult#{ - auth_result => success, - anonymous => false}; - <<"normaluser">> -> - AuthResult#{ - auth_result => success, - anonymous => false}; - _ -> - case maps:get(auth_result, AuthResult) of - success -> - #{auth_result => success, - anonymous => false}; - _ -> - #{auth_result => not_authorized, - anonymous => false} - end - end, - ?assertEqual(ExpectedAuthResult, OutAuthResult), - - {'on_client_authenticate', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{result => authresult_to_bool(AuthResult), - clientinfo => from_clientinfo(ClientInfo) - }, - ?assertEqual(Expected, Resp), - true - end). - -prop_client_check_acl() -> - ?ALL({ClientInfo0, PubSub, Topic, Result}, - {clientinfo(), oneof([publish, subscribe]), - topic(), oneof([allow, deny])}, - begin - ClientInfo = inject_magic_into(username, ClientInfo0), - OutResult = emqx_hooks:run_fold( - 'client.check_acl', - [ClientInfo, PubSub, Topic], - Result), - ExpectedOutResult = case maps:get(username, ClientInfo) of - <<"baduser">> -> deny; - <<"gooduser">> -> allow; - <<"normaluser">> -> allow; - _ -> Result - end, - ?assertEqual(ExpectedOutResult, OutResult), - - {'on_client_check_acl', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{result => aclresult_to_bool(Result), - type => pubsub_to_enum(PubSub), - topic => Topic, - clientinfo => from_clientinfo(ClientInfo) - }, - ?assertEqual(Expected, Resp), - true - end). - -prop_client_connected() -> - ?ALL({ClientInfo, ConnInfo}, - {clientinfo(), conninfo()}, - begin - ok = emqx_hooks:run('client.connected', [ClientInfo, ConnInfo]), - {'on_client_connected', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{clientinfo => from_clientinfo(ClientInfo) - }, - ?assertEqual(Expected, Resp), - true - end). - -prop_client_disconnected() -> - ?ALL({ClientInfo, Reason, ConnInfo}, - {clientinfo(), shutdown_reason(), conninfo()}, - begin - ok = emqx_hooks:run('client.disconnected', [ClientInfo, Reason, ConnInfo]), - {'on_client_disconnected', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{reason => stringfy(Reason), - clientinfo => from_clientinfo(ClientInfo) - }, - ?assertEqual(Expected, Resp), - true - end). - -prop_client_subscribe() -> - ?ALL({ClientInfo, SubProps, TopicTab}, - {clientinfo(), sub_properties(), topictab()}, - begin - ok = emqx_hooks:run('client.subscribe', [ClientInfo, SubProps, TopicTab]), - {'on_client_subscribe', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{props => properties(SubProps), - topic_filters => topicfilters(TopicTab), - clientinfo => from_clientinfo(ClientInfo) - }, - ?assertEqual(Expected, Resp), - true - end). - -prop_client_unsubscribe() -> - ?ALL({ClientInfo, UnSubProps, TopicTab}, - {clientinfo(), unsub_properties(), topictab()}, - begin - ok = emqx_hooks:run('client.unsubscribe', [ClientInfo, UnSubProps, TopicTab]), - {'on_client_unsubscribe', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{props => properties(UnSubProps), - topic_filters => topicfilters(TopicTab), - clientinfo => from_clientinfo(ClientInfo) - }, - ?assertEqual(Expected, Resp), - true - end). - -prop_session_created() -> - ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()}, - begin - ok = emqx_hooks:run('session.created', [ClientInfo, SessInfo]), - {'on_session_created', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{clientinfo => from_clientinfo(ClientInfo) - }, - ?assertEqual(Expected, Resp), - true - end). - -prop_session_subscribed() -> - ?ALL({ClientInfo, Topic, SubOpts}, - {clientinfo(), topic(), subopts()}, - begin - ok = emqx_hooks:run('session.subscribed', [ClientInfo, Topic, SubOpts]), - {'on_session_subscribed', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{topic => Topic, - subopts => subopts(SubOpts), - clientinfo => from_clientinfo(ClientInfo) - }, - ?assertEqual(Expected, Resp), - true - end). - -prop_session_unsubscribed() -> - ?ALL({ClientInfo, Topic, SubOpts}, - {clientinfo(), topic(), subopts()}, - begin - ok = emqx_hooks:run('session.unsubscribed', [ClientInfo, Topic, SubOpts]), - {'on_session_unsubscribed', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{topic => Topic, - clientinfo => from_clientinfo(ClientInfo) - }, - ?assertEqual(Expected, Resp), - true - end). - -prop_session_resumed() -> - ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()}, - begin - ok = emqx_hooks:run('session.resumed', [ClientInfo, SessInfo]), - {'on_session_resumed', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{clientinfo => from_clientinfo(ClientInfo) - }, - ?assertEqual(Expected, Resp), - true - end). - -prop_session_discared() -> - ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()}, - begin - ok = emqx_hooks:run('session.discarded', [ClientInfo, SessInfo]), - {'on_session_discarded', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{clientinfo => from_clientinfo(ClientInfo) - }, - ?assertEqual(Expected, Resp), - true - end). - -prop_session_takeovered() -> - ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()}, - begin - ok = emqx_hooks:run('session.takeovered', [ClientInfo, SessInfo]), - {'on_session_takeovered', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{clientinfo => from_clientinfo(ClientInfo) - }, - ?assertEqual(Expected, Resp), - true - end). - -prop_session_terminated() -> - ?ALL({ClientInfo, Reason, SessInfo}, - {clientinfo(), shutdown_reason(), sessioninfo()}, - begin - ok = emqx_hooks:run('session.terminated', [ClientInfo, Reason, SessInfo]), - {'on_session_terminated', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{reason => stringfy(Reason), - clientinfo => from_clientinfo(ClientInfo) - }, - ?assertEqual(Expected, Resp), - true - end). - -prop_message_publish() -> - ?ALL(Msg0, message(), - begin - Msg = emqx_message:from_map( - inject_magic_into(from, emqx_message:to_map(Msg0))), - OutMsg= emqx_hooks:run_fold('message.publish', [], Msg), - case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of - true -> - ?assertEqual(Msg, OutMsg), - skip; - _ -> - ExpectedOutMsg = case emqx_message:from(Msg) of - <<"baduser">> -> - MsgMap = emqx_message:to_map(Msg), - emqx_message:from_map( - MsgMap#{qos => 0, - topic => <<"">>, - payload => <<"">> - }); - <<"gooduser">> = From -> - MsgMap = emqx_message:to_map(Msg), - emqx_message:from_map( - MsgMap#{topic => From, - payload => From - }); - _ -> Msg - end, - ?assertEqual(ExpectedOutMsg, OutMsg), - - {'on_message_publish', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{message => from_message(Msg) - }, - ?assertEqual(Expected, Resp) - end, - true - end). - -prop_message_dropped() -> - ?ALL({Msg, By, Reason}, {message(), hardcoded, shutdown_reason()}, - begin - ok = emqx_hooks:run('message.dropped', [Msg, By, Reason]), - case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of - true -> skip; - _ -> - {'on_message_dropped', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{reason => stringfy(Reason), - message => from_message(Msg) - }, - ?assertEqual(Expected, Resp) - end, - true - end). - -prop_message_delivered() -> - ?ALL({ClientInfo, Msg}, {clientinfo(), message()}, - begin - ok = emqx_hooks:run('message.delivered', [ClientInfo, Msg]), - case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of - true -> skip; - _ -> - {'on_message_delivered', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{clientinfo => from_clientinfo(ClientInfo), - message => from_message(Msg) - }, - ?assertEqual(Expected, Resp) - end, - true - end). - -prop_message_acked() -> - ?ALL({ClientInfo, Msg}, {clientinfo(), message()}, - begin - ok = emqx_hooks:run('message.acked', [ClientInfo, Msg]), - case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of - true -> skip; - _ -> - {'on_message_acked', Resp} = emqx_exhook_demo_svr:take(), - Expected = - #{clientinfo => from_clientinfo(ClientInfo), - message => from_message(Msg) - }, - ?assertEqual(Expected, Resp) - end, - true - end). - -nodestr() -> - stringfy(node()). - -peerhost(#{peername := {Host, _}}) -> - ntoa(Host). - -sockport(#{sockname := {_, Port}}) -> - Port. - -%% copied from emqx_exhook - -ntoa({0,0,0,0,0,16#ffff,AB,CD}) -> - list_to_binary(inet_parse:ntoa({AB bsr 8, AB rem 256, CD bsr 8, CD rem 256})); -ntoa(IP) -> - list_to_binary(inet_parse:ntoa(IP)). - -maybe(undefined) -> <<>>; -maybe(B) -> B. - -properties(undefined) -> []; -properties(M) when is_map(M) -> - maps:fold(fun(K, V, Acc) -> - [#{name => stringfy(K), - value => stringfy(V)} | Acc] - end, [], M). - -topicfilters(Tfs) when is_list(Tfs) -> - [#{name => Topic, qos => Qos} || {Topic, #{qos := Qos}} <- Tfs]. - -%% @private -stringfy(Term) when is_binary(Term) -> - Term; -stringfy(Term) when is_integer(Term) -> - integer_to_binary(Term); -stringfy(Term) when is_atom(Term) -> - atom_to_binary(Term, utf8); -stringfy(Term) -> - unicode:characters_to_binary((io_lib:format("~0p", [Term]))). - -subopts(SubOpts) -> - #{qos => maps:get(qos, SubOpts, 0), - rh => maps:get(rh, SubOpts, 0), - rap => maps:get(rap, SubOpts, 0), - nl => maps:get(nl, SubOpts, 0), - share => maps:get(share, SubOpts, <<>>) - }. - -authresult_to_bool(AuthResult) -> - maps:get(auth_result, AuthResult, undefined) == success. - -aclresult_to_bool(Result) -> - Result == allow. - -pubsub_to_enum(publish) -> 'PUBLISH'; -pubsub_to_enum(subscribe) -> 'SUBSCRIBE'. - -from_conninfo(ConnInfo) -> - #{node => nodestr(), - clientid => maps:get(clientid, ConnInfo), - username => maybe(maps:get(username, ConnInfo, <<>>)), - peerhost => peerhost(ConnInfo), - sockport => sockport(ConnInfo), - proto_name => maps:get(proto_name, ConnInfo), - proto_ver => stringfy(maps:get(proto_ver, ConnInfo)), - keepalive => maps:get(keepalive, ConnInfo) - }. - -from_clientinfo(ClientInfo) -> - #{node => nodestr(), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo, <<>>)), - password => maybe(maps:get(password, ClientInfo, <<>>)), - peerhost => ntoa(maps:get(peerhost, ClientInfo)), - sockport => maps:get(sockport, ClientInfo), - protocol => stringfy(maps:get(protocol, ClientInfo)), - mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)), - is_superuser => maps:get(is_superuser, ClientInfo, false), - anonymous => maps:get(anonymous, ClientInfo, true), - cn => maybe(maps:get(cn, ClientInfo, <<>>)), - dn => maybe(maps:get(dn, ClientInfo, <<>>)) - }. - -from_message(Msg) -> - #{node => nodestr(), - id => emqx_guid:to_hexstr(emqx_message:id(Msg)), - qos => emqx_message:qos(Msg), - from => stringfy(emqx_message:from(Msg)), - topic => emqx_message:topic(Msg), - payload => emqx_message:payload(Msg), - timestamp => emqx_message:timestamp(Msg) - }. - -%%-------------------------------------------------------------------- -%% Helper -%%-------------------------------------------------------------------- - -do_setup() -> - logger:set_primary_config(#{level => warning}), - _ = emqx_exhook_demo_svr:start(), - emqx_ct_helpers:start_apps([emqx_exhook], fun set_special_cfgs/1), - %% waiting first loaded event - {'on_provider_loaded', _} = emqx_exhook_demo_svr:take(), - ok. - -do_teardown(_) -> - emqx_ct_helpers:stop_apps([emqx_exhook]), - %% waiting last unloaded event - {'on_provider_unloaded', _} = emqx_exhook_demo_svr:take(), - _ = emqx_exhook_demo_svr:stop(), - logger:set_primary_config(#{level => notice}), - timer:sleep(2000), - ok. - -set_special_cfgs(emqx) -> - application:set_env(emqx, allow_anonymous, false), - application:set_env(emqx, enable_acl_cache, false), - application:set_env(emqx, modules_loaded_file, undefined), - application:set_env(emqx, plugins_loaded_file, - emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins")); -set_special_cfgs(emqx_exhook) -> - ok. - -%%-------------------------------------------------------------------- -%% Generators -%%-------------------------------------------------------------------- - -conn_properties() -> - #{}. - -ack_properties() -> - #{}. - -sub_properties() -> - #{}. - -unsub_properties() -> - #{}. - -shutdown_reason() -> - oneof([utf8(), {shutdown, emqx_ct_proper_types:limited_atom()}]). - -authresult() -> - ?LET(RC, connack_return_code(), #{auth_result => RC}). - -inject_magic_into(Key, Object) -> - case castspell() of - muggles -> Object; - Spell -> - Object#{Key => Spell} - end. - -castspell() -> - L = [<<"baduser">>, <<"gooduser">>, <<"normaluser">>, muggles], - lists:nth(rand:uniform(length(L)), L). diff --git a/apps/emqx_exproto/.gitignore b/apps/emqx_exproto/.gitignore deleted file mode 100644 index 384f2255a..000000000 --- a/apps/emqx_exproto/.gitignore +++ /dev/null @@ -1,48 +0,0 @@ -.eunit -deps -!deps/.placeholder -*.o -*.beam -*.plt -erl_crash.dump -ebin -!ebin/.placeholder -.concrete/DEV_MODE -.rebar -test/ebin/*.beam -.exrc -plugins/*/ebin -log/ -*.swp -*.so -.erlang.mk/ -cover/ -emqx.d -eunit.coverdata -test/ct.cover.spec -logs -ct.coverdata -.idea/ -emqx.iml -_rel/ -data/ -_build -.rebar3 -rebar3.crashdump -.DS_Store -emqx.iml -bbmustache/ -etc/gen.emqx.conf -compile_commands.json -cuttlefish -rebar.lock -xrefr -erlang.mk -*.coverdata -etc/emqx_exproto.conf.rendered -Mnesia.*/ -src/emqx_exproto_pb.erl -src/emqx_exproto_v_1_connection_adapter_bhvr.erl -src/emqx_exproto_v_1_connection_adapter_client.erl -src/emqx_exproto_v_1_connection_handler_bhvr.erl -src/emqx_exproto_v_1_connection_handler_client.erl diff --git a/apps/emqx_exproto/docs/design-cn.md b/apps/emqx_exproto/docs/design-cn.md deleted file mode 100644 index 7af7dbdb3..000000000 --- a/apps/emqx_exproto/docs/design-cn.md +++ /dev/null @@ -1,127 +0,0 @@ -# 多语言 - 协议接入 - -`emqx-exproto` 插件用于协议解析的多语言支持。它能够允许其他编程语言（例如：Python，Java 等）直接处理数据流实现协议的解析，并提供 Pub/Sub 接口以实现与系统其它组件的通信。 - -该插件给 EMQ X 带来的扩展性十分的强大，它能以你熟悉语言处理任何的私有协议，并享受由 EMQ X 系统带来的高连接，和高并发的优点。 - -## 特性 - -- 极强的扩展能力。使用 gRPC 作为 RPC 通信框架，支持各个主流编程语言 -- 高吞吐。连接层以完全的异步非阻塞式 I/O 的方式实现 -- 连接层透明。完全的支持 TCP\TLS UDP\DTLS 类型的连接管理，并对上层提供统一的 API 接口 -- 连接层的管理能力。例如，最大连接数，连接和吞吐的速率限制，IP 黑名单 等 - -## 架构 - -![Extension-Protocol Arch](images/exproto-arch.jpg) - -该插件主要需要处理的内容包括： - -1. **连接层：** 该部分主要 **维持 Socket 的生命周期，和数据的收发**。它的功能要求包括： - - 监听某个端口。当有新的 TCP/UDP 连接到达后，启动一个连接进程，来维持连接的状态。 - - 调用 `OnSocketCreated` 回调。用于通知外部模块**已新建立了一个连接**。 - - 调用 `OnScoektClosed` 回调。用于通知外部模块连接**已关闭**。 - - 调用 `OnReceivedBytes` 回调。用于通知外部模块**该连接新收到的数据包**。 - - 提供 `Send` 接口。供外部模块调用，**用于发送数据包**。 - - 提供 `Close` 接口。供外部模块调用，**用于主动关闭连接**。 - -2. **协议/会话层：**该部分主要**提供 PUB/SUB 接口**，以实现与 EMQ X Broker 系统的消息互通。包括： - - - 提供 `Authenticate` 接口。供外部模块调用，用于向集群注册客户端。 - - 提供 `StartTimer` 接口。供外部模块调用，用于为该连接进程启动心跳等定时器。 - - 提供 `Publish` 接口。供外部模块调用，用于发布消息 EMQ X Broker 中。 - - 提供 `Subscribe` 接口。供外部模块调用，用于订阅某主题，以实现从 EMQ X Broker 中接收某些下行消息。 - - 提供 `Unsubscribe` 接口。供外部模块调用，用于取消订阅某主题。 - - 调用 `OnTimerTimeout` 回调。用于处理定时器超时的事件。 - - 调用 `OnReceivedMessages` 回调。用于接收下行消息（在订阅主题成功后，如果主题上有消息，便会回调该方法） - -## 接口设计 - -从 gRPC 上的逻辑来说，emqx-exproto 会作为客户端向用户的 `ConnectionHandler` 服务发送回调请求。同时，它也会作为服务端向用户提供 `ConnectionAdapter` 服务，以提供 emqx-exproto 各个接口的访问。如图： - -![Extension Protocol gRPC Arch](images/exproto-grpc-arch.jpg) - - -详情参见：`priv/protos/exproto.proto`，例如接口的定义有： - -```protobuff -syntax = "proto3"; - -package emqx.exproto.v1; - -// The Broker side serivce. It provides a set of APIs to -// handle a protcol access -service ConnectionAdapter { - - // -- socket layer - - rpc Send(SendBytesRequest) returns (CodeResponse) {}; - - rpc Close(CloseSocketRequest) returns (CodeResponse) {}; - - // -- protocol layer - - rpc Authenticate(AuthenticateRequest) returns (CodeResponse) {}; - - rpc StartTimer(TimerRequest) returns (CodeResponse) {}; - - // -- pub/sub layer - - rpc Publish(PublishRequest) returns (CodeResponse) {}; - - rpc Subscribe(SubscribeRequest) returns (CodeResponse) {}; - - rpc Unsubscribe(UnsubscribeRequest) returns (CodeResponse) {}; -} - -service ConnectionHandler { - - // -- socket layer - - rpc OnSocketCreated(stream SocketCreatedRequest) returns (EmptySuccess) {}; - - rpc OnSocketClosed(stream SocketClosedRequest) returns (EmptySuccess) {}; - - rpc OnReceivedBytes(stream ReceivedBytesRequest) returns (EmptySuccess) {}; - - // -- pub/sub layer - - rpc OnTimerTimeout(stream TimerTimeoutRequest) returns (EmptySuccess) {}; - - rpc OnReceivedMessages(stream ReceivedMessagesRequest) returns (EmptySuccess) {}; -} -``` - -## 配置项设计 - -1. 以 **监听器(Listener)** 为基础，提供 TCP/UDP 的监听。 - - Listener 目前仅支持：TCP、TLS、UDP、DTLS。(ws、wss、quic 暂不支持) -2. 每个监听器，会指定一个 `ConnectionHandler` 的服务地址，用于调用外部模块的接口。 -3. emqx-exproto 还会监听一个 gRPC 端口用于提供对 `ConnectionAdapter` 服务的访问。 - -例如： - -``` properties -## gRPC 服务监听地址 (HTTP) -## -exproto.server.http.url = http://127.0.0.1:9002 - -## gRPC 服务监听地址 (HTTPS) -## -exproto.server.https.url = https://127.0.0.1:9002 -exproto.server.https.cacertfile = ca.pem -exproto.server.https.certfile = cert.pem -exproto.server.https.keyfile = key.pem - -## Listener 配置 -## 例如，名称为 protoname 协议的 TCP 监听器配置 -exproto.listener.protoname = tcp://0.0.0.0:7993 - -## ConnectionHandler 服务地址及 https 的证书配置 -exproto.listener.protoname.connection_handler_url = http://127.0.0.1:9001 -#exproto.listener.protoname.connection_handler_certfile = -#exproto.listener.protoname.connection_handler_cacertfile = -#exproto.listener.protoname.connection_handler_keyfile = - -# ... -``` diff --git a/apps/emqx_exproto/docs/images/exproto-arch.jpg b/apps/emqx_exproto/docs/images/exproto-arch.jpg deleted file mode 100644 index dddf7996b..000000000 Binary files a/apps/emqx_exproto/docs/images/exproto-arch.jpg and /dev/null differ diff --git a/apps/emqx_exproto/docs/images/exproto-grpc-arch.jpg b/apps/emqx_exproto/docs/images/exproto-grpc-arch.jpg deleted file mode 100644 index 71efa76f9..000000000 Binary files a/apps/emqx_exproto/docs/images/exproto-grpc-arch.jpg and /dev/null differ diff --git a/apps/emqx_exproto/etc/emqx_exproto.conf b/apps/emqx_exproto/etc/emqx_exproto.conf deleted file mode 100644 index 7a7667271..000000000 --- a/apps/emqx_exproto/etc/emqx_exproto.conf +++ /dev/null @@ -1,252 +0,0 @@ -##==================================================================== -## EMQ X ExProto -##==================================================================== - -exproto.server.http.port = 9100 - -exproto.server.https.port = 9101 -exproto.server.https.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" -exproto.server.https.certfile = "{{ platform_etc_dir }}/certs/cert.pem" -exproto.server.https.keyfile = "{{ platform_etc_dir }}/certs/key.pem" - -##-------------------------------------------------------------------- -## Listeners -##-------------------------------------------------------------------- - -##-------------------------------------------------------------------- -## MQTT/TCP - External TCP Listener for MQTT Protocol - -## The IP address and port that the listener will bind. -## -## Value: ://: -## -## Examples: "tcp://0.0.0.0:7993" | "ssl://127.0.0.1:7994" -exproto.listener.protoname.endpoint = "tcp://0.0.0.0:7993" - -## The ConnectionHandler server address -## -exproto.listener.protoname.connection_handler_url = "http://127.0.0.1:9001" - -#exproto.listener.protoname.connection_handler_certfile = -#exproto.listener.protoname.connection_handler_cacertfile = -#exproto.listener.protoname.connection_handler_keyfile = - -## The acceptor pool for external MQTT/TCP listener. -## -## Value: Number -exproto.listener.protoname.acceptors = 8 - -## Maximum number of concurrent MQTT/TCP connections. -## -## Value: Number -exproto.listener.protoname.max_connections = 1024000 - -## Maximum external connections per second. -## -## Value: Number -exproto.listener.protoname.max_conn_rate = 1000 - -## Specify the {active, N} option for the external MQTT/TCP Socket. -## -## Value: Number -exproto.listener.protoname.active_n = 100 - -## Idle timeout -## -## Value: Duration -exproto.listener.protoname.idle_timeout = 30s - -## The access control rules for the MQTT/TCP listener. -## -## See: https://github.com/emqtt/esockd#allowdeny -## -## Value: ACL Rule -## -## Example: "allow 192.168.0.0/24" -exproto.listener.protoname.access.1 = "allow all" - -## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed -## behind HAProxy or Nginx. -## -## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ -## -## Value: on | off -## exproto.listener.protoname.proxy_protocol = on - -## Sets the timeout for proxy protocol. EMQ X will close the TCP connection -## if no proxy protocol packet recevied within the timeout. -## -## Value: Duration -#exproto.listener.protoname.proxy_protocol_timeout = 3s - -## The TCP backlog defines the maximum length that the queue of pending -## connections can grow to. -## -## Value: Number >= 0 -exproto.listener.protoname.backlog = 1024 - -## The TCP send timeout for external MQTT connections. -## -## Value: Duration -exproto.listener.protoname.send_timeout = 15s - -## Close the TCP connection if send timeout. -## -## Value: on | off -exproto.listener.protoname.send_timeout_close = on - -## The TCP receive buffer(os kernel) for MQTT connections. -## -## See: http://erlang.org/doc/man/inet.html -## -## Value: Bytes -#exproto.listener.protoname.recbuf = 2KB - -## The TCP send buffer(os kernel) for MQTT connections. -## -## See: http://erlang.org/doc/man/inet.html -## -## Value: Bytes -#exproto.listener.protoname.sndbuf = 2KB - -## The size of the user-level software buffer used by the driver. -## Not to be confused with options sndbuf and recbuf, which correspond -## to the Kernel socket buffers. It is recommended to have val(buffer) -## >= max(val(sndbuf),val(recbuf)) to avoid performance issues because -## of unnecessary copying. val(buffer) is automatically set to the above -## maximum when values sndbuf or recbuf are set. -## -## See: http://erlang.org/doc/man/inet.html -## -## Value: Bytes -#exproto.listener.protoname.buffer = 2KB - -## Sets the 'buffer = max(sndbuf, recbuf)' if this option is enabled. -## -## Value: on | off -#exproto.listener.protoname.tune_buffer = off - -## The TCP_NODELAY flag for MQTT connections. Small amounts of data are -## sent immediately if the option is enabled. -## -## Value: true | false -exproto.listener.protoname.nodelay = true - -## The SO_REUSEADDR flag for TCP listener. -## -## Value: true | false -exproto.listener.protoname.reuseaddr = true - - -##-------------------------------------------------------------------- -## TLS/DTLS options - -## TLS versions only to protect from POODLE attack. -## -## See: http://erlang.org/doc/man/ssl.html -## -## Value: String, seperated by ',' -#exproto.listener.protoname.tls_versions = "tlsv1.2,tlsv1.1,tlsv1" - -## Path to the file containing the user's private PEM-encoded key. -## -## See: http://erlang.org/doc/man/ssl.html -## -## Value: File -#exproto.listener.protoname.keyfile = "{{ platform_etc_dir }}/certs/key.pem" - -## Path to a file containing the user certificate. -## -## See: http://erlang.org/doc/man/ssl.html -## -## Value: File -#exproto.listener.protoname.certfile = "{{ platform_etc_dir }}/certs/cert.pem" - -## Path to the file containing PEM-encoded CA certificates. The CA certificates -## are used during server authentication and when building the client certificate chain. -## -## Value: File -#exproto.listener.protoname.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" - -## The Ephemeral Diffie-Helman key exchange is a very effective way of -## ensuring Forward Secrecy by exchanging a set of keys that never hit -## the wire. Since the DH key is effectively signed by the private key, -## it needs to be at least as strong as the private key. In addition, -## the default DH groups that most of the OpenSSL installations have -## are only a handful (since they are distributed with the OpenSSL -## package that has been built for the operating system it’s running on) -## and hence predictable (not to mention, 1024 bits only). -## In order to escape this situation, first we need to generate a fresh, -## strong DH group, store it in a file and then use the option above, -## to force our SSL application to use the new DH group. Fortunately, -## OpenSSL provides us with a tool to do that. Simply run: -## openssl dhparam -out dh-params.pem 2048 -## -## Value: File -#exproto.listener.protoname.dhfile = "{{ platform_etc_dir }}/certs/dh-params.pem" - -## A server only does x509-path validation in mode verify_peer, -## as it then sends a certificate request to the client (this -## message is not sent if the verify option is verify_none). -## You can then also want to specify option fail_if_no_peer_cert. -## More information at: http://erlang.org/doc/man/ssl.html -## -## Value: verify_peer | verify_none -#exproto.listener.protoname.verify = verify_peer - -## Used together with {verify, verify_peer} by an SSL server. If set to true, -## the server fails if the client does not have a certificate to send, that is, -## sends an empty certificate. -## -## Value: true | false -#exproto.listener.protoname.fail_if_no_peer_cert = true - -## This is the single most important configuration option of an Erlang SSL -## application. Ciphers (and their ordering) define the way the client and -## server encrypt information over the wire, from the initial Diffie-Helman -## key exchange, the session key encryption ## algorithm and the message -## digest algorithm. Selecting a good cipher suite is critical for the -## application’s data security, confidentiality and performance. -## -## The cipher list above offers: -## -## A good balance between compatibility with older browsers. -## It can get stricter for Machine-To-Machine scenarios. -## Perfect Forward Secrecy. -## No old/insecure encryption and HMAC algorithms -## -## Most of it was copied from Mozilla’s Server Side TLS article -## -## Value: Ciphers -#exproto.listener.protoname.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" - -## Ciphers for TLS PSK. -## Note that 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot -## be configured at the same time. -## See 'https://tools.ietf.org/html/rfc4279#section-2'. -#exproto.listener.protoname.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" - -## SSL parameter renegotiation is a feature that allows a client and a server -## to renegotiate the parameters of the SSL connection on the fly. -## RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation, -## you drop support for the insecure renegotiation, prone to MitM attacks. -## -## Value: on | off -#exproto.listener.protoname.secure_renegotiate = off - -## A performance optimization setting, it allows clients to reuse -## pre-existing sessions, instead of initializing new ones. -## Read more about it here. -## -## See: http://erlang.org/doc/man/ssl.html -## -## Value: on | off -#exproto.listener.protoname.reuse_sessions = on - -## An important security setting, it forces the cipher to be set based -## on the server-specified order instead of the client-specified order, -## hence enforcing the (usually more properly configured) security -## ordering of the server administrator. -## -## Value: on | off -#exproto.listener.protoname.honor_cipher_order = on diff --git a/apps/emqx_exproto/priv/emqx_exproto.schema b/apps/emqx_exproto/priv/emqx_exproto.schema deleted file mode 100644 index 4bd215847..000000000 --- a/apps/emqx_exproto/priv/emqx_exproto.schema +++ /dev/null @@ -1,364 +0,0 @@ -%% -*-: erlang -*- - -%%-------------------------------------------------------------------- -%% Services - -{mapping, "exproto.server.http.port", "emqx_exproto.servers", [ - {datatype, integer} -]}. - -{mapping, "exproto.server.https.port", "emqx_exproto.servers", [ - {datatype, integer} -]}. - -{mapping, "exproto.server.https.cacertfile", "emqx_exproto.servers", [ - {datatype, string} -]}. - -{mapping, "exproto.server.https.certfile", "emqx_exproto.servers", [ - {datatype, string} -]}. - -{mapping, "exproto.server.https.keyfile", "emqx_exproto.servers", [ - {datatype, string} -]}. - -{translation, "emqx_exproto.servers", fun(Conf) -> - Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, - Http = case cuttlefish:conf_get("exproto.server.http.port", Conf, undefined) of - undefined -> []; - P1 -> [{http, P1, []}] - end, - Https = case cuttlefish:conf_get("exproto.server.https.port", Conf, undefined) of - undefined -> []; - P2 -> - [{https, P2, - Filter([{ssl, true}, - {certfile, cuttlefish:conf_get("exproto.server.https.certfile", Conf)}, - {keyfile, cuttlefish:conf_get("exproto.server.https.keyfile", Conf)}, - {cacertfile, cuttlefish:conf_get("exproto.server.https.cacertfile", Conf)}])}] - end, - Http ++ Https -end}. - -%%-------------------------------------------------------------------- -%% Listeners - -{mapping, "exproto.listener.$proto.endpoint", "emqx_exproto.listeners", [ - {datatype, string} -]}. - -{mapping, "exproto.listener.$proto.connection_handler_url", "emqx_exproto.listeners", [ - {datatype, string} -]}. - -{mapping, "exproto.listener.$proto.connection_handler_certfile", "emqx_exproto.listeners", [ - {datatype, string} -]}. - -{mapping, "exproto.listener.$proto.connection_handler_cacertfile", "emqx_exproto.listeners", [ - {datatype, string} -]}. - -{mapping, "exproto.listener.$proto.connection_handler_keyfile", "emqx_exproto.listeners", [ - {datatype, string} -]}. - -{mapping, "exproto.listener.$proto.acceptors", "emqx_exproto.listeners", [ - {default, 8}, - {datatype, integer} -]}. - -{mapping, "exproto.listener.$proto.max_connections", "emqx_exproto.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "exproto.listener.$proto.max_conn_rate", "emqx_exproto.listeners", [ - {datatype, integer} -]}. - -{mapping, "exproto.listener.$proto.active_n", "emqx_exproto.listeners", [ - {default, 100}, - {datatype, integer} -]}. - -{mapping, "exproto.listener.$proto.idle_timeout", "emqx_exproto.listeners", [ - {default, "30s"}, - {datatype, {duration, ms}} -]}. - -{mapping, "exproto.listener.$proto.access.$id", "emqx_exproto.listeners", [ - {datatype, string} -]}. - -{mapping, "exproto.listener.$proto.proxy_protocol", "emqx_exproto.listeners", [ - {datatype, flag} -]}. - -{mapping, "exproto.listener.$proto.proxy_protocol_timeout", "emqx_exproto.listeners", [ - {datatype, {duration, ms}} -]}. - -{mapping, "exproto.listener.$proto.backlog", "emqx_exproto.listeners", [ - {datatype, integer}, - {default, 1024} -]}. - -{mapping, "exproto.listener.$proto.send_timeout", "emqx_exproto.listeners", [ - {datatype, {duration, ms}}, - {default, "15s"} -]}. - -{mapping, "exproto.listener.$proto.send_timeout_close", "emqx_exproto.listeners", [ - {datatype, flag}, - {default, on} -]}. - -{mapping, "exproto.listener.$proto.recbuf", "emqx_exproto.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "exproto.listener.$proto.sndbuf", "emqx_exproto.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "exproto.listener.$proto.buffer", "emqx_exproto.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "exproto.listener.$proto.tune_buffer", "emqx_exproto.listeners", [ - {datatype, flag}, - hidden -]}. - -{mapping, "exproto.listener.$proto.nodelay", "emqx_exproto.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -{mapping, "exproto.listener.$proto.reuseaddr", "emqx_exproto.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -%%-------------------------------------------------------------------- -%% TLS Options - -{mapping, "exproto.listener.$proto.tls_versions", "emqx_exproto.listeners", [ - {datatype, string} -]}. - -{mapping, "exproto.listener.$proto.ciphers", "emqx_exproto.listeners", [ - {datatype, string} -]}. - -{mapping, "exproto.listener.$proto.psk_ciphers", "emqx_exproto.listeners", [ - {datatype, string} -]}. - -{mapping, "exproto.listener.$proto.dhfile", "emqx_exproto.listeners", [ - {datatype, string} -]}. - -{mapping, "exproto.listener.$proto.keyfile", "emqx_exproto.listeners", [ - {datatype, string} -]}. - -{mapping, "exproto.listener.$proto.certfile", "emqx_exproto.listeners", [ - {datatype, string} -]}. - -{mapping, "exproto.listener.$proto.cacertfile", "emqx_exproto.listeners", [ - {datatype, string} -]}. - -{mapping, "exproto.listener.$proto.verify", "emqx_exproto.listeners", [ - {datatype, atom} -]}. - -{mapping, "exproto.listener.$proto.fail_if_no_peer_cert", "emqx_exproto.listeners", [ - {datatype, {enum, [true, false]}} -]}. - -{mapping, "exproto.listener.$proto.secure_renegotiate", "emqx_exproto.listeners", [ - {datatype, flag} -]}. - -{mapping, "exproto.listener.$proto.reuse_sessions", "emqx_exproto.listeners", [ - {default, on}, - {datatype, flag} -]}. - -{mapping, "exproto.listener.$proto.honor_cipher_order", "emqx_exproto.listeners", [ - {datatype, flag} -]}. - -{translation, "emqx_exproto.listeners", fun(Conf) -> - - Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, - - Atom = fun(undefined) -> undefined; (S) -> list_to_atom(S) end, - - Access = fun(S) -> - [A, CIDR] = string:tokens(S, " "), - {list_to_atom(A), case CIDR of "all" -> all; _ -> CIDR end} - end, - - AccOpts = fun(Prefix) -> - case cuttlefish_variable:filter_by_prefix(Prefix ++ ".access", Conf) of - [] -> []; - Rules -> [{access_rules, [Access(Rule) || {_, Rule} <- Rules]}] - end - end, - - RateLimit = fun(undefined) -> - undefined; - (Val) -> - [L, D] = string:tokens(Val, ", "), - Limit = case cuttlefish_bytesize:parse(L) of - Sz when is_integer(Sz) -> Sz; - {error, Reason} -> error(Reason) - end, - Duration = case cuttlefish_duration:parse(D, s) of - Secs when is_integer(Secs) -> Secs; - {error, Reason1} -> error(Reason1) - end, - Rate = Limit / Duration, - {Rate, Limit} - end, - - HandlerOpts = fun(Prefix) -> - Opts = - case http_uri:parse(cuttlefish:conf_get(Prefix ++ ".connection_handler_url", Conf)) of - {ok, {http, _, Host, Port, _, _}} -> - [{scheme, http}, {host, Host}, {port, Port}]; - {ok, {https, _, Host, Port, _, _}} -> - [{scheme, https}, {host, Host}, {port, Port}, - {ssl_options, - Filter([{certfile, cuttlefish:conf_get(Prefix ++ ".connection_handler_certfile", Conf)}, - {keyfile, cuttlefish:conf_get(Prefix ++ ".connection_handler_keyfile", Conf)}, - {cacertfile, cuttlefish:conf_get(Prefix ++ ".connection_handler_cacertfile", Conf)} - ])}]; - _ -> - error(invaild_connection_handler_url) - end, - [{handler, Opts}] - end, - - ConnOpts = fun(Prefix) -> - Filter([{active_n, cuttlefish:conf_get(Prefix ++ ".active_n", Conf, undefined)}, - {idle_timeout, cuttlefish:conf_get(Prefix ++ ".idle_timeout", Conf, undefined)}]) - end, - - LisOpts = fun(Prefix) -> - Filter([{acceptors, cuttlefish:conf_get(Prefix ++ ".acceptors", Conf)}, - {max_connections, cuttlefish:conf_get(Prefix ++ ".max_connections", Conf)}, - {max_conn_rate, cuttlefish:conf_get(Prefix ++ ".max_conn_rate", Conf, undefined)}, - {tune_buffer, cuttlefish:conf_get(Prefix ++ ".tune_buffer", Conf, undefined)}, - {proxy_protocol, cuttlefish:conf_get(Prefix ++ ".proxy_protocol", Conf, undefined)}, - {proxy_protocol_timeout, cuttlefish:conf_get(Prefix ++ ".proxy_protocol_timeout", Conf, undefined)} | AccOpts(Prefix)]) - end, - - TcpOpts = fun(Prefix) -> - Filter([{backlog, cuttlefish:conf_get(Prefix ++ ".backlog", Conf, undefined)}, - {send_timeout, cuttlefish:conf_get(Prefix ++ ".send_timeout", Conf, undefined)}, - {send_timeout_close, cuttlefish:conf_get(Prefix ++ ".send_timeout_close", Conf, undefined)}, - {recbuf, cuttlefish:conf_get(Prefix ++ ".recbuf", Conf, undefined)}, - {sndbuf, cuttlefish:conf_get(Prefix ++ ".sndbuf", Conf, undefined)}, - {buffer, cuttlefish:conf_get(Prefix ++ ".buffer", Conf, undefined)}, - {nodelay, cuttlefish:conf_get(Prefix ++ ".nodelay", Conf, true)}, - {reuseaddr, cuttlefish:conf_get(Prefix ++ ".reuseaddr", Conf, undefined)}]) - end, - SplitFun = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end, - MapPSKCiphers = fun(PSKCiphers) -> - lists:map( - fun("PSK-AES128-CBC-SHA") -> {psk, aes_128_cbc, sha}; - ("PSK-AES256-CBC-SHA") -> {psk, aes_256_cbc, sha}; - ("PSK-3DES-EDE-CBC-SHA") -> {psk, '3des_ede_cbc', sha}; - ("PSK-RC4-SHA") -> {psk, rc4_128, sha} - end, PSKCiphers) - end, - SslOpts = fun(Prefix) -> - Versions = case SplitFun(cuttlefish:conf_get(Prefix ++ ".tls_versions", Conf, undefined)) of - undefined -> undefined; - L -> [list_to_atom(V) || V <- L] - end, - TLSCiphers = cuttlefish:conf_get(Prefix++".ciphers", Conf, undefined), - PSKCiphers = cuttlefish:conf_get(Prefix++".psk_ciphers", Conf, undefined), - Ciphers = - case {TLSCiphers, PSKCiphers} of - {undefined, undefined} -> - cuttlefish:invalid(Prefix++".ciphers or "++Prefix++".psk_ciphers is absent"); - {TLSCiphers, undefined} -> - SplitFun(TLSCiphers); - {undefined, PSKCiphers} -> - MapPSKCiphers(SplitFun(PSKCiphers)); - {_TLSCiphers, _PSKCiphers} -> - cuttlefish:invalid(Prefix++".ciphers and "++Prefix++".psk_ciphers cannot be configured at the same time") - end, - UserLookupFun = - case PSKCiphers of - undefined -> undefined; - _ -> {fun emqx_psk:lookup/3, <<>>} - end, - Filter([{versions, Versions}, - {ciphers, Ciphers}, - {user_lookup_fun, UserLookupFun}, - %{handshake_timeout, cuttlefish:conf_get(Prefix ++ ".handshake_timeout", Conf, undefined)}, - {dhfile, cuttlefish:conf_get(Prefix ++ ".dhfile", Conf, undefined)}, - {keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, - {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, - {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)}, - {verify, cuttlefish:conf_get(Prefix ++ ".verify", Conf, undefined)}, - {fail_if_no_peer_cert, cuttlefish:conf_get(Prefix ++ ".fail_if_no_peer_cert", Conf, undefined)}, - {secure_renegotiate, cuttlefish:conf_get(Prefix ++ ".secure_renegotiate", Conf, undefined)}, - {reuse_sessions, cuttlefish:conf_get(Prefix ++ ".reuse_sessions", Conf, undefined)}, - {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)}]) - end, - - UdpOpts = fun(Prefix) -> - Filter([{recbuf, cuttlefish:conf_get(Prefix ++ ".recbuf", Conf, undefined)}, - {sndbuf, cuttlefish:conf_get(Prefix ++ ".sndbuf", Conf, undefined)}, - {buffer, cuttlefish:conf_get(Prefix ++ ".buffer", Conf, undefined)}, - {reuseaddr, cuttlefish:conf_get(Prefix ++ ".reuseaddr", Conf, undefined)}]) - end, - - ParseListenOn = fun(ListenOn) -> - case string:tokens(ListenOn, "://") of - [Port] -> {tcp, list_to_integer(Port)}; - [T, Ip, Port] - when T =:= "tcp"; T =:= "ssl"; - T =:= "udp"; T =:= "dtls" -> - {Atom(T), {Ip, list_to_integer(Port)}} - end - end, - - Listeners = fun(Proto) -> - Prefix = string:join(["exproto","listener", Proto], "."), - Opts = HandlerOpts(Prefix) ++ ConnOpts(Prefix) ++ LisOpts(Prefix), - case cuttlefish:conf_get(Prefix ++ ".endpoint", Conf, undefined) of - undefined -> []; - ListenOn0 -> - case ParseListenOn(ListenOn0) of - {tcp, ListenOn} -> - [{Proto, tcp, ListenOn, [{tcp_options, TcpOpts(Prefix)} | Opts]}]; - {ssl, ListenOn} -> - [{Proto, ssl, ListenOn, [{tcp_options, TcpOpts(Prefix)}, - {ssl_options, SslOpts(Prefix)} | Opts]}]; - {udp, ListenOn} -> - [{Proto, udp, ListenOn, [{udp_options, UdpOpts(Prefix)} | Opts]}]; - {dtls, ListenOn} -> - [{Proto, dtls, ListenOn, [{udp_options, UdpOpts(Prefix)}, - {dtls_options, SslOpts(Prefix)} | Opts]}]; - {_, _} -> - cuttlefish:invalid("Not supported listener type") - end - end - end, - lists:flatten([Listeners(Proto) || {[_, "listener", Proto, "endpoint"], ListenOn} - <- cuttlefish_variable:filter_by_prefix("exproto.listener", Conf)]) -end}. diff --git a/apps/emqx_exproto/rebar.config b/apps/emqx_exproto/rebar.config deleted file mode 100644 index 3fa9f6f8a..000000000 --- a/apps/emqx_exproto/rebar.config +++ /dev/null @@ -1,51 +0,0 @@ -%%-*- mode: erlang -*- -{edoc_opts, [{preprocess, true}]}. - -{erl_opts, [warn_unused_vars, - warn_shadow_vars, - warn_unused_import, - warn_obsolete_guard, - debug_info, - {parse_transform}]}. -{plugins, - [rebar3_proper, - {grpc_plugin, {git, "https://github.com/HJianBo/grpc_plugin", {tag, "v0.10.2"}}} -]}. - -{deps, - [{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.2"}}} - ]}. - -{grpc, - [{type, all}, - {protos, ["priv/protos"]}, - {gpb_opts, [{module_name_prefix, "emqx_"}, - {module_name_suffix, "_pb"}]} -]}. - -{provider_hooks, - [{pre, [{compile, {grpc, gen}}, - {clean, {grpc, clean}}]} -]}. - -{xref_checks, [undefined_function_calls, undefined_functions, - locals_not_used, deprecated_function_calls, - warnings_as_errors, deprecated_functions]}. - -{xref_ignores, [emqx_exproto_pb]}. - -{cover_enabled, true}. -{cover_opts, [verbose]}. -{cover_export_enabled, true}. -{cover_excl_mods, [emqx_exproto_pb, - emqx_exproto_v_1_connection_adapter_client, - emqx_exproto_v_1_connection_adapter_bhvr, - emqx_exproto_v_1_connection_handler_client, - emqx_exproto_v_1_connection_handler_bhvr]}. - -{profiles, - [{test, - [{deps, - []} - ]} -]}. diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src deleted file mode 100644 index 9841f5bcb..000000000 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ /dev/null @@ -1,12 +0,0 @@ -{application, emqx_exproto, - [{description, "EMQ X Extension for Protocol"}, - {vsn, "4.4.0"}, %% strict semver - {modules, []}, - {registered, []}, - {mod, {emqx_exproto_app, []}}, - {applications, [kernel,stdlib,grpc]}, - {env,[]}, - {licenses, ["Apache-2.0"]}, - {maintainers, ["EMQ X Team "]}, - {links, [{"Homepage", "https://emqx.io/"}]} - ]}. diff --git a/apps/emqx_exproto/src/emqx_exproto.erl b/apps/emqx_exproto/src/emqx_exproto.erl deleted file mode 100644 index 453b4989b..000000000 --- a/apps/emqx_exproto/src/emqx_exproto.erl +++ /dev/null @@ -1,187 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_exproto). - --include("emqx_exproto.hrl"). - --export([ start_listeners/0 - , stop_listeners/0 - , start_listener/1 - , start_listener/4 - , stop_listener/4 - , stop_listener/1 - ]). - --export([ start_servers/0 - , stop_servers/0 - , start_server/1 - , stop_server/1 - ]). - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - --spec(start_listeners() -> ok). -start_listeners() -> - Listeners = application:get_env(?APP, listeners, []), - NListeners = [start_connection_handler_instance(Listener) - || Listener <- Listeners], - lists:foreach(fun start_listener/1, NListeners). - --spec(stop_listeners() -> ok). -stop_listeners() -> - Listeners = application:get_env(?APP, listeners, []), - lists:foreach(fun stop_connection_handler_instance/1, Listeners), - lists:foreach(fun stop_listener/1, Listeners). - --spec(start_servers() -> ok). -start_servers() -> - lists:foreach(fun start_server/1, application:get_env(?APP, servers, [])). - --spec(stop_servers() -> ok). -stop_servers() -> - lists:foreach(fun stop_server/1, application:get_env(?APP, servers, [])). - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -start_connection_handler_instance({_Proto, _LisType, _ListenOn, Opts}) -> - Name = name(_Proto, _LisType), - {value, {_, HandlerOpts}, LisOpts} = lists:keytake(handler, 1, Opts), - {SvrAddr, ChannelOptions} = handler_opts(HandlerOpts), - case emqx_exproto_sup:start_grpc_client_channel(Name, SvrAddr, ChannelOptions) of - {ok, _ClientChannelPid} -> - {_Proto, _LisType, _ListenOn, [{handler, Name} | LisOpts]}; - {error, Reason} -> - io:format(standard_error, "Failed to start ~s's connection handler: ~0p~n", - [Name, Reason]), - error(Reason) - end. - -stop_connection_handler_instance({_Proto, _LisType, _ListenOn, _Opts}) -> - Name = name(_Proto, _LisType), - _ = emqx_exproto_sup:stop_grpc_client_channel(Name), - ok. - -start_server({Name, Port, SSLOptions}) -> - case emqx_exproto_sup:start_grpc_server(Name, Port, SSLOptions) of - {ok, _} -> - io:format("Start ~s gRPC server on ~w successfully.~n", - [Name, Port]); - {error, Reason} -> - io:format(standard_error, "Failed to start ~s gRPC server on ~w: ~0p~n", - [Name, Port, Reason]), - error({failed_start_server, Reason}) - end. - -stop_server({Name, Port, _SSLOptions}) -> - ok = emqx_exproto_sup:stop_grpc_server(Name), - io:format("Stop ~s gRPC server on ~w successfully.~n", [Name, Port]). - -start_listener({Proto, LisType, ListenOn, Opts}) -> - Name = name(Proto, LisType), - case start_listener(LisType, Name, ListenOn, Opts) of - {ok, _} -> - io:format("Start ~s listener on ~s successfully.~n", - [Name, format(ListenOn)]); - {error, Reason} -> - io:format(standard_error, "Failed to start ~s listener on ~s: ~0p~n", - [Name, format(ListenOn), Reason]), - error(Reason) - end. - -%% @private -start_listener(LisType, Name, ListenOn, LisOpts) - when LisType =:= tcp; - LisType =:= ssl -> - SockOpts = esockd:parse_opt(LisOpts), - esockd:open(Name, ListenOn, merge_tcp_default(SockOpts), - {emqx_exproto_conn, start_link, [LisOpts-- SockOpts]}); - -start_listener(udp, Name, ListenOn, LisOpts) -> - SockOpts = esockd:parse_opt(LisOpts), - esockd:open_udp(Name, ListenOn, merge_udp_default(SockOpts), - {emqx_exproto_conn, start_link, [LisOpts-- SockOpts]}); - -start_listener(dtls, Name, ListenOn, LisOpts) -> - SockOpts = esockd:parse_opt(LisOpts), - esockd:open_dtls(Name, ListenOn, merge_udp_default(SockOpts), - {emqx_exproto_conn, start_link, [LisOpts-- SockOpts]}). - -stop_listener({Proto, LisType, ListenOn, Opts}) -> - Name = name(Proto, LisType), - StopRet = stop_listener(LisType, Name, ListenOn, Opts), - case StopRet of - ok -> - io:format("Stop ~s listener on ~s successfully.~n", - [Name, format(ListenOn)]); - {error, Reason} -> - io:format(standard_error, "Failed to stop ~s listener on ~s: ~0p~n", - [Name, format(ListenOn), Reason]) - end, - StopRet. - -%% @private -stop_listener(_LisType, Name, ListenOn, _Opts) -> - esockd:close(Name, ListenOn). - -%% @private -name(Proto, LisType) -> - list_to_atom(lists:flatten(io_lib:format("~s:~s", [Proto, LisType]))). - -%% @private -format(Port) when is_integer(Port) -> - io_lib:format("0.0.0.0:~w", [Port]); -format({Addr, Port}) when is_list(Addr) -> - io_lib:format("~s:~w", [Addr, Port]); -format({Addr, Port}) when is_tuple(Addr) -> - io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). - -%% @private -merge_tcp_default(Opts) -> - case lists:keytake(tcp_options, 1, Opts) of - {value, {tcp_options, TcpOpts}, Opts1} -> - [{tcp_options, emqx_misc:merge_opts(?TCP_SOCKOPTS, TcpOpts)} | Opts1]; - false -> - [{tcp_options, ?TCP_SOCKOPTS} | Opts] - end. - -merge_udp_default(Opts) -> - case lists:keytake(udp_options, 1, Opts) of - {value, {udp_options, TcpOpts}, Opts1} -> - [{udp_options, emqx_misc:merge_opts(?UDP_SOCKOPTS, TcpOpts)} | Opts1]; - false -> - [{udp_options, ?UDP_SOCKOPTS} | Opts] - end. - -%% @private -handler_opts(Opts) -> - Scheme = proplists:get_value(scheme, Opts), - Host = proplists:get_value(host, Opts), - Port = proplists:get_value(port, Opts), - SvrAddr = lists:flatten(io_lib:format("~s://~s:~w", [Scheme, Host, Port])), - ClientOpts = case Scheme of - https -> - SslOpts = lists:keydelete(ssl, 1, proplists:get_value(ssl_options, Opts, [])), - #{gun_opts => - #{transport => ssl, - transport_opts => SslOpts}}; - _ -> #{} - end, - {SvrAddr, ClientOpts}. diff --git a/apps/emqx_exproto/src/emqx_exproto_app.erl b/apps/emqx_exproto/src/emqx_exproto_app.erl deleted file mode 100644 index d9598bcd9..000000000 --- a/apps/emqx_exproto/src/emqx_exproto_app.erl +++ /dev/null @@ -1,37 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_exproto_app). - --behaviour(application). - --emqx_plugin(extension). - --export([start/2, prep_stop/1, stop/1]). - -start(_StartType, _StartArgs) -> - {ok, Sup} = emqx_exproto_sup:start_link(), - emqx_exproto:start_servers(), - emqx_exproto:start_listeners(), - {ok, Sup}. - -prep_stop(State) -> - emqx_exproto:stop_servers(), - emqx_exproto:stop_listeners(), - State. - -stop(_State) -> - ok. diff --git a/apps/emqx_exproto/src/emqx_exproto_sup.erl b/apps/emqx_exproto/src/emqx_exproto_sup.erl deleted file mode 100644 index fc70b8131..000000000 --- a/apps/emqx_exproto/src/emqx_exproto_sup.erl +++ /dev/null @@ -1,83 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_exproto_sup). - --behaviour(supervisor). - --export([start_link/0]). - --export([ start_grpc_server/3 - , stop_grpc_server/1 - , start_grpc_client_channel/3 - , stop_grpc_client_channel/1 - ]). - --export([init/1]). - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - --spec start_grpc_server(atom(), inet:port_number(), list()) - -> {ok, pid()} | {error, term()}. -start_grpc_server(Name, Port, SSLOptions) -> - Services = #{protos => [emqx_exproto_pb], - services => #{'emqx.exproto.v1.ConnectionAdapter' => emqx_exproto_gsvr} - }, - Options = case SSLOptions of - [] -> []; - _ -> - [{ssl_options, lists:keydelete(ssl, 1, SSLOptions)}] - end, - grpc:start_server(prefix(Name), Port, Services, Options). - --spec stop_grpc_server(atom()) -> ok. -stop_grpc_server(Name) -> - grpc:stop_server(prefix(Name)). - --spec start_grpc_client_channel( - atom(), - uri_string:uri_string(), - grpc_client:grpc_opts()) -> {ok, pid()} | {error, term()}. -start_grpc_client_channel(Name, SvrAddr, ClientOpts) -> - grpc_client_sup:create_channel_pool(Name, SvrAddr, ClientOpts). - --spec stop_grpc_client_channel(atom()) -> ok. -stop_grpc_client_channel(Name) -> - grpc_client_sup:stop_channel_pool(Name). - -%% @private -prefix(Name) when is_atom(Name) -> - "exproto:" ++ atom_to_list(Name); -prefix(Name) when is_binary(Name) -> - "exproto:" ++ binary_to_list(Name); -prefix(Name) when is_list(Name) -> - "exproto:" ++ Name. - -%%-------------------------------------------------------------------- -%% Supervisor callbacks -%%-------------------------------------------------------------------- - -init([]) -> - %% gRPC Client Pool - PoolSize = emqx_vm:schedulers() * 2, - Pool = emqx_pool_sup:spec([exproto_gcli_pool, hash, PoolSize, - {emqx_exproto_gcli, start_link, []}]), - {ok, {{one_for_one, 10, 5}, [Pool]}}. diff --git a/apps/emqx_exhook/.formatter.exs b/apps/emqx_gateway/.formatter.exs similarity index 100% rename from apps/emqx_exhook/.formatter.exs rename to apps/emqx_gateway/.formatter.exs diff --git a/apps/emqx_gateway/.gitignore b/apps/emqx_gateway/.gitignore new file mode 100644 index 000000000..5bff8a84d --- /dev/null +++ b/apps/emqx_gateway/.gitignore @@ -0,0 +1,25 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ +rebar.lock +src/exproto/emqx_exproto_pb.erl +src/exproto/emqx_exproto_v_1_connection_adapter_bhvr.erl +src/exproto/emqx_exproto_v_1_connection_adapter_client.erl +src/exproto/emqx_exproto_v_1_connection_handler_bhvr.erl +src/exproto/emqx_exproto_v_1_connection_handler_client.erl diff --git a/apps/emqx_gateway/LICENSE b/apps/emqx_gateway/LICENSE new file mode 100644 index 000000000..1f15def74 --- /dev/null +++ b/apps/emqx_gateway/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2021, JianBo He . + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/apps/emqx_gateway/Makefile b/apps/emqx_gateway/Makefile new file mode 100644 index 000000000..b2a54f7dd --- /dev/null +++ b/apps/emqx_gateway/Makefile @@ -0,0 +1,28 @@ +## shallow clone for speed + +REBAR_GIT_CLONE_OPTIONS += --depth 1 +export REBAR_GIT_CLONE_OPTIONS + +REBAR = rebar3 +all: compile + +compile: + $(REBAR) compile + +clean: distclean + +ct: + $(REBAR) as test ct -v + +eunit: + $(REBAR) as test eunit + +xref: + $(REBAR) xref + +cover: + $(REBAR) cover + +distclean: + @rm -rf _build + @rm -f data/app.*.config data/vm.*.args rebar.lock diff --git a/apps/emqx_gateway/README.md b/apps/emqx_gateway/README.md new file mode 100644 index 000000000..4d39190c8 --- /dev/null +++ b/apps/emqx_gateway/README.md @@ -0,0 +1,332 @@ +# emqx_gateway + +***This is a very early prototype application*** for Gateway in EMQ X Broker 5.0 + +## Concept + + EMQ X Gateway Managment + - Gateway-Registry (or Gateway Type) + - *Load + - *UnLoad + - *List + + - Gateway + - *Create + - *Delete + - *Update + - *Stop-And-Start + - *Hot-Upgrade + - *Satrt/Enable + - *Stop/Disable + - Listener + +## ROADMAP + +Gateway v0.1: "Basic Functionals" + - Management support + - Conn/Frame/Protocol Template + - Support Stomp/MQTT-SN/CoAP/LwM2M/ExProto + +Gateway v0.2: "Integration & Friendly Management" + - Hooks & Metrics & Statistic + - HTTP APIs + - Management in the cluster + - Integrate with AuthN + - Integrate with `emqx_config` + - Improve hocon config + - Mountpoint & ClientInfo's Metadata + - The Concept Review + +Gateway v0.3: "Fault tolerance and high availability" + - A common session modoule for message delivery policy + - The restart mechanism for gateway-instance + - Consistency of cluster state + - Configuration hot update + +Gateway v1.0: "Best practices for each type of protocol" + - CoAP + - Stomp + - MQTT-SN + - LwM2M + +### Compatible with EMQ X + +> Why we need to compatible + +1. Authentication +2. Hooks/Event system +3. Messages Mode & Rule Engine +4. Cluster registration +5. Metrics & Statistic + +> How to do it + +> + +### User Interface + +#### Configurations + +```hocon +gateway { + + ## ... some confs for top scope + .. + ## End. + + ## Gateway Instances + + lwm2m[.name] { + + ## variable support + mountpoint: lwm2m/%e/ + + lifetime_min: 1s + lifetime_max: 86400s + #qmode_time_window: 22 + #auto_observe: off + + #update_msg_publish_condition: contains_object_list + + xml_dir: {{ platform_etc_dir }}/lwm2m_xml + + clientinfo_override: { + username: ${register.opts.uname} + password: ${register.opts.passwd} + clientid: ${epn} + } + + #authenticator: allow_anonymous + authenticator: [ + { + type: auth-http + method: post + //?? how to generate clientinfo ?? + params: $client.credential + } + ] + + translator: { + downlink: "dn/#" + uplink: { + notify: "up/notify" + response: "up/resp" + register: "up/resp" + update: "up/reps" + } + } + + %% ?? listener.$type.name ?? + listener.udp[.name] { + listen_on: 0.0.0.0:5683 + max_connections: 1024000 + max_conn_rate: 1000 + ## ?? udp keepalive in socket level ??? + #keepalive: + ## ?? udp proxy-protocol in socket level ??? + #proxy_protocol: on + #proxy_timeout: 30s + recbuf: 2KB + sndbuf: 2KB + buffer: 2KB + tune_buffer: off + #access: allow all + read_packets: 20 + } + + listener.dtls[.name] { + listen_on: 0.0.0.0:5684 + ... + } + } + + ## The CoAP Gateway + coap[.name] { + + #enable_stats: on + + authenticator: [ + ... + ] + + listener.udp[.name] { + ... + } + + listener.dtls[.name] { + ... + } +} + + ## The Stomp Gateway + stomp[.name] { + + allow_anonymous: true + + default_user.login: guest + default_user.passcode: guest + + frame.max_headers: 10 + frame.max_header_length: 1024 + frame.max_body_length: 8192 + + listener.tcp[.name] { + ... + } + + listener.ssl[.name] { + ... + } + } + + exproto[.name] { + + proto_name: DL-648 + + authenticators: [...] + + adapter: { + type: grpc + options: { + listen_on: 9100 + } + } + + handler: { + type: grpc + options: { + url: + } + } + + listener.tcp[.name] { + ... + } + } + + ## ============================ Enterpise gateways + + ## The JT/T 808 Gateway + jtt808[.name] { + + idle_timeout: 30s + enable_stats: on + max_packet_size: 8192 + + clientinfo_override: { + clientid: $phone + username: xxx + password: xxx + } + + authenticator: [ + { + type: auth-http + method: post + params: $clientinfo.credential + } + ] + + translator: { + subscribe: [jt808/%c/dn] + publish: [jt808/%c/up] + } + + listener.tcp[.name] { + ... + } + + listener.ssl[.name] { + ... + } + } + + gbt32960[.name] { + + frame.max_length: 8192 + retx_interval: 8s + retx_max_times: 3 + message_queue_len: 10 + + authenticators: [...] + + translator: { + ## upstream + login: gbt32960/${vin}/upstream/vlogin + logout: gbt32960/${vin}/upstream/vlogout + informing: gbt32960/${vin}/upstream/info + reinforming: gbt32960/${vin}/upstream/reinfo + ## downstream + downstream: gbt32960/${vin}/dnstream + response: gbt32960/${vin}/upstream/response + } + + listener.tcp[.name] { + ... + } + + listener.ssl[.name] { + ... + } + } + + privtcp[.name] { + + max_packet_size: 65535 + idle_timeout: 15s + + enable_stats: on + + force_gc_policy: 1000|1MB + force_shutdown_policy: 8000|800MB + + translator: { + up_topic: tcp/%c/up + dn_topic: tcp/%c/dn + } + + listener.tcp[.name]: { + ... + } + } +} +``` + +#### CLI + +##### Gateway + +```bash +## List all started gateway and gateway-instance +emqx_ctl gateway list +emqx_ctl gateway lookup +emqx_ctl gateway stop +emqx_ctl gateway start + +emqx_ctl gateway-registry re-searching +emqx_ctl gateway-registry list + +emqx_ctl gateway-clients list +emqx_ctl gateway-clients show +emqx_ctl gateway-clients kick + +## Banned ?? +emqx_ctl gateway-banned + +## Metrics +emqx_ctl gateway-metrics [] +``` + +#### Mangement by HTTP-API/Dashboard/ + +#### How to integrate a protocol to your platform + +### Develop your protocol gateway + +There are 3 way to create your protocol gateway for EMQ X 5.0: + +1. Use Erlang to create a new emqx plugin to handle all of protocol packets (same as v5.0 before) + +2. Based on the emqx-gateway-impl-bhvr and emqx-gateway + +3. Use the gRPC Gateway diff --git a/apps/emqx_coap/etc/emqx_coap.conf b/apps/emqx_gateway/etc/emqx_coap.conf similarity index 100% rename from apps/emqx_coap/etc/emqx_coap.conf rename to apps/emqx_gateway/etc/emqx_coap.conf diff --git a/apps/emqx_exhook/etc/emqx_exhook.conf b/apps/emqx_gateway/etc/emqx_exhook.conf similarity index 100% rename from apps/emqx_exhook/etc/emqx_exhook.conf rename to apps/emqx_gateway/etc/emqx_exhook.conf diff --git a/apps/emqx_gateway/etc/emqx_gateway.conf b/apps/emqx_gateway/etc/emqx_gateway.conf new file mode 100644 index 000000000..b6dbff834 --- /dev/null +++ b/apps/emqx_gateway/etc/emqx_gateway.conf @@ -0,0 +1,130 @@ +##-------------------------------------------------------------------- +## EMQ X Gateway configurations +##-------------------------------------------------------------------- + +gateway: { + + stomp.1: { + frame: { + max_headers: 10 + max_headers_length: 1024 + max_body_length: 8192 + } + + clientinfo_override: { + username: "${Packet.headers.login}" + password: "${Packet.headers.passcode}" + } + + authenticator: allow_anonymous + + listener.tcp.1: { + bind: 61613 + acceptors: 16 + max_connections: 1024000 + max_conn_rate: 1000 + active_n: 100 + } + } + + coap.1: { + enable_stats: false + authenticator: allow_anonymous + heartbeat: 30s + resource: mqtt + notify_type: qos + subscribe_qos: qos0 + publish_qos: qos1 + listener.udp.1: { + bind: 5687 + } + } + + coap.2: { + enable_stats: false + authenticator: allow_anonymous + heartbeat: 30s + resource: pubsub + notify_type: non + subscribe_qos: qos2 + publish_qos: coap + listener.udp.1: { + bind: 5683 + } + } + + mqttsn.1: { + ## The MQTT-SN Gateway ID in ADVERTISE message. + gateway_id: 1 + + ## Enable broadcast this gateway to WLAN + broadcast: true + + ## To control whether write statistics data into ETS table + ## for dashbord to read. + enable_stats: true + + ## To control whether accept and process the received + ## publish message with qos=-1. + enable_qos3: true + + ## Idle timeout for a MQTT-SN channel + idle_timeout: 30s + + ## The pre-defined topic name corresponding to the pre-defined topic + ## id of N. + ## Note that the pre-defined topic id of 0 is reserved. + predefined: [ + { id: 1 + topic: "/predefined/topic/name/hello" + }, + { id: 2 + topic: "/predefined/topic/name/nice" + } + ] + + ### ClientInfo override + clientinfo_override: { + username: "mqtt_sn_user" + password: "abc" + } + + listener.udp.1: { + bind: 1884 + max_connections: 10240000 + max_conn_rate: 1000 + } + } + + ## Extension Protocol Gateway + exproto.1: { + + ## The gRPC server to accept requests + server: { + bind: 9100 + #ssl.keyfile: + #ssl.certfile: + #ssl.cacertfile: + } + + handler: { + address: "http://127.0.0.1:9001" + #ssl.keyfile: + #ssl.certfile: + #ssl.cacertfile: + } + + authenticator: allow_anonymous + + listener.tcp.1: { + bind: 7993 + acceptors: 8 + max_connections: 10240 + max_conn_rate: 1000 + } + + #listener.ssl.1: {} + #listener.udp.1: {} + #listener.dtls.1: {} + } +} diff --git a/apps/emqx_lwm2m/etc/emqx_lwm2m.conf b/apps/emqx_gateway/etc/emqx_lwm2m.conf similarity index 100% rename from apps/emqx_lwm2m/etc/emqx_lwm2m.conf rename to apps/emqx_gateway/etc/emqx_lwm2m.conf diff --git a/apps/emqx_coap/priv/emqx_coap.schema b/apps/emqx_gateway/etc/priv/emqx_coap.schema similarity index 100% rename from apps/emqx_coap/priv/emqx_coap.schema rename to apps/emqx_gateway/etc/priv/emqx_coap.schema diff --git a/apps/emqx_exhook/priv/emqx_exhook.schema b/apps/emqx_gateway/etc/priv/emqx_exhook.schema similarity index 100% rename from apps/emqx_exhook/priv/emqx_exhook.schema rename to apps/emqx_gateway/etc/priv/emqx_exhook.schema diff --git a/apps/emqx_lwm2m/priv/emqx_lwm2m.schema b/apps/emqx_gateway/etc/priv/emqx_lwm2m.schema similarity index 100% rename from apps/emqx_lwm2m/priv/emqx_lwm2m.schema rename to apps/emqx_gateway/etc/priv/emqx_lwm2m.schema diff --git a/apps/emqx_exhook/priv/protos/exhook.proto b/apps/emqx_gateway/etc/priv/exhook.proto similarity index 97% rename from apps/emqx_exhook/priv/protos/exhook.proto rename to apps/emqx_gateway/etc/priv/exhook.proto index 72ba26581..97a011352 100644 --- a/apps/emqx_exhook/priv/protos/exhook.proto +++ b/apps/emqx_gateway/etc/priv/exhook.proto @@ -40,7 +40,7 @@ service HookProvider { rpc OnClientAuthenticate(ClientAuthenticateRequest) returns (ValuedResponse) {}; - rpc OnClientCheckAcl(ClientCheckAclRequest) returns (ValuedResponse) {}; + rpc OnClientAuthorize(ClientAuthorizeRequest) returns (ValuedResponse) {}; rpc OnClientSubscribe(ClientSubscribeRequest) returns (EmptySuccess) {}; @@ -123,18 +123,18 @@ message ClientAuthenticateRequest { bool result = 2; } -message ClientCheckAclRequest { +message ClientAuthorizeRequest { ClientInfo clientinfo = 1; - enum AclReqType { + enum AuthzReqType { PUBLISH = 0; SUBSCRIBE = 1; } - AclReqType type = 2; + AuthzReqType type = 2; string topic = 3; @@ -253,7 +253,7 @@ message ValuedResponse { oneof value { - // Boolean result, used on the 'client.authenticate', 'client.check_acl' hooks + // Boolean result, used on the 'client.authenticate', 'client.authorize' hooks bool bool_result = 3; // Message result, used on the 'message.*' hooks @@ -279,7 +279,7 @@ message HookSpec { // Available value: // "client.connect", "client.connack" // "client.connected", "client.disconnected" - // "client.authenticate", "client.check_acl" + // "client.authenticate", "client.authorize" // "client.subscribe", "client.unsubscribe" // // "session.created", "session.subscribed" diff --git a/apps/emqx_psk_file/src/emqx_psk_file_app.erl b/apps/emqx_gateway/include/emqx_gateway.hrl similarity index 54% rename from apps/emqx_psk_file/src/emqx_psk_file_app.erl rename to apps/emqx_gateway/include/emqx_gateway.hrl index 934ffe49e..35fad7f23 100644 --- a/apps/emqx_psk_file/src/emqx_psk_file_app.erl +++ b/apps/emqx_gateway/include/emqx_gateway.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -14,20 +14,22 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_psk_file_app). +-ifndef(EMQX_GATEWAY_HRL). +-define(EMQX_GATEWAY_HRL, 1). --behaviour(application). +-type instance_id() :: atom(). +-type gateway_type() :: atom(). --emqx_plugin(?MODULE). +%% @doc The Gateway Instace defination +-type instance() :: + #{ id := instance_id() + , type := gateway_type() + , name := binary() + , descr => binary() | undefined + %% Appears only in creating or detailed info + , rawconf => map() + %% Appears only in getting instance status/info + , status => stopped | running + }. -%% Application callbacks --export([start/2, stop/1]). - -start(_StartType, _StartArgs) -> - {ok, Sup} = emqx_psk_file_sup:start_link(), - _ = emqx_psk_file:load( - application:get_all_env(emqx_psk_file)), - {ok, Sup}. - -stop(_State) -> - emqx_psk_file:unload(). +-endif. diff --git a/apps/emqx_sn/mix.exs b/apps/emqx_gateway/mix.exs similarity index 58% rename from apps/emqx_sn/mix.exs rename to apps/emqx_gateway/mix.exs index f78ca2061..a2bbc6f44 100644 --- a/apps/emqx_sn/mix.exs +++ b/apps/emqx_gateway/mix.exs @@ -1,9 +1,9 @@ -defmodule EMQXSn.MixProject do +defmodule EMQXStomp.MixProject do use Mix.Project def project do [ - app: :emqx_sn, + app: :emqx_gateway, version: "4.4.0", build_path: "../../_build", config_path: "../../config/config.exs", @@ -12,19 +12,23 @@ defmodule EMQXSn.MixProject do elixir: "~> 1.12", start_permanent: Mix.env() == :prod, deps: deps(), - description: "EMQ X MQTT-SN Plugin" + description: "The Gateway Management Application" ] end def application do [ - mod: {:emqx_sn_app, []}, + registered: [:], + mod: {:emqx_gateway_app, []}, extra_applications: [:logger] ] end defp deps do [ + {:emqx, in_umbrella: true, runtime: false}, + {:lwm2m_coap, github: "emqx/lwm2m-coap", tag: "v2.0.0"}, + {:grpc, github: "emqx/grpc-erl", tag: "0.6.2"}, {:esockd, github: "emqx/esockd", tag: "5.7.4"} ] end diff --git a/apps/emqx_gateway/rebar.config b/apps/emqx_gateway/rebar.config new file mode 100644 index 000000000..8a0ad51e8 --- /dev/null +++ b/apps/emqx_gateway/rebar.config @@ -0,0 +1,29 @@ +{erl_opts, [debug_info]}. +{deps, [ + {lwm2m_coap, {git, "https://github.com/emqx/lwm2m-coap", {tag, "v2.0.0"}}}, + {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.2"}}} +]}. + +{plugins, [ + {grpc_plugin, {git, "https://github.com/HJianBo/grpc_plugin", {tag, "v0.10.2"}}} +]}. + +{grpc, + [{protos, ["src/exproto/protos"]}, + {out_dir, "src/exproto/"}, + {gpb_opts, [{module_name_prefix, "emqx_"}, + {module_name_suffix, "_pb"}]} +]}. + +{provider_hooks, + [{pre, [{compile, {grpc, gen}}, + {clean, {grpc, clean}}]} +]}. + +{xref_ignores, [emqx_exproto_pb]}. + +{cover_excl_mods, [emqx_exproto_pb, + emqx_exproto_v_1_connection_adapter_client, + emqx_exproto_v_1_connection_adapter_bhvr, + emqx_exproto_v_1_connection_handler_client, + emqx_exproto_v_1_connection_handler_bhvr]}. diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl new file mode 100644 index 000000000..1a032a017 --- /dev/null +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl @@ -0,0 +1,97 @@ +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The Gateway channel behavior +%% +%% This module does not export any functions at the moment. +%% It is only used to standardize the implement of emqx_foo_channel.erl +%% module if it integrated with emqx_gateway_conn module +-module(emqx_gateway_channel). + +-type channel() :: any(). + +%%-------------------------------------------------------------------- +%% Info & Stats + +%% @doc Get the channel detailed infomation. +-callback info(channel()) -> emqx_types:infos(). + +-callback info(Key :: atom() | [atom()], channel()) -> any(). + +%% @doc Get the channel statistic items +-callback stats(channel()) -> emqx_types:stats(). + +%%-------------------------------------------------------------------- +%% Init + +%% @doc Initialize the channel state +-callback init(emqx_types:conniinfo(), map()) -> channel(). + +%%-------------------------------------------------------------------- +%% Handles + +-type conn_state() :: idle | connecting | connected | disconnected | atom(). + +-type reply() :: {outgoing, emqx_gateway_frame:packet()} + | {outgoing, [emqx_gateway_frame:packet()]} + | {event, conn_state() | updated} + | {close, Reason :: atom()}. + +-type replies() :: reply() | [reply()]. + +%% @doc Handle the incoming frame +-callback handle_in(emqx_gateway_frame:frame() | {frame_error, any()}, + channel()) + -> {ok, channel()} + | {ok, replies(), channel()} + | {shutdown, Reason :: any(), channel()} + | {shutdown, Reason :: any(), replies(), channel()}. + +%% @doc Handle the outgoing messages dispatched from PUB/SUB system +-callback handle_deliver(list(emqx_types:deliver()), channel()) + -> {ok, channel()} + | {ok, replies(), channel()}. + +%% @doc Handle the timeout event +-callback handle_timeout(reference(), Msg :: any(), channel()) + -> {ok, channel()} + | {ok, replies(), channel()} + | {shutdown, Reason :: any(), channel()}. + +%% @doc Handle the custom gen_server:call/2 for its connection process +-callback handle_call(Req :: any(), channel()) + -> {reply, Reply :: any(), channel()} + | {shutdown, Reason :: any(), Reply :: any(), channel()} + | {shutdown, Reason :: any(), Reply :: any(), + emqx_gateway_frame:frame(), channel()}. + +%% @doc Handle the custom gen_server:cast/2 for its connection process +-callback handle_cast(Req :: any(), channel()) + -> ok + | {ok, channel()} + | {shutdown, Reason :: any(), channel()}. + +%% @doc Handle the custom process messages for its connection process +-callback handle_info(Info :: any(), channel()) + -> ok + | {ok, channel()} + | {shutdown, Reason :: any(), channel()}. + +%%-------------------------------------------------------------------- +%% Terminate + +%% @doc The callback for process terminated +-callback terminate(any(), channel()) -> ok. + diff --git a/apps/emqx_exproto/src/emqx_exproto_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl similarity index 58% rename from apps/emqx_exproto/src/emqx_exproto_conn.erl rename to apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl index da655bcb4..7349ec310 100644 --- a/apps/emqx_exproto/src/emqx_exproto_conn.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -14,13 +14,13 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% TCP/TLS/UDP/DTLS Connection --module(emqx_exproto_conn). +%% @doc The behavior abstrat for TCP based gateway conn +-module(emqx_gateway_conn). -include_lib("emqx/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). --logger_header("[ExProto Conn]"). +-logger_header("[PGW-Conn]"). %% API -export([ start_link/3 @@ -37,7 +37,7 @@ ]). %% Callback --export([init/4]). +-export([init/6]). %% Sys callbacks -export([ system_continue/3 @@ -47,9 +47,8 @@ ]). %% Internal callback --export([wakeup_from_hib/2]). +-export([wakeup_from_hib/2, recvloop/2]). --import(emqx_misc, [start_timer/2]). -record(state, { %% TCP/SSL/UDP/DTLS Wrapped Socket @@ -62,15 +61,16 @@ sockstate :: emqx_types:sockstate(), %% The {active, N} option active_n :: pos_integer(), - %% BACKW: e4.2.0-e4.2.1 - %% We should remove it - sendfun :: function() | undefined, %% Limiter limiter :: maybe(emqx_limiter:limiter()), %% Limit Timer limit_timer :: maybe(reference()), + %% Parse State + parse_state :: emqx_gateway_frame:parse_state(), + %% Serialize options + serialize :: emqx_gateway_frame:serialize_opts(), %% Channel State - channel :: emqx_exproto_channel:channel(), + channel :: emqx_gateway_channel:channel(), %% GC State gc_state :: maybe(emqx_gc:gc_state()), %% Stats Timer @@ -78,12 +78,17 @@ %% Idle Timeout idle_timeout :: integer(), %% Idle Timer - idle_timer :: maybe(reference()) + idle_timer :: maybe(reference()), + %% OOM Policy + oom_policy :: maybe(emqx_types:oom_policy()), + %% Frame Module + frame_mod :: atom(), + %% Channel Module + chann_mod :: atom() }). -type(state() :: #state{}). --define(ACTIVE_N, 100). -define(INFO_KEYS, [socktype, peername, sockname, sockstate, active_n]). -define(CONN_STATS, [recv_pkt, recv_msg, send_pkt, send_msg]). -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). @@ -96,22 +101,23 @@ , handle_msg/2 , shutdown/3 , stop/3 + , parse_incoming/3 ]}). %% udp start_link(Socket = {udp, _SockPid, _Sock}, Peername, Options) -> - Args = [self(), Socket, Peername, Options], + Args = [self(), Socket, Peername, Options] ++ callback_modules(Options), {ok, proc_lib:spawn_link(?MODULE, init, Args)}; %% tcp/ssl/dtls start_link(esockd_transport, Sock, Options) -> Socket = {esockd_transport, Sock}, - case esockd_transport:peername(Sock) of - {ok, Peername} -> - Args = [self(), Socket, Peername, Options], - {ok, proc_lib:spawn_link(?MODULE, init, Args)}; - R = {error, _} -> R - end. + Args = [self(), Socket, undefined, Options] ++ callback_modules(Options), + {ok, proc_lib:spawn_link(?MODULE, init, Args)}. + +callback_modules(Options) -> + [maps:get(frame_mod, Options), + maps:get(chann_mod, Options)]. %%-------------------------------------------------------------------- %% API @@ -121,8 +127,8 @@ start_link(esockd_transport, Sock, Options) -> -spec(info(pid()|state()) -> emqx_types:infos()). info(CPid) when is_pid(CPid) -> call(CPid, info); -info(State = #state{channel = Channel}) -> - ChanInfo = emqx_exproto_channel:info(Channel), +info(State = #state{chann_mod = ChannMod, channel = Channel}) -> + ChanInfo = ChannMod:info(Channel), SockInfo = maps:from_list( info(?INFO_KEYS, State)), ChanInfo#{sockinfo => SockInfo}. @@ -143,14 +149,15 @@ info(active_n, #state{active_n = ActiveN}) -> -spec(stats(pid()|state()) -> emqx_types:stats()). stats(CPid) when is_pid(CPid) -> call(CPid, stats); -stats(#state{socket = Socket, +stats(#state{socket = Socket, + chann_mod = ChannMod, channel = Channel}) -> SockStats = case esockd_getstat(Socket, ?SOCK_STATS) of {ok, Ss} -> Ss; {error, _} -> [] end, ConnStats = emqx_pd:get_counters(?CONN_STATS), - ChanStats = emqx_exproto_channel:stats(Channel), + ChanStats = ChannMod:stats(Channel), ProcStats = emqx_misc:proc_stats(), lists:append([SockStats, ConnStats, ChanStats, ProcStats]). @@ -170,6 +177,12 @@ stop(Pid) -> %% Wrapped funcs %%-------------------------------------------------------------------- +esockd_peername({udp, _SockPid, _Sock}, Peername) -> + Peername; +esockd_peername({esockd_transport, Sock}, _Peername) -> + {ok, Peername} = esockd_transport:ensure_ok_or_exit(peername, [Sock]), + Peername. + esockd_wait(Socket = {udp, _SockPid, _Sock}) -> {ok, Socket}; esockd_wait({esockd_transport, Sock}) -> @@ -208,29 +221,28 @@ esockd_getstat({udp, _SockPid, Sock}, Stats) -> esockd_getstat({esockd_transport, Sock}, Stats) -> esockd_transport:getstat(Sock, Stats). -send(Data, #state{socket = {udp, _SockPid, Sock}, peername = {Ip, Port}}) -> +esockd_send(Data, #state{socket = {udp, _SockPid, Sock}, + peername = {Ip, Port}}) -> gen_udp:send(Sock, Ip, Port, Data); -send(Data, #state{socket = {esockd_transport, Sock}}) -> +esockd_send(Data, #state{socket = {esockd_transport, Sock}}) -> esockd_transport:async_send(Sock, Data). %%-------------------------------------------------------------------- %% callbacks %%-------------------------------------------------------------------- --define(DEFAULT_GC_OPTS, #{count => 1000, bytes => 1024*1024}). --define(DEFAULT_IDLE_TIMEOUT, 30000). --define(DEFAULT_OOM_POLICY, #{max_heap_size => 4194304,message_queue_len => 32000}). - -init(Parent, WrappedSock, Peername, Options) -> +init(Parent, WrappedSock, Peername0, Options, FrameMod, ChannMod) -> case esockd_wait(WrappedSock) of {ok, NWrappedSock} -> - run_loop(Parent, init_state(NWrappedSock, Peername, Options)); + Peername = esockd_peername(NWrappedSock, Peername0), + run_loop(Parent, init_state(NWrappedSock, Peername, + Options, FrameMod, ChannMod)); {error, Reason} -> ok = esockd_close(WrappedSock), exit_on_sock_error(Reason) end. -init_state(WrappedSock, Peername, Options) -> +init_state(WrappedSock, Peername, Options, FrameMod, ChannMod) -> {ok, Sockname} = esockd_ensure_ok_or_exit(sockname, WrappedSock), Peercert = esockd_ensure_ok_or_exit(peercert, WrappedSock), ConnInfo = #{socktype => esockd_type(WrappedSock), @@ -239,36 +251,43 @@ init_state(WrappedSock, Peername, Options) -> peercert => Peercert, conn_mod => ?MODULE }, - - ActiveN = proplists:get_value(active_n, Options, ?ACTIVE_N), - + ActiveN = emqx_gateway_utils:active_n(Options), %% FIXME: %%Limiter = emqx_limiter:init(Options), - - Channel = emqx_exproto_channel:init(ConnInfo, Options), - - GcState = emqx_gc:init(?DEFAULT_GC_OPTS), - - IdleTimeout = proplists:get_value(idle_timeout, Options, ?DEFAULT_IDLE_TIMEOUT), - IdleTimer = start_timer(IdleTimeout, idle_timeout), + Limiter = undefined, + FrameOpts = emqx_gateway_utils:frame_options(Options), + ParseState = FrameMod:initial_parse_state(FrameOpts), + Serialize = FrameMod:serialize_opts(), + Channel = ChannMod:init(ConnInfo, Options), + GcState = emqx_gateway_utils:init_gc_state(Options), + StatsTimer = emqx_gateway_utils:stats_timer(Options), + IdleTimeout = emqx_gateway_utils:idle_timeout(Options), + OomPolicy = emqx_gateway_utils:oom_policy(Options), + IdleTimer = emqx_misc:start_timer(IdleTimeout, idle_timeout), #state{socket = WrappedSock, peername = Peername, sockname = Sockname, sockstate = idle, active_n = ActiveN, - sendfun = undefined, - limiter = undefined, + limiter = Limiter, + parse_state = ParseState, + serialize = Serialize, channel = Channel, gc_state = GcState, - stats_timer = undefined, + stats_timer = StatsTimer, idle_timeout = IdleTimeout, - idle_timer = IdleTimer + idle_timer = IdleTimer, + oom_policy = OomPolicy, + frame_mod = FrameMod, + chann_mod = ChannMod }. run_loop(Parent, State = #state{socket = Socket, - peername = Peername}) -> + peername = Peername, + oom_policy = OomPolicy + }) -> emqx_logger:set_metadata_peername(esockd:format(Peername)), - _ = emqx_misc:tune_heap_size(?DEFAULT_OOM_POLICY), + _ = emqx_misc:tune_heap_size(OomPolicy), case activate_socket(State) of {ok, NState} -> hibernate(Parent, NState); @@ -292,33 +311,42 @@ exit_on_sock_error(Reason) -> recvloop(Parent, State = #state{idle_timeout = IdleTimeout}) -> receive - {system, From, Request} -> - sys:handle_system_msg(Request, From, Parent, ?MODULE, [], State); - {'EXIT', Parent, Reason} -> - terminate(Reason, State); Msg -> - process_msg([Msg], Parent, ensure_stats_timer(IdleTimeout, State)) + handle_recv(Msg, Parent, State) after - IdleTimeout -> + IdleTimeout + 100 -> hibernate(Parent, cancel_stats_timer(State)) end. +handle_recv({system, From, Request}, Parent, State) -> + sys:handle_system_msg(Request, From, Parent, ?MODULE, [], State); +handle_recv({'EXIT', Parent, Reason}, Parent, State) -> + %% FIXME: it's not trapping exit, should never receive an EXIT + terminate(Reason, State); +handle_recv(Msg, Parent, State = #state{idle_timeout = IdleTimeout}) -> + case process_msg([Msg], ensure_stats_timer(IdleTimeout, State)) of + {ok, NewState} -> + ?MODULE:recvloop(Parent, NewState); + {stop, Reason, NewSate} -> + terminate(Reason, NewSate) + end. + hibernate(Parent, State) -> proc_lib:hibernate(?MODULE, wakeup_from_hib, [Parent, State]). %% Maybe do something here later. -wakeup_from_hib(Parent, State) -> recvloop(Parent, State). +wakeup_from_hib(Parent, State) -> + ?MODULE:recvloop(Parent, State). %%-------------------------------------------------------------------- %% Ensure/cancel stats timer --compile({inline, [ensure_stats_timer/2]}). ensure_stats_timer(Timeout, State = #state{stats_timer = undefined}) -> - State#state{stats_timer = start_timer(Timeout, emit_stats)}; + State#state{stats_timer = emqx_misc:start_timer(Timeout, emit_stats)}; ensure_stats_timer(_Timeout, State) -> State. --compile({inline, [cancel_stats_timer/1]}). -cancel_stats_timer(State = #state{stats_timer = TRef}) when is_reference(TRef) -> +cancel_stats_timer(State = #state{stats_timer = TRef}) + when is_reference(TRef) -> ok = emqx_misc:cancel_timer(TRef), State#state{stats_timer = undefined}; cancel_stats_timer(State) -> State. @@ -326,25 +354,33 @@ cancel_stats_timer(State) -> State. %%-------------------------------------------------------------------- %% Process next Msg -process_msg([], Parent, State) -> recvloop(Parent, State); - -process_msg([Msg|More], Parent, State) -> - case catch handle_msg(Msg, State) of - ok -> - process_msg(More, Parent, State); - {ok, NState} -> - process_msg(More, Parent, NState); - {ok, Msgs, NState} -> - process_msg(append_msg(More, Msgs), Parent, NState); - {stop, Reason} -> - terminate(Reason, State); - {stop, Reason, NState} -> - terminate(Reason, NState); - {'EXIT', Reason} -> - terminate(Reason, State) +process_msg([], State) -> + {ok, State}; +process_msg([Msg|More], State) -> + try + case handle_msg(Msg, State) of + ok -> + process_msg(More, State); + {ok, NState} -> + process_msg(More, NState); + {ok, Msgs, NState} -> + process_msg(append_msg(More, Msgs), NState); + {stop, Reason, NState} -> + {stop, Reason, NState} + end + catch + exit : normal -> + {stop, normal, State}; + exit : shutdown -> + {stop, shutdown, State}; + exit : {shutdown, _} = Shutdown -> + {stop, Shutdown, State}; + Exception : Context : Stack -> + {stop, #{exception => Exception, + context => Context, + stacktrace => Stack}, State} end. --compile({inline, [append_msg/2]}). append_msg([], Msgs) when is_list(Msgs) -> Msgs; append_msg([], Msg) -> [Msg]; @@ -373,10 +409,19 @@ handle_msg({'$gen_cast', Req}, State) -> with_channel(handle_cast, [Req], State); handle_msg({datagram, _SockPid, Data}, State) -> - process_incoming(Data, State); + parse_incoming(Data, State); -handle_msg({Inet, _Sock, Data}, State) when Inet == tcp; Inet == ssl -> - process_incoming(Data, State); +handle_msg({Inet, _Sock, Data}, State) + when Inet == tcp; + Inet == ssl -> + parse_incoming(Data, State); + +handle_msg({incoming, Packet}, + State = #state{idle_timer = IdleTimer}) -> + IdleTimer /= undefined andalso + emqx_misc:cancel_timer(IdleTimer), + NState = State#state{idle_timer = undefined}, + handle_incoming(Packet, NState); handle_msg({outgoing, Data}, State) -> handle_outgoing(Data, State); @@ -426,21 +471,35 @@ handle_msg({close, Reason}, State) -> ?LOG(debug, "Force to close the socket due to ~p", [Reason]), handle_info({sock_closed, Reason}, close_socket(State)); -handle_msg({event, connected}, State = #state{channel = Channel}) -> - ClientId = emqx_exproto_channel:info(clientid, Channel), - emqx_cm:insert_channel_info(ClientId, info(State), stats(State)); +handle_msg({event, connected}, State = #state{ + chann_mod = ChannMod, + channel = Channel}) -> + Ctx = ChannMod:info(ctx, Channel), + ClientId = ChannMod:info(clientid, Channel), + emqx_gateway_ctx:insert_channel_info( + Ctx, + ClientId, + info(State), + stats(State) + ); -handle_msg({event, disconnected}, State = #state{channel = Channel}) -> - ClientId = emqx_exproto_channel:info(clientid, Channel), - emqx_cm:set_chan_info(ClientId, info(State)), - emqx_cm:connection_closed(ClientId), +handle_msg({event, disconnected}, State = #state{ + chann_mod = ChannMod, + channel = Channel}) -> + Ctx = ChannMod:info(ctx, Channel), + ClientId = ChannMod:info(clientid, Channel), + emqx_gateway_ctx:set_chan_info(Ctx, ClientId, info(State)), + emqx_gateway_ctx:connection_closed(Ctx, ClientId), {ok, State}; -%handle_msg({event, _Other}, State = #state{channel = Channel}) -> -% ClientId = emqx_exproto_channel:info(clientid, Channel), -% emqx_cm:set_chan_info(ClientId, info(State)), -% emqx_cm:set_chan_stats(ClientId, stats(State)), -% {ok, State}; +handle_msg({event, _Other}, State = #state{ + chann_mod = ChannMod, + channel = Channel}) -> + Ctx = ChannMod:info(ctx, Channel), + ClientId = ChannMod:info(clientid, Channel), + emqx_gateway_ctx:set_chan_info(Ctx, ClientId, info(State)), + emqx_gateway_ctx:set_chan_stats(Ctx, ClientId, stats(State)), + {ok, State}; handle_msg({timeout, TRef, TMsg}, State) -> handle_timeout(TRef, TMsg, State); @@ -455,9 +514,11 @@ handle_msg(Msg, State) -> %% Terminate -spec terminate(atom(), state()) -> no_return(). -terminate(Reason, State = #state{channel = Channel}) -> +terminate(Reason, State = #state{ + chann_mod = ChannMod, + channel = Channel}) -> ?LOG(debug, "Terminated due to ~p", [Reason]), - _ = emqx_exproto_channel:terminate(Reason, Channel), + _ = ChannMod:terminate(Reason, Channel), _ = close_socket(State), exit(Reason). @@ -465,7 +526,7 @@ terminate(Reason, State = #state{channel = Channel}) -> %% Sys callbacks system_continue(Parent, _Debug, State) -> - recvloop(Parent, State). + ?MODULE:recvloop(Parent, State). system_terminate(Reason, _Parent, _Debug, State) -> terminate(Reason, State). @@ -484,8 +545,10 @@ handle_call(_From, info, State) -> handle_call(_From, stats, State) -> {reply, stats(State), State}; -handle_call(_From, Req, State = #state{channel = Channel}) -> - case emqx_exproto_channel:handle_call(Req, Channel) of +handle_call(_From, Req, State = #state{ + chann_mod = ChannMod, + channel = Channel}) -> + case ChannMod:handle_call(Req, Channel) of {reply, Reply, NChannel} -> {reply, Reply, State#state{channel = NChannel}}; {reply, Reply, Replies, NChannel} -> @@ -505,26 +568,31 @@ handle_timeout(_TRef, limit_timeout, State) -> limit_timer = undefined }, handle_info(activate_socket, NState); -handle_timeout(TRef, keepalive, State = #state{socket = Socket, - channel = Channel})-> - case emqx_exproto_channel:info(conn_state, Channel) of +handle_timeout(TRef, Keepalive, State = #state{ + chann_mod = ChannMod, + socket = Socket, + channel = Channel}) + when Keepalive == keepalive; + Keepalive == keepalive_send -> + Stat = case Keepalive of + keepalive -> recv_oct; + keepalive_send -> send_oct + end, + case ChannMod:info(conn_state, Channel) of disconnected -> {ok, State}; _ -> - case esockd_getstat(Socket, [recv_oct]) of - {ok, [{recv_oct, RecvOct}]} -> - handle_timeout(TRef, {keepalive, RecvOct}, State); + case esockd_getstat(Socket, [Stat]) of + {ok, [{Stat, RecvOct}]} -> + handle_timeout(TRef, {Keepalive, RecvOct}, State); {error, Reason} -> handle_info({sock_error, Reason}, State) end end; handle_timeout(_TRef, emit_stats, State = - #state{channel = Channel}) -> - case emqx_exproto_channel:info(clientid, Channel) of - undefined -> - ignore; - ClientId -> - emqx_cm:set_chan_stats(ClientId, stats(State)) - end, + #state{chann_mod = ChannMod, channel = Channel}) -> + Ctx = ChannMod:info(ctx, Channel), + ClientId = ChannMod:info(clientid, Channel), + emqx_gateway_ctx:set_chan_stats(Ctx, ClientId, stats(State)), {ok, State#state{stats_timer = undefined}}; handle_timeout(TRef, Msg, State) -> @@ -533,52 +601,115 @@ handle_timeout(TRef, Msg, State) -> %%-------------------------------------------------------------------- %% Parse incoming data --compile({inline, [process_incoming/2]}). -process_incoming(Data, State = #state{idle_timer = IdleTimer}) -> +parse_incoming(Data, State = #state{ + chann_mod = ChannMod, + channel = Channel}) -> ?LOG(debug, "RECV ~0p", [Data]), Oct = iolist_size(Data), inc_counter(incoming_bytes, Oct), - inc_counter(incoming_pkt, 1), - inc_counter(recv_pkt, 1), - inc_counter(recv_msg, 1), - % TODO: - %ok = emqx_metrics:inc('bytes.received', Oct), + Ctx = ChannMod:info(ctx, Channel), + ok = emqx_gateway_ctx:metrics_inc(Ctx, 'bytes.received', Oct), + {Packets, NState} = parse_incoming(Data, [], State), + {ok, next_incoming_msgs(Packets), NState}. - ok = emqx_misc:cancel_timer(IdleTimer), - NState = State#state{idle_timer = undefined}, +parse_incoming(<<>>, Packets, State) -> + {Packets, State}; - with_channel(handle_in, [Data], NState). +parse_incoming(Data, Packets, + State = #state{ + frame_mod = FrameMod, + parse_state = ParseState}) -> + try FrameMod:parse(Data, ParseState) of + {more, NParseState} -> + {Packets, State#state{parse_state = NParseState}}; + {ok, Packet, Rest, NParseState} -> + NState = State#state{parse_state = NParseState}, + parse_incoming(Rest, [Packet|Packets], NState) + catch + error:Reason:Stk -> + ?LOG(error, "~nParse failed for ~0p~n~0p~nFrame data:~0p", + [Reason, Stk, Data]), + {[{frame_error, Reason}|Packets], State} + end. + +next_incoming_msgs([Packet]) -> + {incoming, Packet}; +next_incoming_msgs(Packets) -> + [{incoming, Packet} || Packet <- lists:reverse(Packets)]. + +%%-------------------------------------------------------------------- +%% Handle incoming packet + +handle_incoming(Packet, State = #state{ + channel = Channel, + frame_mod = FrameMod, + chann_mod = ChannMod + }) -> + Ctx = ChannMod:info(ctx, Channel), + ok = inc_incoming_stats(Ctx, FrameMod, Packet), + ?LOG(debug, "RECV ~s", [FrameMod:format(Packet)]), + with_channel(handle_in, [Packet], State). %%-------------------------------------------------------------------- %% With Channel -with_channel(Fun, Args, State = #state{channel = Channel}) -> - case erlang:apply(emqx_exproto_channel, Fun, Args ++ [Channel]) of +with_channel(Fun, Args, State = #state{ + chann_mod = ChannMod, + channel = Channel}) -> + case erlang:apply(ChannMod, Fun, Args ++ [Channel]) of ok -> {ok, State}; {ok, NChannel} -> {ok, State#state{channel = NChannel}}; {ok, Replies, NChannel} -> {ok, next_msgs(Replies), State#state{channel = NChannel}}; {shutdown, Reason, NChannel} -> - shutdown(Reason, State#state{channel = NChannel}) + shutdown(Reason, State#state{channel = NChannel}); + {shutdown, Reason, Packet, NChannel} -> + NState = State#state{channel = NChannel}, + ok = handle_outgoing(Packet, NState), + shutdown(Reason, NState) end. %%-------------------------------------------------------------------- %% Handle outgoing packets -handle_outgoing(IoData, State = #state{socket = Socket}) -> - ?LOG(debug, "SEND ~0p", [IoData]), +handle_outgoing(Packets, State) when is_list(Packets) -> + send(lists:map(serialize_and_inc_stats_fun(State), Packets), State); +handle_outgoing(Packet, State) -> + send((serialize_and_inc_stats_fun(State))(Packet), State). + +serialize_and_inc_stats_fun(#state{ + frame_mod = FrameMod, + chann_mod = ChannMod, + serialize = Serialize, + channel = Channel}) -> + Ctx = ChannMod:info(ctx, Channel), + fun(Packet) -> + case FrameMod:serialize_pkt(Packet, Serialize) of + <<>> -> ?LOG(warning, "~s is discarded due to the frame is too large!", + [FrameMod:format(Packet)]), + ok = emqx_gateway_ctx:metrics_inc(Ctx, 'delivery.dropped.too_large'), + ok = emqx_gateway_ctx:metrics_inc(Ctx, 'delivery.dropped'), + <<>>; + Data -> ?LOG(debug, "SEND ~s", [FrameMod:format(Packet)]), + ok = inc_outgoing_stats(Ctx, FrameMod, Packet), + Data + end + end. + +%%-------------------------------------------------------------------- +%% Send data + +-spec(send(iodata(), state()) -> ok). +send(IoData, State = #state{socket = Socket, + chann_mod = ChannMod, + channel = Channel}) -> + Ctx = ChannMod:info(ctx, Channel), Oct = iolist_size(IoData), - - inc_counter(send_pkt, 1), - inc_counter(send_msg, 1), - inc_counter(outgoing_pkt, 1), + ok = emqx_gateway_ctx:metrics_inc(Ctx, 'bytes.sent', Oct), inc_counter(outgoing_bytes, Oct), - - %% FIXME: - %%ok = emqx_metrics:inc('bytes.sent', Oct), - case send(IoData, State) of + case esockd_send(IoData, State) of ok -> ok; Error = {error, _Reason} -> %% Send an inet_reply to postpone handling the error @@ -617,7 +748,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> State#state{limiter = Limiter1}; {pause, Time, Limiter1} -> ?LOG(warning, "Pause ~pms due to rate limit", [Time]), - TRef = start_timer(Time, limit_timeout), + TRef = emqx_misc:start_timer(Time, limit_timeout), State#state{sockstate = blocked, limiter = Limiter1, limit_timer = TRef @@ -634,11 +765,11 @@ run_gc(Stats, State = #state{gc_state = GcSt}) -> State#state{gc_state = GcSt1} end. -check_oom(State) -> - OomPolicy = ?DEFAULT_OOM_POLICY, +check_oom(State = #state{oom_policy = OomPolicy}) -> case ?ENABLED(OomPolicy) andalso emqx_misc:check_oom(OomPolicy) of - Shutdown = {shutdown, _Reason} -> - erlang:send(self(), Shutdown); + {shutdown, Reason} -> + %% triggers terminate/2 callback immediately + erlang:exit({shutdown, Reason}); _Other -> ok end, State. @@ -646,7 +777,6 @@ check_oom(State) -> %%-------------------------------------------------------------------- %% Activate Socket --compile({inline, [activate_socket/1]}). activate_socket(State = #state{sockstate = closed}) -> {ok, State}; activate_socket(State = #state{sockstate = blocked}) -> @@ -668,6 +798,35 @@ close_socket(State = #state{socket = Socket}) -> ok = esockd_close(Socket), State#state{sockstate = closed}. +%%-------------------------------------------------------------------- +%% Inc incoming/outgoing stats + +inc_incoming_stats(Ctx, FrameMod, Packet) -> + inc_counter(recv_pkt, 1), + case FrameMod:is_message(Packet) of + true -> + inc_counter(recv_msg, 1), + inc_counter(incoming_pubs, 1); + false -> + ok + end, + Name = list_to_atom( + lists:concat(["packets.", FrameMod:type(Packet), ".recevied"])), + emqx_gateway_ctx:metrics_inc(Ctx, Name). + +inc_outgoing_stats(Ctx, FrameMod, Packet) -> + inc_counter(send_pkt, 1), + case FrameMod:is_message(Packet) of + true -> + inc_counter(send_msg, 1), + inc_counter(outgoing_pubs, 1); + false -> + ok + end, + Name = list_to_atom( + lists:concat(["packets.", FrameMod:type(Packet), ".sent"])), + emqx_gateway_ctx:metrics_inc(Ctx, Name). + %%-------------------------------------------------------------------- %% Helper functions @@ -677,14 +836,12 @@ next_msgs(Event) when is_tuple(Event) -> next_msgs(More) when is_list(More) -> More. --compile({inline, [shutdown/2, shutdown/3]}). shutdown(Reason, State) -> stop({shutdown, Reason}, State). shutdown(Reason, Reply, State) -> stop({shutdown, Reason}, Reply, State). --compile({inline, [stop/2, stop/3]}). stop(Reason, State) -> {stop, Reason, State}. diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_frame.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_frame.erl new file mode 100644 index 000000000..87410f7d8 --- /dev/null +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_frame.erl @@ -0,0 +1,57 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The Gateway frame behavior +%% +%% This module does not export any functions at the moment. +%% It is only used to standardize the implement of emqx_foo_frame.erl +%% module if it integrated with emqx_gateway_conn module +%% +-module(emqx_gateway_frame). + +-type parse_state() :: map(). + +-type frame() :: any(). + +-type parse_result() :: {ok, frame(), + Rest :: binary(), NewState :: parse_state()} + | {more, NewState :: parse_state()}. + +-type serialize_options() :: map(). + +%% Callbacks + +%% @doc Initial the frame parser states +-callback initial_parse_state(map()) -> parse_state(). + +%% @doc +-callback serialize_opts() -> serialize_options(). + +%% @doc +-callback serialize_pkt(Frame :: any(), serialize_options()) -> iodata(). + +%% @doc +-callback parse(binary(), parse_state()) -> parse_result(). + +%% @doc +-callback format(Frame :: any()) -> string(). + +%% @doc +-callback type(Frame :: any()) -> atom(). + +%% @doc +-callback is_message(Frame :: any()) -> boolean(). + diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl new file mode 100644 index 000000000..8d413e49c --- /dev/null +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl @@ -0,0 +1,50 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway_impl). + +-include("include/emqx_gateway.hrl"). + +-type state() :: map(). +-type reason() :: any(). + +%% @doc +-callback init(Options :: list()) -> {error, reason()} | {ok, GwState :: state()}. + +%% @doc +-callback on_insta_create(Insta :: instance(), + Ctx :: emqx_gateway_ctx:context(), + GwState :: state() + ) + -> {error, reason()} + | {ok, [GwInstaPid :: pid()], GwInstaState :: state()} + %% TODO: v0.2 The child spec is better for restarting child process + | {ok, [Childspec :: supervisor:child_spec()], GwInstaState :: state()}. + +%% @doc +-callback on_insta_update(NewInsta :: instance(), + OldInsta :: instance(), + GwInstaState :: state(), + GwState :: state()) + -> ok + | {ok, [GwInstaPid :: pid()], GwInstaState :: state()} + | {ok, [Childspec :: supervisor:child_spec()], GwInstaState :: state()} + | {error, reason()}. + +%% @doc +-callback on_insta_destroy(Insta :: instance(), + GwInstaState :: state(), + GwState :: state()) -> ok. diff --git a/apps/emqx_gateway/src/coap/README.md b/apps/emqx_gateway/src/coap/README.md new file mode 100644 index 000000000..f71938c92 --- /dev/null +++ b/apps/emqx_gateway/src/coap/README.md @@ -0,0 +1,190 @@ + +# Table of Contents + +1. [EMQX 5.0 CoAP Gateway](#org6feb6de) +2. [CoAP Message Processing Flow](#org8458c1a) + 1. [Request Timing Diagram](#orgeaa4f53) + 1. [Transport && Transport Manager](#org88207b8) + 2. [Resource](#orgb32ce94) +3. [Resource](#org8956f90) + 1. [MQTT Resource](#orge8c21b1) + 2. [PubSub Resource](#org68ddce7) +4. [Heartbeat](#orgffdfecd) +5. [Command](#org43004c2) +6. [MQTT QOS <=> CoAP non/con](#org0157b5c) + + + + + +# EMQX 5.0 CoAP Gateway + +emqx-coap is a CoAP Gateway for EMQ X Broker. +It translates CoAP messages into MQTT messages and make it possible to communiate between CoAP clients and MQTT clients. + + + + +# CoAP Message Processing Flow + + + + +## Request Timing Diagram + + + ,------. ,------------. ,-----------------. ,---------. ,--------. + |client| |coap_gateway| |transport_manager| |transport| |resource| + `--+---' `-----+------' `--------+--------' `----+----' `---+----' + | | | | | + | -------------------> | | | + | | | | | + | | | | | + | | ------------------------>| | | + | | | | | + | | | | | + | | |----------------------->| | + | | | | | + | | | | | + | | | |------------------>| + | | | | | + | | | | | + | | | |<------------------| + | | | | | + | | | | | + | | |<-----------------------| | + | | | | | + | | | | | + | | <------------------------| | | + | | | | | + | | | | | + | <------------------- | | | + ,--+---. ,-----+------. ,--------+--------. ,----+----. ,---+----. + |client| |coap_gateway| |transport_manager| |transport| |resource| + `------' `------------' `-----------------' `---------' `--------' + + + + +### Transport && Transport Manager + +Transport is a module that manages the life cycle and behaviour of CoAP messages\ +And the transport manager is to manage all transport which in this gateway + + + + +### Resource + +The Resource is a behaviour that must implement GET/PUT/POST/DELETE method\ +Different Resources can have different implementations of this four method\ +Each gateway can only use one Resource module to process CoAP Request Message + + + + +# Resource + + + + +## MQTT Resource + +The MQTT Resource is a simple CoAP to MQTT adapter, the implementation of each method is as follows: + +- use uri path as topic +- GET: subscribe the topic +- PUT: publish message to this topic +- POST: like PUT +- DELETE: unsubscribe the topic + + + + +## PubSub Resource + +The PubSub Resource like the MQTT Resource, but has a retained topic's message database\ +This Resource is shared, only can has one instance. The implementation: + +- use uri path as topic +- GET: + - GET with observe = 0: subscribe the topic + - GET with observe = 1: unsubscribe the topic + - GET without observe: read lastest message from the message database, key is the topic +- PUT: + insert message into the message database, key is the topic +- POST: + like PUT, but will publish the message +- DELETE: + delete message from the database, key is topic + + + + +# Heartbeat + +At present, the CoAP gateway only supports UDP/DTLS connection, don't support UDP over TCP and UDP over WebSocket. +Because UDP is connectionless, so the client needs to send heartbeat ping to the server interval. Otherwise, the server will close related resources +Use ****POST with empty uri path**** as a heartbeat ping + +example: +``` +coap-client -m post coap://127.0.0.1 +``` + + + +# Command + +Command is means the operation which outside the CoAP protocol, like authorization +The Command format: + +1. use ****POST**** method +2. uri path is empty +3. query string is like ****action=comandX&argX=valuex&argY=valueY**** + +example: +1. connect: +``` +coap-client -m post coap://127.0.0.1?action=connect&clientid=XXX&username=XXX&password=XXX +``` +2. disconnect: +``` +coap-client -m post coap://127.0.0.1?action=disconnect +``` + + + +# MQTT QOS <=> CoAP non/con + +CoAP gateway uses some options to control the conversion between MQTT qos and coap non/con: + +1.notify_type +Control the type of notify messages when the observed object has changed.Can be: + +- non +- con +- qos + in this value, MQTT QOS0 -> non, QOS1/QOS2 -> con + +2.subscribe_qos +Control the qos of subscribe.Can be: + +- qos0 +- qos1 +- qos2 +- coap + in this value, CoAP non -> qos0, con -> qos1 + +3.publish_qos +like subscribe_qos, but control the qos of the publish MQTT message + +License +------- + +Apache License Version 2.0 + +Author +------ + +EMQ X Team. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl new file mode 100644 index 000000000..d5b9b7293 --- /dev/null +++ b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl @@ -0,0 +1,387 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_coap_channel). + +-behavior(emqx_gateway_channel). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). + +%% API +-export([]). + +-export([ info/1 + , info/2 + , stats/1 + , auth_publish/2 + , auth_subscribe/2 + , reply/4 + , ack/4 + , transfer_result/3]). + +-export([ init/2 + , handle_in/2 + , handle_deliver/2 + , handle_timeout/3 + , terminate/2 + ]). + +-export([ handle_call/2 + , handle_cast/2 + , handle_info/2 + ]). + +-export_type([channel/0]). + +-record(channel, { + %% Context + ctx :: emqx_gateway_ctx:context(), + %% Connection Info + conninfo :: emqx_types:conninfo(), + %% Client Info + clientinfo :: emqx_types:clientinfo(), + %% Session + session :: emqx_coap_session:session() | undefined, + %% Keepalive + keepalive :: emqx_keepalive:keepalive() | undefined, + %% Timer + timers :: #{atom() => disable | undefined | reference()}, + config :: hocon:config() + }). + +-type channel() :: #channel{}. +-define(DISCONNECT_WAIT_TIME, timer:seconds(10)). +-define(INFO_KEYS, [conninfo, conn_state, clientinfo, session]). + +%%%=================================================================== +%%% API +%%%=================================================================== + +info(Channel) -> + maps:from_list(info(?INFO_KEYS, Channel)). + +info(Keys, Channel) when is_list(Keys) -> + [{Key, info(Key, Channel)} || Key <- Keys]; + +info(conninfo, #channel{conninfo = ConnInfo}) -> + ConnInfo; +info(conn_state, _) -> + connected; +info(clientinfo, #channel{clientinfo = ClientInfo}) -> + ClientInfo; +info(session, #channel{session = Session}) -> + emqx_misc:maybe_apply(fun emqx_session:info/1, Session); +info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> + ClientId; +info(ctx, #channel{ctx = Ctx}) -> + Ctx. + +stats(_) -> + []. + +init(ConnInfo = #{peername := {PeerHost, _}, + sockname := {_, SockPort}}, + #{ctx := Ctx} = Config) -> + Peercert = maps:get(peercert, ConnInfo, undefined), + Mountpoint = maps:get(mountpoint, Config, undefined), + ClientInfo = set_peercert_infos( + Peercert, + #{ zone => default + , protocol => 'mqtt-coap' + , peerhost => PeerHost + , sockport => SockPort + , clientid => emqx_guid:to_base62(emqx_guid:gen()) + , username => undefined + , is_bridge => false + , is_superuser => false + , mountpoint => Mountpoint + } + ), + + #channel{ ctx = Ctx + , conninfo = ConnInfo + , clientinfo = ClientInfo + , timers = #{} + , session = emqx_coap_session:new() + , config = Config#{clientinfo => ClientInfo, + ctx => Ctx} + , keepalive = emqx_keepalive:init(maps:get(heartbeat, Config)) + }. + +auth_publish(Topic, + #{ctx := Ctx, + clientinfo := ClientInfo}) -> + emqx_gateway_ctx:authorize(Ctx, ClientInfo, publish, Topic). + +auth_subscribe(Topic, + #{ctx := Ctx, + clientinfo := ClientInfo}) -> + emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, Topic). + +transfer_result(Result, From, Value) -> + ?TRANSFER_RESULT(Result, [out], From, Value). + +%%-------------------------------------------------------------------- +%% Handle incoming packet +%%-------------------------------------------------------------------- +%% treat post to root path as a heartbeat +%% treat post to root path with query string as a command +handle_in(#coap_message{method = post, + options = Options} = Msg, ChannelT) -> + Channel = ensure_keepalive_timer(ChannelT), + case maps:get(uri_path, Options, <<>>) of + <<>> -> + handle_command(Msg, Channel); + _ -> + call_session(Channel, received, [Msg]) + end; + +handle_in(Msg, Channel) -> + call_session(ensure_keepalive_timer(Channel), received, [Msg]). + +%%-------------------------------------------------------------------- +%% Handle Delivers from broker to client +%%-------------------------------------------------------------------- +handle_deliver(Delivers, Channel) -> + call_session(Channel, deliver, [Delivers]). + +%%-------------------------------------------------------------------- +%% Handle timeout +%%-------------------------------------------------------------------- +handle_timeout(_, {keepalive, NewVal}, #channel{keepalive = KeepAlive} = Channel) -> + case emqx_keepalive:check(NewVal, KeepAlive) of + {ok, NewKeepAlive} -> + Channel2 = ensure_keepalive_timer(Channel, fun make_timer/4), + {ok, Channel2#channel{keepalive = NewKeepAlive}}; + {error, timeout} -> + {shutdown, timeout, Channel} + end; + +handle_timeout(_, {transport, Msg}, Channel) -> + call_session(Channel, timeout, [Msg]); + +handle_timeout(_, disconnect, Channel) -> + {shutdown, normal, Channel}; + +handle_timeout(_, _, Channel) -> + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Handle call +%%-------------------------------------------------------------------- +handle_call(Req, Channel) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignored, Channel}. + +%%-------------------------------------------------------------------- +%% Handle Cast +%%-------------------------------------------------------------------- +handle_cast(Req, Channel) -> + ?LOG(error, "Unexpected cast: ~p", [Req]), + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Handle Info +%%-------------------------------------------------------------------- +handle_info(Info, Channel) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Terminate +%%-------------------------------------------------------------------- +terminate(_Reason, _Channel) -> + ok. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== +set_peercert_infos(NoSSL, ClientInfo) + when NoSSL =:= nossl; + NoSSL =:= undefined -> + ClientInfo; +set_peercert_infos(Peercert, ClientInfo) -> + {DN, CN} = {esockd_peercert:subject(Peercert), + esockd_peercert:common_name(Peercert)}, + ClientInfo#{dn => DN, cn => CN}. + +ensure_timer(Name, Time, Msg, #channel{timers = Timers} = Channel) -> + case maps:get(Name, Timers, undefined) of + undefined -> + make_timer(Name, Time, Msg, Channel); + _ -> + Channel + end. + +make_timer(Name, Time, Msg, Channel = #channel{timers = Timers}) -> + TRef = emqx_misc:start_timer(Time, Msg), + Channel#channel{timers = Timers#{Name => TRef}}. + +ensure_keepalive_timer(Channel) -> + ensure_keepalive_timer(Channel, fun ensure_timer/4). + +ensure_keepalive_timer(#channel{config = Cfg} = Channel, Fun) -> + Interval = maps:get(heartbeat, Cfg), + Fun(keepalive, Interval, keepalive, Channel). + +handle_command(#coap_message{options = Options} = Msg, Channel) -> + case maps:get(uri_query, Options, []) of + [] -> + %% heartbeat + ack(Channel, {ok, valid}, <<>>, Msg); + QueryPairs -> + Queries = lists:foldl(fun(Pair, Acc) -> + [{K, V}] = cow_qs:parse_qs(Pair), + Acc#{K => V} + end, + #{}, + QueryPairs), + case maps:get(<<"action">>, Queries, undefined) of + undefined -> + ack(Channel, {error, bad_request}, <<"command without actions">>, Msg); + Action -> + handle_command(Action, Queries, Msg, Channel) + end + end. + +handle_command(<<"connect">>, Queries, Msg, Channel) -> + case emqx_misc:pipeline( + [ fun run_conn_hooks/2 + , fun enrich_clientinfo/2 + , fun set_log_meta/2 + , fun auth_connect/2 + ], + {Queries, Msg}, + Channel) of + {ok, _Input, NChannel} -> + process_connect(ensure_connected(NChannel), Msg); + {error, ReasonCode, NChannel} -> + ErrMsg = io_lib:format("Login Failed: ~s", [ReasonCode]), + ack(NChannel, {error, bad_request}, ErrMsg, Msg) + end; + +handle_command(<<"disconnect">>, _, Msg, Channel) -> + Channel2 = ensure_timer(disconnect, ?DISCONNECT_WAIT_TIME, disconnect, Channel), + ack(Channel2, {ok, deleted}, <<>>, Msg); + +handle_command(_, _, Msg, Channel) -> + ack(Channel, {error, bad_request}, <<"invalid action">>, Msg). + +run_conn_hooks(Input, Channel = #channel{ctx = Ctx, + conninfo = ConnInfo}) -> + ConnProps = #{}, + case run_hooks(Ctx, 'client.connect', [ConnInfo], ConnProps) of + Error = {error, _Reason} -> Error; + _NConnProps -> + {ok, Input, Channel} + end. + +enrich_clientinfo({Queries, Msg}, + Channel = #channel{clientinfo = ClientInfo0, + config = Cfg}) -> + case Queries of + #{<<"username">> := UserName, + <<"password">> := Password, + <<"clientid">> := ClientId} -> + ClientInfo = ClientInfo0#{username => UserName, + password => Password, + clientid => ClientId}, + {ok, NClientInfo} = fix_mountpoint(Msg, ClientInfo), + {ok, Channel#channel{clientinfo = NClientInfo, + config = Cfg#{clientinfo := NClientInfo}}}; + _ -> + {error, "invalid queries", Channel} + end. + +set_log_meta(_Input, #channel{clientinfo = #{clientid := ClientId}}) -> + emqx_logger:set_metadata_clientid(ClientId), + ok. + +auth_connect(_Input, Channel = #channel{ctx = Ctx, + clientinfo = ClientInfo}) -> + #{clientid := ClientId, + username := Username} = ClientInfo, + case emqx_gateway_ctx:authenticate(Ctx, ClientInfo) of + {ok, NClientInfo} -> + {ok, Channel#channel{clientinfo = NClientInfo}}; + {error, Reason} -> + ?LOG(warning, "Client ~s (Username: '~s') login failed for ~0p", + [ClientId, Username, Reason]), + {error, Reason} + end. + +fix_mountpoint(_Packet, #{mountpoint := undefined}) -> ok; +fix_mountpoint(_Packet, ClientInfo = #{mountpoint := Mountpoint}) -> + %% TODO: Enrich the varibale replacement???? + %% i.e: ${ClientInfo.auth_result.productKey} + Mountpoint1 = emqx_mountpoint:replvar(Mountpoint, ClientInfo), + {ok, ClientInfo#{mountpoint := Mountpoint1}}. + +ensure_connected(Channel = #channel{ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo}) -> + NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, + ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]), + Channel#channel{conninfo = NConnInfo}. + +process_connect(Channel = #channel{ctx = Ctx, + session = Session, + conninfo = ConnInfo, + clientinfo = ClientInfo}, + Msg) -> + SessFun = fun(_,_) -> Session end, + case emqx_gateway_ctx:open_session( + Ctx, + true, + ClientInfo, + ConnInfo, + SessFun + ) of + {ok, _Sess} -> + ack(Channel, {ok, created}, <<"connected">>, Msg); + {error, Reason} -> + ?LOG(error, "Failed to open session du to ~p", [Reason]), + ack(Channel, {error, bad_request}, <<>>, Msg) + end. + +run_hooks(Ctx, Name, Args) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name), + emqx_hooks:run(Name, Args). + +run_hooks(Ctx, Name, Args, Acc) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name), + emqx_hooks:run_fold(Name, Args, Acc). + +reply(Channel, Method, Payload, Req) -> + call_session(Channel, reply, [Req, Method, Payload]). + +ack(Channel, Method, Payload, Req) -> + call_session(Channel, piggyback, [Req, Method, Payload]). + +call_session(#channel{session = Session, + config = Cfg} = Channel, F, A) -> + case erlang:apply(emqx_coap_session, F, [Session, Cfg | A]) of + #{out := Out, + session := Session2} -> + {ok, {outgoing, Out}, Channel#channel{session = Session2}}; + #{out := Out} -> + {ok, {outgoing, Out}, Channel}; + #{session := Session2} -> + {ok, Channel#channel{session = Session2}}; + _ -> + {ok, Channel} + end. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl b/apps/emqx_gateway/src/coap/emqx_coap_frame.erl new file mode 100644 index 000000000..039190646 --- /dev/null +++ b/apps/emqx_gateway/src/coap/emqx_coap_frame.erl @@ -0,0 +1,423 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_coap_frame). + +-behavior(emqx_gateway_frame). + +%% emqx_gateway_frame API +-export([ initial_parse_state/1 + , serialize_opts/0 + , serialize_pkt/2 + , parse/2 + , format/1 + , type/1 + , is_message/1]). + +%% API +-export([]). + +-include("include/emqx_coap.hrl"). +-include("apps/emqx/include/types.hrl"). + +-define(VERSION, 1). + +-define(OPTION_IF_MATCH, 1). +-define(OPTION_URI_HOST, 3). +-define(OPTION_ETAG, 4). +-define(OPTION_IF_NONE_MATCH, 5). +-define(OPTION_OBSERVE, 6). % draft-ietf-core-observe-16 +-define(OPTION_URI_PORT, 7). +-define(OPTION_LOCATION_PATH, 8). +-define(OPTION_URI_PATH, 11). +-define(OPTION_CONTENT_FORMAT, 12). +-define(OPTION_MAX_AGE, 14). +-define(OPTION_URI_QUERY, 15). +-define(OPTION_ACCEPT, 17). +-define(OPTION_LOCATION_QUERY, 20). +-define(OPTION_BLOCK2, 23). % draft-ietf-core-block-17 +-define(OPTION_BLOCK1, 27). +-define(OPTION_PROXY_URI, 35). +-define(OPTION_PROXY_SCHEME, 39). +-define(OPTION_SIZE1, 60). + +%%%=================================================================== +%%% API +%%%=================================================================== + +initial_parse_state(_) -> + #{}. + +serialize_opts() -> + #{}. + +%%%=================================================================== +%%% serialize_pkt +%%%=================================================================== +%% empty message +serialize_pkt(#coap_message{type = Type, method = undefined, id = MsgId}, _Opts) -> + <>; + +serialize_pkt(#coap_message{ type = Type + , method = Method + , id = MsgId + , token = Token + , options = Options + , payload = Payload + }, + _Opts) -> + TKL = byte_size(Token), + {Class, Code} = method_to_class_code(Method), + Head = <>, + FlatOpts = flatten_options(Options), + encode_option_list(FlatOpts, 0, Head, Payload). + +-spec encode_type(message_type()) -> 0 .. 3. +encode_type(con) -> 0; +encode_type(non) -> 1; +encode_type(ack) -> 2; +encode_type(reset) -> 3. + +flatten_options(Opts) -> + flatten_options(maps:to_list(Opts), []). + +flatten_options([{_OptId, undefined} | T], Acc) -> + flatten_options(T, Acc); + +flatten_options([{OptId, OptVal} | T], Acc) -> + flatten_options(T, + case is_repeatable_option(OptId) of + false -> + [encode_option(OptId, OptVal) | Acc]; + _ -> + lists:foldl(fun(undefined, InnerAcc) -> + InnerAcc; + (E, InnerAcc) -> + [encode_option(OptId, E) | InnerAcc] + end, Acc, OptVal) + end); + +flatten_options([], Acc) -> + %% sort by option id for calculate the deltas + lists:keysort(1, Acc). + +encode_option_list([{OptNum, OptVal} | OptionList], LastNum, Acc, Payload) -> + NumDiff = OptNum - LastNum, + {Delta, ExtNum} = if + NumDiff >= 269 -> + {14, <<(NumDiff - 269):16>>}; + OptNum - LastNum >= 13 -> + {13, <<(NumDiff - 13)>>}; + true -> + {NumDiff, <<>>} + end, + Binaryize = byte_size(OptVal), + {Len, ExtLen} = if + Binaryize >= 269 -> + {14, <<(Binaryize - 269):16>>}; + Binaryize >= 13 -> + {13, <<(Binaryize - 13)>>}; + true -> + {Binaryize, <<>>} + end, + Acc2 = <>, + encode_option_list(OptionList, OptNum, Acc2, Payload); + +encode_option_list([], _LastNum, Acc, <<>>) -> + Acc; +encode_option_list([], _, Acc, Payload) -> + <>. + +%% RFC 7252 +encode_option(if_match, OptVal) -> {?OPTION_IF_MATCH, OptVal}; +encode_option(uri_host, OptVal) -> {?OPTION_URI_HOST, OptVal}; +encode_option(etag, OptVal) -> {?OPTION_ETAG, OptVal}; +encode_option(if_none_match, true) -> {?OPTION_IF_NONE_MATCH, <<>>}; +encode_option(uri_port, OptVal) -> {?OPTION_URI_PORT, binary:encode_unsigned(OptVal)}; +encode_option(location_path, OptVal) -> {?OPTION_LOCATION_PATH, OptVal}; +encode_option(uri_path, OptVal) -> {?OPTION_URI_PATH, OptVal}; +encode_option(content_format, OptVal) when is_integer(OptVal) -> + {?OPTION_CONTENT_FORMAT, binary:encode_unsigned(OptVal)}; +encode_option(content_format, OptVal) -> + Num = content_format_to_code(OptVal), + {?OPTION_CONTENT_FORMAT, binary:encode_unsigned(Num)}; +encode_option(max_age, OptVal) -> {?OPTION_MAX_AGE, binary:encode_unsigned(OptVal)}; +encode_option(uri_query, OptVal) -> {?OPTION_URI_QUERY, OptVal}; +encode_option('accept', OptVal) -> {?OPTION_ACCEPT, binary:encode_unsigned(OptVal)}; +encode_option(location_query, OptVal) -> {?OPTION_LOCATION_QUERY, OptVal}; +encode_option(proxy_uri, OptVal) -> {?OPTION_PROXY_URI, OptVal}; +encode_option(proxy_scheme, OptVal) -> {?OPTION_PROXY_SCHEME, OptVal}; +encode_option(size1, OptVal) -> {?OPTION_SIZE1, binary:encode_unsigned(OptVal)}; +%% draft-ietf-ore-observe-16 +encode_option(observe, OptVal) -> {?OPTION_OBSERVE, binary:encode_unsigned(OptVal)}; +%% draft-ietf-ore-block-17 +encode_option(block2, OptVal) -> {?OPTION_BLOCK2, encode_block(OptVal)}; +encode_option(block1, OptVal) -> {?OPTION_BLOCK1, encode_block(OptVal)}; +%% unknown opton +encode_option(Option, Value) -> + erlang:throw({bad_option, Option, Value}). + +encode_block({Num, More, Size}) -> + encode_block1(Num, + if More -> 1; true -> 0 end, + trunc(math:log2(Size))-4). + +encode_block1(Num, M, SizEx) when Num < 16 -> + <>; +encode_block1(Num, M, SizEx) when Num < 4096 -> + <>; +encode_block1(Num, M, SizEx) -> + <>. + +-spec content_format_to_code(binary()) -> non_neg_integer(). +content_format_to_code(<<"text/plain">>) -> 0; +content_format_to_code(<<"application/link-format">>) -> 40; +content_format_to_code(<<"application/xml">>) ->41; +content_format_to_code(<<"application/octet-stream">>) -> 42; +content_format_to_code(<<"application/exi">>) -> 47; +content_format_to_code(<<"application/json">>) -> 50; +content_format_to_code(<<"application/cbor">>) -> 60; +content_format_to_code(_) -> 42. %% use octet-stream as default + +method_to_class_code(get) -> {0, 01}; +method_to_class_code(post) -> {0, 02}; +method_to_class_code(put) -> {0, 03}; +method_to_class_code(delete) -> {0, 04}; +method_to_class_code({ok, created}) -> {2, 01}; +method_to_class_code({ok, deleted}) -> {2, 02}; +method_to_class_code({ok, valid}) -> {2, 03}; +method_to_class_code({ok, changed}) -> {2, 04}; +method_to_class_code({ok, content}) -> {2, 05}; +method_to_class_code({ok, nocontent}) -> {2, 07}; +method_to_class_code({ok, continue}) -> {2, 31}; +method_to_class_code({error, bad_request}) -> {4, 00}; +method_to_class_code({error, unauthorized}) -> {4, 01}; +method_to_class_code({error, bad_option}) -> {4, 02}; +method_to_class_code({error, forbidden}) -> {4, 03}; +method_to_class_code({error, not_found}) -> {4, 04}; +method_to_class_code({error, method_not_allowed}) -> {4, 05}; +method_to_class_code({error, not_acceptable}) -> {4, 06}; +method_to_class_code({error, request_entity_incomplete}) -> {4, 08}; +method_to_class_code({error, precondition_failed}) -> {4, 12}; +method_to_class_code({error, request_entity_too_large}) -> {4, 13}; +method_to_class_code({error, unsupported_content_format}) -> {4, 15}; +method_to_class_code({error, internal_server_error}) -> {5, 00}; +method_to_class_code({error, not_implemented}) -> {5, 01}; +method_to_class_code({error, bad_gateway}) -> {5, 02}; +method_to_class_code({error, service_unavailable}) -> {5, 03}; +method_to_class_code({error, gateway_timeout}) -> {5, 04}; +method_to_class_code({error, proxying_not_supported}) -> {5, 05}; +method_to_class_code(Method) -> + erlang:throw({bad_method, Method}). + +%%%=================================================================== +%%% parse +%%%=================================================================== +parse(<>, ParseState) -> + {ok, + #coap_message{ type = decode_type(Type) + , id = MsgId}, + <<>>, + ParseState}; + +parse(<>, + ParseState) -> + {Options, Payload} = decode_option_list(Tail), + Options2 = maps:fold(fun(K, V, Acc) -> + case is_repeatable_option(K) of + true -> + Acc#{K => lists:reverse(V)}; + _ -> + Acc#{K => V} + end + end, + #{}, + Options), + {ok, + #coap_message{ type = decode_type(Type) + , method = class_code_to_method({Class, Code}) + , id = MsgId + , token = Token + , options = Options2 + , payload = Payload + }, + <<>>, + ParseState}. + +-spec decode_type(X) -> message_type() + when X :: 0 .. 3. +decode_type(0) -> con; +decode_type(1) -> non; +decode_type(2) -> ack; +decode_type(3) -> reset. + +-spec decode_option_list(binary()) -> {message_options(), binary()}. +decode_option_list(Bin) -> + decode_option_list(Bin, 0, #{}). + +decode_option_list(<<>>, _OptNum, OptMap) -> + {OptMap, <<>>}; + +decode_option_list(<<16#FF, Payload/binary>>, _OptNum, OptMap) -> + {OptMap, Payload}; + +decode_option_list(<>, OptNum, OptMap) -> + case Delta of + Any when Any < 13 -> + decode_option_len(Bin, OptNum + Delta, Len, OptMap); + 13 -> + <> = Bin, + decode_option_len(NewBin, OptNum + ExtOptNum + 13, Len, OptMap); + 14 -> + <> = Bin, + decode_option_len(NewBin, OptNum + ExtOptNum + 269, Len, OptMap) + end. + +decode_option_len(<>, OptNum, Len, OptMap) -> + case Len of + Any when Any < 13 -> + decode_option_value(Bin, OptNum, Len, OptMap); + 13 -> + <> = Bin, + decode_option_value(NewBin, OptNum, ExtOptLen + 13, OptMap); + 14 -> + <> = Bin, + decode_option_value(NewBin, OptNum, ExtOptLen + 269, OptMap) + end. + +decode_option_value(<>, OptNum, OptLen, OptMap) -> + case Bin of + <> -> + decode_option_list(NewBin, OptNum, append_option(OptNum, OptVal, OptMap)); + <<>> -> + decode_option_list(<<>>, OptNum, append_option(OptNum, <<>>, OptMap)) + end. + +append_option(OptNum, RawOptVal, OptMap) -> + {OptId, OptVal} = decode_option(OptNum, RawOptVal), + case is_repeatable_option(OptId) of + false -> + OptMap#{OptId => OptVal}; + _ -> + case maps:get(OptId, OptMap, undefined) of + undefined -> + OptMap#{OptId => [OptVal]}; + OptVals -> + OptMap#{OptId => [OptVal | OptVals]} + end + end. + +%% RFC 7252 +decode_option(?OPTION_IF_MATCH, OptVal) -> {if_match, OptVal}; +decode_option(?OPTION_URI_HOST, OptVal) -> {uri_host, OptVal}; +decode_option(?OPTION_ETAG, OptVal) -> {etag, OptVal}; +decode_option(?OPTION_IF_NONE_MATCH, <<>>) -> {if_none_match, true}; +decode_option(?OPTION_URI_PORT, OptVal) -> {uri_port, binary:decode_unsigned(OptVal)}; +decode_option(?OPTION_LOCATION_PATH, OptVal) -> {location_path, OptVal}; +decode_option(?OPTION_URI_PATH, OptVal) -> {uri_path, OptVal}; +decode_option(?OPTION_CONTENT_FORMAT, OptVal) -> + Num = binary:decode_unsigned(OptVal), + {content_format, content_code_to_format(Num)}; +decode_option(?OPTION_MAX_AGE, OptVal) -> {max_age, binary:decode_unsigned(OptVal)}; +decode_option(?OPTION_URI_QUERY, OptVal) -> {uri_query, OptVal}; +decode_option(?OPTION_ACCEPT, OptVal) -> {'accept', binary:decode_unsigned(OptVal)}; +decode_option(?OPTION_LOCATION_QUERY, OptVal) -> {location_query, OptVal}; +decode_option(?OPTION_PROXY_URI, OptVal) -> {proxy_uri, OptVal}; +decode_option(?OPTION_PROXY_SCHEME, OptVal) -> {proxy_scheme, OptVal}; +decode_option(?OPTION_SIZE1, OptVal) -> {size1, binary:decode_unsigned(OptVal)}; +%% draft-ietf-core-observe-16 +decode_option(?OPTION_OBSERVE, OptVal) -> {observe, binary:decode_unsigned(OptVal)}; +%% draft-ietf-core-block-17 +decode_option(?OPTION_BLOCK2, OptVal) -> {block2, decode_block(OptVal)}; +decode_option(?OPTION_BLOCK1, OptVal) -> {block1, decode_block(OptVal)}; +%% unknown option +decode_option(OptNum, OptVal) -> {OptNum, OptVal}. + +decode_block(<>) -> decode_block1(Num, M, SizEx); +decode_block(<>) -> decode_block1(Num, M, SizEx); +decode_block(<>) -> decode_block1(Num, M, SizEx). + +decode_block1(Num, M, SizEx) -> + {Num, M =/= 0, trunc(math:pow(2, SizEx + 4))}. + +-spec content_code_to_format(non_neg_integer()) -> binary(). +content_code_to_format(0) -> <<"text/plain">>; +content_code_to_format(40) -> <<"application/link-format">>; +content_code_to_format(41) -> <<"application/xml">>; +content_code_to_format(42) -> <<"application/octet-stream">>; +content_code_to_format(47) -> <<"application/exi">>; +content_code_to_format(50) -> <<"application/json">>; +content_code_to_format(60) -> <<"application/cbor">>; +content_code_to_format(_) -> <<"application/octet-stream">>. %% use octet as default + +%% RFC 7252 +%% atom indicate a request +class_code_to_method({0, 01}) -> get; +class_code_to_method({0, 02}) -> post; +class_code_to_method({0, 03}) -> put; +class_code_to_method({0, 04}) -> delete; + +%% success is a tuple {ok, ...} +class_code_to_method({2, 01}) -> {ok, created}; +class_code_to_method({2, 02}) -> {ok, deleted}; +class_code_to_method({2, 03}) -> {ok, valid}; +class_code_to_method({2, 04}) -> {ok, changed}; +class_code_to_method({2, 05}) -> {ok, content}; +class_code_to_method({2, 07}) -> {ok, nocontent}; +class_code_to_method({2, 31}) -> {ok, continue}; % block + +%% error is a tuple {error, ...} +class_code_to_method({4, 00}) -> {error, bad_request}; +class_code_to_method({4, 01}) -> {error, unauthorized}; +class_code_to_method({4, 02}) -> {error, bad_option}; +class_code_to_method({4, 03}) -> {error, forbidden}; +class_code_to_method({4, 04}) -> {error, not_found}; +class_code_to_method({4, 05}) -> {error, method_not_allowed}; +class_code_to_method({4, 06}) -> {error, not_acceptable}; +class_code_to_method({4, 08}) -> {error, request_entity_incomplete}; % block +class_code_to_method({4, 12}) -> {error, precondition_failed}; +class_code_to_method({4, 13}) -> {error, request_entity_too_large}; +class_code_to_method({4, 15}) -> {error, unsupported_content_format}; +class_code_to_method({5, 00}) -> {error, internal_server_error}; +class_code_to_method({5, 01}) -> {error, not_implemented}; +class_code_to_method({5, 02}) -> {error, bad_gateway}; +class_code_to_method({5, 03}) -> {error, service_unavailable}; +class_code_to_method({5, 04}) -> {error, gateway_timeout}; +class_code_to_method({5, 05}) -> {error, proxying_not_supported}; +class_code_to_method(_) -> undefined. + +format(Msg) -> + io_lib:format("~p", [Msg]). + +type(_) -> + coap. + +is_message(#coap_message{}) -> + true; +is_message(_) -> + false. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== +-spec is_repeatable_option(message_option_name()) -> boolean(). +is_repeatable_option(if_match) -> true; +is_repeatable_option(etag) -> true; +is_repeatable_option(location_path) -> true; +is_repeatable_option(uri_path) -> true; +is_repeatable_option(uri_query) -> true; +is_repeatable_option(location_query) -> true; +is_repeatable_option(_) -> false. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl new file mode 100644 index 000000000..62cdd3bf2 --- /dev/null +++ b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl @@ -0,0 +1,156 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_coap_impl). + +-include_lib("emqx_gateway/include/emqx_gateway.hrl"). + +-behavior(emqx_gateway_impl). + +%% APIs +-export([ load/0 + , unload/0 + ]). + +-export([ init/1 + , on_insta_create/3 + , on_insta_update/4 + , on_insta_destroy/3 + ]). + +-dialyzer({nowarn_function, [load/0]}). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +load() -> + RegistryOptions = [ {cbkmod, ?MODULE} + ], + Options = [], + emqx_gateway_registry:load(coap, RegistryOptions, Options). + +unload() -> + emqx_gateway_registry:unload(coap). + +init([]) -> + GwState = #{}, + {ok, GwState}. + +%%-------------------------------------------------------------------- +%% emqx_gateway_registry callbacks +%%-------------------------------------------------------------------- + +on_insta_create(_Insta = #{id := InstaId, + rawconf := #{resource := Resource} = RawConf + }, Ctx, _GwState) -> + ResourceMod = get_resource_mod(Resource), + Listeners = emqx_gateway_utils:normalize_rawconf(RawConf), + ListenerPids = lists:map(fun(Lis) -> + start_listener(InstaId, Ctx, ResourceMod, Lis) + end, Listeners), + + {ok, ResCtx} = ResourceMod:init(RawConf), + {ok, ListenerPids, #{ctx => Ctx, + res_ctx => ResCtx}}. + +on_insta_update(NewInsta, OldInsta, GwInstaState = #{ctx := Ctx}, GwState) -> + InstaId = maps:get(id, NewInsta), + try + %% XXX: 1. How hot-upgrade the changes ??? + %% XXX: 2. Check the New confs first before destroy old instance ??? + on_insta_destroy(OldInsta, GwInstaState, GwState), + on_insta_create(NewInsta, Ctx, GwState) + catch + Class : Reason : Stk -> + logger:error("Failed to update coap instance ~s; " + "reason: {~0p, ~0p} stacktrace: ~0p", + [InstaId, Class, Reason, Stk]), + {error, {Class, Reason}} + end. + +on_insta_destroy(_Insta = #{ id := InstaId, + rawconf := #{resource := Resource} = RawConf + }, + #{res_ctx := ResCtx} = _GwInstaState, + _GWState) -> + ResourceMod = get_resource_mod(Resource), + ok = ResourceMod:stop(ResCtx), + Listeners = emqx_gateway_utils:normalize_rawconf(RawConf), + lists:foreach(fun(Lis) -> + stop_listener(InstaId, Lis) + end, Listeners). + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +start_listener(InstaId, Ctx, ResourceMod, {Type, ListenOn, SocketOpts, Cfg}) -> + ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), + Cfg2 = Cfg#{resource => ResourceMod}, + case start_listener(InstaId, Ctx, Type, ListenOn, SocketOpts, Cfg2) of + {ok, Pid} -> + io:format("Start coap ~s:~s listener on ~s successfully.~n", + [InstaId, Type, ListenOnStr]), + Pid; + {error, Reason} -> + io:format(standard_error, + "Failed to start coap ~s:~s listener on ~s: ~0p~n", + [InstaId, Type, ListenOnStr, Reason]), + throw({badconf, Reason}) + end. + +start_listener(InstaId, Ctx, Type, ListenOn, SocketOpts, Cfg) -> + Name = name(InstaId, Type), + NCfg = Cfg#{ + ctx => Ctx, + frame_mod => emqx_coap_frame, + chann_mod => emqx_coap_channel + }, + MFA = {emqx_gateway_conn, start_link, [NCfg]}, + do_start_listener(Type, Name, ListenOn, SocketOpts, MFA). + +do_start_listener(udp, Name, ListenOn, SocketOpts, MFA) -> + esockd:open_udp(Name, ListenOn, SocketOpts, MFA); + +do_start_listener(dtls, Name, ListenOn, SocketOpts, MFA) -> + esockd:open_dtls(Name, ListenOn, SocketOpts, MFA). + +name(InstaId, Type) -> + list_to_atom(lists:concat([InstaId, ":", Type])). + +stop_listener(InstaId, {Type, ListenOn, SocketOpts, Cfg}) -> + StopRet = stop_listener(InstaId, Type, ListenOn, SocketOpts, Cfg), + ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), + case StopRet of + ok -> io:format("Stop coap ~s:~s listener on ~s successfully.~n", + [InstaId, Type, ListenOnStr]); + {error, Reason} -> + io:format(standard_error, + "Failed to stop coap ~s:~s listener on ~s: ~0p~n", + [InstaId, Type, ListenOnStr, Reason] + ) + end, + StopRet. + +stop_listener(InstaId, Type, ListenOn, _SocketOpts, _Cfg) -> + Name = name(InstaId, Type), + esockd:close(Name, ListenOn). + +get_resource_mod(mqtt) -> + emqx_coap_mqtt_resource; +get_resource_mod(pubsub) -> + emqx_coap_pubsub_resource. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_message.erl b/apps/emqx_gateway/src/coap/emqx_coap_message.erl new file mode 100644 index 000000000..52a03c418 --- /dev/null +++ b/apps/emqx_gateway/src/coap/emqx_coap_message.erl @@ -0,0 +1,146 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License at +%% http://www.mozilla.org/MPL/ +%% +%% Copyright (c) 2015 Petr Gotthard + +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% convenience functions for message construction +-module(emqx_coap_message). + +-export([request/2, request/3, request/4, ack/1, response/1, response/2, response/3]). +-export([set/3, set_payload/2, get_content/1, set_content/2, set_content/3, get_option/2]). + +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). + +request(Type, Method) -> + request(Type, Method, <<>>, []). + +request(Type, Method, Payload) -> + request(Type, Method, Payload, []). + +request(Type, Method, Payload, Options) when is_binary(Payload) -> + #coap_message{type = Type, method = Method, payload = Payload, options = Options}; + +request(Type, Method, Content=#coap_content{}, Options) -> + set_content(Content, + #coap_message{type = Type, method = Method, options = Options}). + +ack(Request = #coap_message{}) -> + #coap_message{type = ack, + id = Request#coap_message.id}. + +response(#coap_message{type = Type, + id = Id, + token = Token}) -> + #coap_message{type = Type, + id = Id, + token = Token}. + +response(Method, Request) -> + set_method(Method, response(Request)). + +response(Method, Payload, Request) -> + set_method(Method, + set_payload(Payload, + response(Request))). + +%% omit option for its default value +set(max_age, ?DEFAULT_MAX_AGE, Msg) -> Msg; + +%% set non-default value +set(Option, Value, Msg = #coap_message{options = Options}) -> + Msg#coap_message{options = Options#{Option => Value}}. + +get_option(Option, #coap_message{options = Options}) -> + maps:get(Option, Options, undefined). + +set_method(Method, Msg) -> + Msg#coap_message{method = Method}. + +set_payload(Payload = #coap_content{}, Msg) -> + set_content(Payload, undefined, Msg); + +set_payload(Payload, Msg) when is_binary(Payload) -> + Msg#coap_message{payload = Payload}; + +set_payload(Payload, Msg) when is_list(Payload) -> + Msg#coap_message{payload = list_to_binary(Payload)}. + +get_content(#coap_message{options = Options, payload = Payload}) -> + #coap_content{etag = maps:get(etag, Options, undefined), + max_age = maps:get(max_age, Options, ?DEFAULT_MAX_AGE), + format = maps:get(content_format, Options, undefined), + location_path = maps:get(location_path, Options, []), + payload = Payload}. + +set_content(Content, Msg) -> + set_content(Content, undefined, Msg). + +%% segmentation not requested and not required +set_content(#coap_content{etag = ETag, + max_age = MaxAge, + format = Format, + location_path = LocPath, + payload = Payload}, + undefined, + Msg) + when byte_size(Payload) =< ?MAX_BLOCK_SIZE -> + #coap_message{options = Options} = Msg2 = set_payload(Payload, Msg), + Options2 = Options#{etag => [ETag], + max_age => MaxAge, + content_format => Format, + location_path => LocPath}, + Msg2#coap_message{options = Options2}; + +%% segmentation not requested, but required (late negotiation) +set_content(Content, undefined, Msg) -> + set_content(Content, {0, true, ?MAX_BLOCK_SIZE}, Msg); + +%% segmentation requested (early negotiation) +set_content(#coap_content{etag = ETag, + max_age = MaxAge, + format = Format, + payload = Payload}, + Block, + Msg) -> + #coap_message{options = Options} = Msg2 = set_payload_block(Payload, Block, Msg), + Options2 = Options#{etag => [ETag], + max => MaxAge, + content_format => Format}, + Msg2#coap_message{options = Options2}. + +set_payload_block(Content, Block, Msg = #coap_message{method = Method}) when is_atom(Method) -> + set_payload_block(Content, block1, Block, Msg); + +set_payload_block(Content, Block, Msg = #coap_message{}) -> + set_payload_block(Content, block2, Block, Msg). + +set_payload_block(Content, BlockId, {Num, _, Size}, Msg) -> + ContentSize = erlang:byte_size(Content), + OffsetBegin = Size * Num, + OffsetEnd = OffsetBegin + Size, + case ContentSize > OffsetEnd of + true -> + set(BlockId, {Num, true, Size}, + set_payload(binary:part(Content, OffsetBegin, Size), Msg)); + _ -> + set(BlockId, {Num, false, Size}, + set_payload(binary:part(Content, OffsetBegin, ContentSize - OffsetBegin), Msg)) + end. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_observe_res.erl b/apps/emqx_gateway/src/coap/emqx_coap_observe_res.erl new file mode 100644 index 000000000..199ad0658 --- /dev/null +++ b/apps/emqx_gateway/src/coap/emqx_coap_observe_res.erl @@ -0,0 +1,92 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_coap_observe_res). + +%% API +-export([ new/0, insert/3, remove/2 + , res_changed/2, foreach/2]). +-export_type([manager/0]). + +-define(MAX_SEQ_ID, 16777215). + +-type topic() :: binary(). +-type token() :: binary(). +-type seq_id() :: 0 .. ?MAX_SEQ_ID. +-type res() :: #{ token := token() + , seq_id := seq_id() + }. + +-type manager() :: #{topic => res()}. + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +-spec new() -> manager(). +new() -> + #{}. + +-spec insert(manager(), topic(), token()) -> manager(). +insert(Manager, Topic, Token) -> + case maps:get(Topic, Manager, undefined) of + undefined -> + Manager#{Topic => new_res(Token)}; + _ -> + Manager + end. + +-spec remove(manager(), topic()) -> manager(). +remove(Manager, Topic) -> + maps:remove(Topic, Manager). + +-spec res_changed(manager(), topic()) -> undefined | {token(), seq_id(), manager()}. +res_changed(Manager, Topic) -> + case maps:get(Topic, Manager, undefined) of + undefined -> + undefined; + Res -> + #{token := Token, + seq_id := SeqId} = Res2 = res_changed(Res), + {Token, SeqId, Manager#{Topic := Res2}} + end. + +foreach(F, Manager) -> + maps:fold(fun(K, V, _) -> + F(K, V) + end, + ok, + Manager), + ok. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +-spec new_res(token()) -> res(). +new_res(Token) -> + #{token => Token, + seq_id => 0}. + +-spec res_changed(res()) -> res(). +res_changed(#{seq_id := SeqId} = Res) -> + NewSeqId = SeqId + 1, + NewSeqId2 = + case NewSeqId > ?MAX_SEQ_ID of + true -> + 1; + _ -> + NewSeqId + end, + Res#{seq_id := NewSeqId2}. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_resource.erl b/apps/emqx_gateway/src/coap/emqx_coap_resource.erl new file mode 100644 index 000000000..93fe82aba --- /dev/null +++ b/apps/emqx_gateway/src/coap/emqx_coap_resource.erl @@ -0,0 +1,37 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_coap_resource). + +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). + +-type context() :: any(). +-type topic() :: binary(). +-type token() :: token(). + +-type register() :: {topic(), token()} + | topic() + | undefined. + +-type result() :: emqx_coap_message() + | {has_sub, emqx_coap_message(), register()}. + +-callback init(hocon:confg()) -> context(). +-callback stop(context()) -> ok. +-callback get(emqx_coap_message(), hocon:config()) -> result(). +-callback put(emqx_coap_message(), hocon:config()) -> result(). +-callback post(emqx_coap_message(), hocon:config()) -> result(). +-callback delete(emqx_coap_message(), hocon:config()) -> result(). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_session.erl b/apps/emqx_gateway/src/coap/emqx_coap_session.erl new file mode 100644 index 000000000..dac4ac924 --- /dev/null +++ b/apps/emqx_gateway/src/coap/emqx_coap_session.erl @@ -0,0 +1,195 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_coap_session). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). + +%% API +-export([new/0, transfer_result/3]). + +-export([ received/3 + , reply/4 + , reply/5 + , ack/3 + , piggyback/4 + , deliver/3 + , timeout/3]). + +-export_type([session/0]). + +-record(session, { transport_manager :: emqx_coap_tm:manager() + , observe_manager :: emqx_coap_observe_res:manager() + , next_msg_id :: coap_message_id() + }). + +-type session() :: #session{}. + +%%%------------------------------------------------------------------- +%%% API +%%%------------------------------------------------------------------- +-spec new() -> session(). +new() -> + _ = emqx_misc:rand_seed(), + #session{ transport_manager = emqx_coap_tm:new() + , observe_manager = emqx_coap_observe_res:new() + , next_msg_id = rand:uniform(?MAX_MESSAGE_ID)}. + +%%%------------------------------------------------------------------- +%%% Process Message +%%%------------------------------------------------------------------- +received(Session, Cfg, #coap_message{type = ack} = Msg) -> + handle_response(Session, Cfg, Msg); + +received(Session, Cfg, #coap_message{type = reset} = Msg) -> + handle_response(Session, Cfg, Msg); + +received(Session, Cfg, #coap_message{method = Method} = Msg) when is_atom(Method) -> + handle_request(Session, Cfg, Msg); + +received(Session, Cfg, Msg) -> + handle_response(Session, Cfg, Msg). + +reply(Session, Cfg, Req, Method) -> + reply(Session, Cfg, Req, Method, <<>>). + +reply(Session, Cfg, Req, Method, Payload) -> + Response = emqx_coap_message:response(Method, Payload, Req), + handle_out(Session, Cfg, Response). + +ack(Session, Cfg, Req) -> + piggyback(Session, Cfg, Req, <<>>). + +piggyback(Session, Cfg, Req, Payload) -> + Response = emqx_coap_message:ack(Req), + Response2 = emqx_coap_message:set_payload(Payload, Response), + handle_out(Session, Cfg, Response2). + +deliver(Session, Cfg, Delivers) -> + Fun = fun({_, Topic, Message}, + #{out := OutAcc, + session := #session{observe_manager = OM, + next_msg_id = MsgId} = SAcc} = Acc) -> + case emqx_coap_observe_res:res_changed(OM, Topic) of + undefined -> + Acc; + {Token, SeqId, OM2} -> + Msg = mqtt_to_coap(Message, MsgId, Token, SeqId, Cfg), + SAcc2 = SAcc#session{next_msg_id = next_msg_id(MsgId), + observe_manager = OM2}, + #{out := Out} = Result = call_transport_manager(SAcc2, Cfg, Msg, handle_out), + Result#{out := [Out | OutAcc]} + end + end, + lists:foldl(Fun, + #{out => [], + session => Session}, + Delivers). + +timeout(Session, Cfg, Timer) -> + call_transport_manager(Session, Cfg, Timer, ?FUNCTION_NAME). + +transfer_result(Result, From, Value) -> + ?TRANSFER_RESULT(Result, [out, subscribe], From, Value). + +%%%------------------------------------------------------------------- +%%% Internal functions +%%%------------------------------------------------------------------- +handle_request(Session, Cfg, Msg) -> + call_transport_manager(Session, Cfg, Msg, ?FUNCTION_NAME). + +handle_response(Session, Cfg, Msg) -> + call_transport_manager(Session, Cfg, Msg, ?FUNCTION_NAME). + +handle_out(Session, Cfg, Msg) -> + call_transport_manager(Session, Cfg, Msg, ?FUNCTION_NAME). + +call_transport_manager(#session{transport_manager = TM} = Session, + Cfg, + Msg, + Fun) -> + try + Result = emqx_coap_tm:Fun(Msg, TM, Cfg), + {ok, _, Session2} = emqx_misc:pipeline([fun process_tm/2, + fun process_subscribe/2], + Result, + Session), + emqx_coap_channel:transfer_result(Result, session, Session2) + catch Type:Reason:Stack -> + ?ERROR("process transmission with, message:~p failed~n +Type:~p,Reason:~p~n,StackTrace:~p~n", [Msg, Type, Reason, Stack]), + #{out => emqx_coap_message:response({error, internal_server_error}, Msg)} + end. + +process_tm(#{tm := TM}, Session) -> + {ok, Session#session{transport_manager = TM}}; +process_tm(_, Session) -> + {ok, Session}. + +process_subscribe(#{subscribe := Sub}, #session{observe_manager = OM} = Session) -> + case Sub of + undefined -> + {ok, Session}; + {Topic, Token} -> + OM2 = emqx_coap_observe_res:insert(OM, Topic, Token), + {ok, Session#session{observe_manager = OM2}}; + Topic -> + OM2 = emqx_coap_observe_res:remove(OM, Topic), + {ok, Session#session{observe_manager = OM2}} + end; +process_subscribe(_, Session) -> + {ok, Session}. + +mqtt_to_coap(MQTT, MsgId, Token, SeqId, Cfg) -> + #message{payload = Payload} = MQTT, + #coap_message{type = get_notify_type(MQTT, Cfg), + method = {ok, content}, + id = MsgId, + token = Token, + payload = Payload, + options = #{observe => SeqId, + max_age => get_max_age(MQTT)}}. + +get_notify_type(#message{qos = Qos}, #{notify_type := Type}) -> + case Type of + qos -> + case Qos of + ?QOS_0 -> + non; + _ -> + con + end; + Other -> + Other + end. + +-spec get_max_age(#message{}) -> max_age(). +get_max_age(#message{headers = #{properties := #{'Message-Expiry-Interval' := 0}}}) -> + ?MAXIMUM_MAX_AGE; +get_max_age(#message{headers = #{properties := #{'Message-Expiry-Interval' := Interval}}, + timestamp = Ts}) -> + Now = erlang:system_time(millisecond), + Diff = (Now - Ts + Interval * 1000) / 1000, + erlang:max(1, erlang:floor(Diff)); +get_max_age(_) -> + ?DEFAULT_MAX_AGE. + +next_msg_id(MsgId) when MsgId >= ?MAX_MESSAGE_ID -> + 1; +next_msg_id(MsgId) -> + MsgId + 1. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl b/apps/emqx_gateway/src/coap/emqx_coap_tm.erl new file mode 100644 index 000000000..677292529 --- /dev/null +++ b/apps/emqx_gateway/src/coap/emqx_coap_tm.erl @@ -0,0 +1,196 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% transport manager +-module(emqx_coap_tm). + +-export([ new/0 + , handle_request/3 + , handle_response/3 + , handle_out/3 + , timeout/3]). + +-export_type([manager/0, event_result/2]). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). + +-type direction() :: in | out. +-type transport_id() :: {direction(), non_neg_integer()}. + +-record(transport, { id :: transport_id() + , state :: atom() + , timers :: maps:map() + , data :: any()}). +-type transport() :: #transport{}. + +-type message_id() :: 0 .. ?MAX_MESSAGE_ID. + +-type manager() :: #{message_id() => transport()}. + +-type ttimeout() :: {state_timeout, pos_integer(), any()} + | {stop_timeout, pos_integer()}. + +-type topic() :: binary(). +-type token() :: binary(). +-type sub_register() :: {topic(), token()} | topic(). + +-type event_result(State, Data) :: + #{next => State, + outgoing => emqx_coap_message(), + timeouts => list(ttimeout()), + has_sub => undefined | sub_register(), + data => Data}. + +%%%=================================================================== +%%% API +%%%=================================================================== +new() -> + #{}. + +handle_request(#coap_message{id = MsgId} = Msg, TM, Cfg) -> + Id = {in, MsgId}, + case maps:get(Id, TM, undefined) of + undefined -> + Data = emqx_coap_transport:new(), + Transport = new_transport(Id, Data), + process_event(in, Msg, TM, Transport, Cfg); + TP -> + process_event(in, Msg, TM, TP, Cfg) + end. + +handle_response(#coap_message{type = Type, id = MsgId} = Msg, TM, Cfg) -> + Id = {out, MsgId}, + case maps:get(Id, TM, undefined) of + undefined -> + case Type of + reset -> + ?EMPTY_RESULT; + _ -> + #{out => #coap_message{type = reset, + id = MsgId}} + end; + TP -> + process_event(in, Msg, TM, TP, Cfg) + end. + +handle_out(#coap_message{id = MsgId} = Msg, TM, Cfg) -> + Id = {out, MsgId}, + case maps:get(Id, TM, undefined) of + undefined -> + Data = emqx_coap_transport:new(), + Transport = new_transport(Id, Data), + process_event(out, Msg, TM, Transport, Cfg); + _ -> + ?WARN("Repeat sending message with id:~p~n", [Id]), + ?EMPTY_RESULT + end. + +timeout({Id, Type, Msg}, TM, Cfg) -> + case maps:get(Id, TM, undefined) of + undefined -> + ?EMPTY_RESULT; + #transport{timers = Timers} = TP -> + %% maybe timer has been canceled + case maps:is_key(Type, Timers) of + true -> + process_event(Type, Msg, TM, TP, Cfg); + _ -> + ?EMPTY_RESULT + end + end. + +%%-------------------------------------------------------------------- +%% @doc +%% @spec +%% @end +%%-------------------------------------------------------------------- + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== +new_transport(Id, Data) -> + #transport{id = Id, + state = idle, + timers = #{}, + data = Data}. + +process_event(stop_timeout, + _, + TM, + #transport{id = Id, + timers = Timers}, + _) -> + lists:foreach(fun({_, Ref}) -> + emqx_misc:cancel_timer(Ref) + end, + maps:to_list(Timers)), + #{tm => maps:remove(Id, TM)}; + +process_event(Event, + Msg, + TM, + #transport{id = Id, + state = State, + data = Data} = TP, + Cfg) -> + Result = emqx_coap_transport:State(Event, Msg, Data, Cfg), + {ok, _, TP2} = emqx_misc:pipeline([fun process_state_change/2, + fun process_data_change/2, + fun process_timeouts/2], + Result, + TP), + TM2 = TM#{Id => TP2}, + emqx_coap_session:transfer_result(Result, tm, TM2). + +process_state_change(#{next := Next}, TP) -> + {ok, cancel_state_timer(TP#transport{state = Next})}; +process_state_change(_, TP) -> + {ok, TP}. + +cancel_state_timer(#transport{timers = Timers} = TP) -> + case maps:get(state_timer, Timers, undefined) of + undefined -> + TP; + Ref -> + _ = emqx_misc:cancel_timer(Ref), + TP#transport{timers = maps:remove(state_timer, Timers)} + end. + +process_data_change(#{data := Data}, TP) -> + {ok, TP#transport{data = Data}}; +process_data_change(_, TP) -> + {ok, TP}. + +process_timeouts(#{timeouts := []}, TP) -> + {ok, TP}; +process_timeouts(#{timeouts := Timeouts}, + #transport{id = Id, timers = Timers} = TP) -> + NewTimers = lists:foldl(fun({state_timeout, _, _} = Timer, Acc) -> + process_timer(Id, Timer, Acc); + ({stop_timeout, I}, Acc) -> + process_timer(Id, {stop_timeout, I, stop}, Acc) + end, + Timers, + Timeouts), + {ok, TP#transport{timers = NewTimers}}; + +process_timeouts(_, TP) -> + {ok, TP}. + +process_timer(Id, {Type, Interval, Msg}, Timers) -> + Ref = emqx_misc:start_timer(Interval, {transport, {Id, Type, Msg}}), + Timers#{Type => Ref}. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl b/apps/emqx_gateway/src/coap/emqx_coap_transport.erl new file mode 100644 index 000000000..7363b6254 --- /dev/null +++ b/apps/emqx_gateway/src/coap/emqx_coap_transport.erl @@ -0,0 +1,133 @@ +-module(emqx_coap_transport). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). + +-define(ACK_TIMEOUT, 2000). +-define(ACK_RANDOM_FACTOR, 1000). +-define(MAX_RETRANSMIT, 4). +-define(EXCHANGE_LIFETIME, 247000). +-define(NON_LIFETIME, 145000). + +-record(data, { cache :: undefined | emqx_coap_message() + , retry_interval :: non_neg_integer() + , retry_count :: non_neg_integer() + }). + +-type data() :: #data{}. + +-export([ new/0, idle/4, maybe_reset/4 + , maybe_resend/4, wait_ack/4, until_stop/4]). + +-spec new() -> data(). +new() -> + #data{cache = undefined, + retry_interval = 0, + retry_count = 0}. + +idle(in, + #coap_message{type = non, id = MsgId, method = Method} = Msg, + _, + #{resource := Resource} = Cfg) -> + Ret = #{next => until_stop, + timeouts => [{stop_timeout, ?NON_LIFETIME}]}, + case Method of + undefined -> + Ret#{out => #coap_message{type = reset, id = MsgId}}; + _ -> + case erlang:apply(Resource, Method, [Msg, Cfg]) of + #coap_message{} = Result -> + Ret#{out => Result}; + {has_sub, Result, Sub} -> + Ret#{out => Result, subscribe => Sub}; + error -> + Ret#{out => + emqx_coap_message:response({error, internal_server_error}, Msg)} + end + end; + +idle(in, + #coap_message{id = MsgId, + type = con, + method = Method} = Msg, + Data, + #{resource := Resource} = Cfg) -> + Ret = #{next => maybe_resend, + timeouts =>[{stop_timeout, ?EXCHANGE_LIFETIME}]}, + case Method of + undefined -> + ResetMsg = #coap_message{type = reset, id = MsgId}, + Ret#{data => Data#data{cache = ResetMsg}, + out => ResetMsg}; + _ -> + {RetMsg, SubInfo} = + case erlang:apply(Resource, Method, [Msg, Cfg]) of + #coap_message{} = Result -> + {Result, undefined}; + {has_sub, Result, Sub} -> + {Result, Sub}; + error -> + {emqx_coap_message:response({error, internal_server_error}, Msg), + undefined} + end, + RetMsg2 = RetMsg#coap_message{type = ack}, + Ret#{out => RetMsg2, + data => Data#data{cache = RetMsg2}, + subscribe => SubInfo} + end; + +idle(out, #coap_message{type = non} = Msg, _, _) -> + #{next => maybe_reset, + out => Msg, + timeouts => [{stop_timeout, ?NON_LIFETIME}]}; + +idle(out, Msg, Data, _) -> + _ = emqx_misc:rand_seed(), + Timeout = ?ACK_TIMEOUT + rand:uniform(?ACK_RANDOM_FACTOR), + #{next => wait_ack, + data => Data#data{cache = Msg}, + out => Msg, + timeouts => [ {state_timeout, Timeout, ack_timeout} + , {stop_timeout, ?EXCHANGE_LIFETIME}]}. + +maybe_reset(in, Message, _, _) -> + case Message of + #coap_message{type = reset} -> + ?INFO("Reset Message:~p~n", [Message]); + _ -> + ok + end, + ?EMPTY_RESULT. + +maybe_resend(in, _, _, #data{cache = Cache}) -> + #{out => Cache}. + +wait_ack(in, #coap_message{type = Type}, _, _) -> + case Type of + ack -> + #{next => until_stop}; + reset -> + #{next => until_stop}; + _ -> + ?EMPTY_RESULT + end; + +wait_ack(state_timeout, + ack_timeout, + _, + #data{cache = Msg, + retry_interval = Timeout, + retry_count = Count} =Data) -> + case Count < ?MAX_RETRANSMIT of + true -> + Timeout2 = Timeout * 2, + #{data => Data#data{retry_interval = Timeout2, + retry_count = Count + 1}, + out => Msg, + timeouts => [{state_timeout, Timeout2, ack_timeout}]}; + _ -> + #{next_state => until_stop} + end. + +until_stop(_, _, _, _) -> + ?EMPTY_RESULT. diff --git a/apps/emqx_gateway/src/coap/include/emqx_coap.hrl b/apps/emqx_gateway/src/coap/include/emqx_coap.hrl new file mode 100644 index 000000000..0e0b33365 --- /dev/null +++ b/apps/emqx_gateway/src/coap/include/emqx_coap.hrl @@ -0,0 +1,86 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-define(APP, emqx_coap). +-define(DEFAULT_COAP_PORT, 5683). +-define(DEFAULT_COAPS_PORT, 5684). +-define(MAX_MESSAGE_ID, 65535). +-define(MAX_BLOCK_SIZE, 1024). +-define(DEFAULT_MAX_AGE, 60). +-define(MAXIMUM_MAX_AGE, 4294967295). + +-define(EMPTY_RESULT, #{}). +-define(TRANSFER_RESULT(R1, Keys, From, Value), + begin + R2 = maps:with(Keys, R1), + R2#{From => Value} + end). + +-type coap_message_id() :: 1 .. ?MAX_MESSAGE_ID. +-type message_type() :: con | non | ack | reset. +-type max_age() :: 1 .. ?MAXIMUM_MAX_AGE. + +-type message_option_name() :: if_match + | uri_host + | etag + | if_none_match + | uri_port + | location_path + | uri_path + | content_format + | max_age + | uri_query + | 'accept' + | location_query + | proxy_uri + | proxy_scheme + | size1 + | observer + | block1 + | block2. + +-type message_options() :: #{ if_match => list(binary()) + , uri_host => binary() + , etag => list(binary()) + , if_none_match => boolean() + , uri_port => 0 .. 65535 + , location_path => list(binary()) + , uri_path => list(binary()) + , content_format => 0 .. 65535 + , max_age => non_neg_integer() + , uri_query => list(binary()) + , 'accept' => 0 .. 65535 + , location_query => list(binary()) + , proxy_uri => binary() + , proxy_scheme => binary() + , size1 => non_neg_integer() + , observer => non_neg_integer() + , block1 => {non_neg_integer(), boolean(), non_neg_integer()} + , block2 => {non_neg_integer(), boolean(), non_neg_integer()}}. + +-record(coap_mqtt_auth, {clientid, username, password}). + +-record(coap_message, { type :: message_type() + , method + , id + , token = <<>> + , options = #{} + , payload = <<>>}). + +-record(coap_content, {etag, max_age = ?DEFAULT_MAX_AGE, format, location_path = [], payload = <<>>}). + +-type emqx_coap_message() :: #coap_message{}. +-type coap_content() :: #coap_content{}. diff --git a/apps/emqx_gateway/src/coap/resources/emqx_coap_mqtt_resource.erl b/apps/emqx_gateway/src/coap/resources/emqx_coap_mqtt_resource.erl new file mode 100644 index 000000000..f52610492 --- /dev/null +++ b/apps/emqx_gateway/src/coap/resources/emqx_coap_mqtt_resource.erl @@ -0,0 +1,154 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% a coap to mqtt adapter +-module(emqx_coap_mqtt_resource). + +-behaviour(emqx_coap_resource). + +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). + +-logger_header("[CoAP-RES]"). + +-export([ init/1 + , stop/1 + , get/2 + , put/2 + , post/2 + , delete/2 + ]). + +-export([ check_topic/1 + , publish/3 + , subscribe/3 + , unsubscribe/3]). + +-define(SUBOPTS, #{rh => 0, rap => 0, nl => 0, is_new => false}). + +init(_) -> + {ok, undefined}. + +stop(_) -> + ok. + +%% get: subscribe, ignore observe option +get(#coap_message{token = Token} = Msg, Cfg) -> + case check_topic(Msg) of + {ok, Topic} -> + case Token of + <<>> -> + emqx_coap_message:response({error, bad_request}, <<"observer without token">>, Msg); + _ -> + Ret = subscribe(Msg, Topic, Cfg), + RetMsg = emqx_coap_message:response(Ret, Msg), + case Ret of + {ok, _} -> + {has_sub, RetMsg, {Topic, Token}}; + _ -> + RetMsg + end + end; + Any -> + Any + end. + +%% put: equal post +put(Msg, Cfg) -> + post(Msg, Cfg). + +%% post: publish a message +post(Msg, Cfg) -> + case check_topic(Msg) of + {ok, Topic} -> + emqx_coap_message:response(publish(Msg, Topic, Cfg), Msg); + Any -> + Any + end. + +%% delete: ubsubscribe +delete(Msg, Cfg) -> + case check_topic(Msg) of + {ok, Topic} -> + unsubscribe(Msg, Topic, Cfg), + {has_sub, emqx_coap_message:response({ok, deleted}, Msg), Topic}; + Any -> + Any + end. + +check_topic(#coap_message{options = Options} = Msg) -> + case maps:get(uri_path, Options, []) of + [] -> + emqx_coap_message:response({error, bad_request}, <<"invalid topic">> , Msg); + UriPath -> + Sep = <<"/">>, + {ok, lists:foldl(fun(Part, Acc) -> + <> + end, + <<>>, + UriPath)} + end. + +publish(#coap_message{payload = Payload} = Msg, + Topic, + #{clientinfo := ClientInfo, + publish_qos := QOS} = Cfg) -> + case emqx_coap_channel:auth_publish(Topic, Cfg) of + allow -> + #{clientid := ClientId} = ClientInfo, + MQTTMsg = emqx_message:make(ClientId, type_to_qos(QOS, Msg), Topic, Payload), + MQTTMsg2 = emqx_message:set_flag(retain, false, MQTTMsg), + _ = emqx_broker:publish(MQTTMsg2), + {ok, changed}; + _ -> + {error, unauthorized} + end. + +subscribe(Msg, Topic, #{clientinfo := ClientInfo}= Cfg) -> + case emqx_topic:wildcard(Topic) of + false -> + case emqx_coap_channel:auth_subscribe(Topic, Cfg) of + allow -> + #{clientid := ClientId} = ClientInfo, + SubOpts = get_sub_opts(Msg, Cfg), + emqx_broker:subscribe(Topic, ClientId, SubOpts), + emqx_hooks:run('session.subscribed', [ClientInfo, Topic, SubOpts]), + {ok, created}; + _ -> + {error, unauthorized} + end; + _ -> + %% now, we don't support wildcard in subscribe topic + {error, bad_request, <<"">>} + end. + +unsubscribe(Msg, Topic, #{clientinfo := ClientInfo} = Cfg) -> + emqx_broker:unsubscribe(Topic), + emqx_hooks:run('session.unsubscribed', [ClientInfo, Topic, get_sub_opts(Msg, Cfg)]). + +get_sub_opts(Msg, #{subscribe_qos := Type}) -> + ?SUBOPTS#{qos => type_to_qos(Type, Msg)}. + +type_to_qos(qos0, _) -> ?QOS_0; +type_to_qos(qos1, _) -> ?QOS_1; +type_to_qos(qos2, _) -> ?QOS_2; +type_to_qos(coap, #coap_message{type = Type}) -> + case Type of + non -> + ?QOS_0; + _ -> + ?QOS_1 + end. diff --git a/apps/emqx_gateway/src/coap/resources/emqx_coap_pubsub_resource.erl b/apps/emqx_gateway/src/coap/resources/emqx_coap_pubsub_resource.erl new file mode 100644 index 000000000..c7c13da0c --- /dev/null +++ b/apps/emqx_gateway/src/coap/resources/emqx_coap_pubsub_resource.erl @@ -0,0 +1,220 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% a coap to mqtt adapter with a retained topic message database +-module(emqx_coap_pubsub_resource). + +-behaviour(emqx_coap_resource). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). + +-logger_header("[CoAP-PS-RES]"). + +-export([ init/1 + , stop/1 + , get/2 + , put/2 + , post/2 + , delete/2 + ]). +-import(emqx_coap_mqtt_resource, [ check_topic/1, subscribe/3, unsubscribe/3 + , publish/3]). + +-import(emqx_coap_message, [response/2, response/3, set_content/2]). +%%-------------------------------------------------------------------- +%% Resource Callbacks +%%-------------------------------------------------------------------- +init(_) -> + emqx_coap_pubsub_topics:start_link(). + +stop(Pid) -> + emqx_coap_pubsub_topics:stop(Pid). + +%% get: read last publish message +%% get with observe 0: subscribe +%% get with observe 1: unsubscribe +get(#coap_message{token = Token} = Msg, Cfg) -> + case check_topic(Msg) of + {ok, Topic} -> + case emqx_coap_message:get_option(observe, Msg) of + undefined -> + Content = emqx_coap_message:get_content(Msg), + read_last_publish_message(emqx_topic:wildcard(Topic), Msg, Topic, Content); + 0 -> + case Token of + <<>> -> + response({error, bad_reuqest}, <<"observe without token">>, Msg); + _ -> + Ret = subscribe(Msg, Topic, Cfg), + RetMsg = response(Ret, Msg), + case Ret of + {ok, _} -> + {has_sub, RetMsg, {Topic, Token}}; + _ -> + RetMsg + end + end; + 1 -> + unsubscribe(Msg, Topic, Cfg), + {has_sub, response({ok, deleted}, Msg), Topic} + end; + Any -> + Any + end. + +%% put: insert a message into topic database +put(Msg, _) -> + case check_topic(Msg) of + {ok, Topic} -> + Content = emqx_coap_message:get_content(Msg), + #coap_content{payload = Payload, + format = Format, + max_age = MaxAge} = Content, + handle_received_create(Msg, Topic, MaxAge, Format, Payload); + Any -> + Any + end. + +%% post: like put, but will publish the inserted message +post(Msg, Cfg) -> + case check_topic(Msg) of + {ok, Topic} -> + Content = emqx_coap_message:get_content(Msg), + #coap_content{max_age = MaxAge, + format = Format, + payload = Payload} = Content, + handle_received_publish(Msg, Topic, MaxAge, Format, Payload, Cfg); + Any -> + Any + end. + +%% delete: delete a message from topic database +delete(Msg, _) -> + case check_topic(Msg) of + {ok, Topic} -> + delete_topic_info(Msg, Topic); + Any -> + Any + end. + +%%-------------------------------------------------------------------- +%% Internal Functions +%%-------------------------------------------------------------------- +add_topic_info(Topic, MaxAge, Format, Payload) when is_binary(Topic), Topic =/= <<>> -> + case emqx_coap_pubsub_topics:lookup_topic_info(Topic) of + [{_, StoredMaxAge, StoredCT, _, _}] -> + ?LOG(debug, "publish topic=~p already exists, need reset the topic info", [Topic]), + %% check whether the ct value stored matches the ct option in this POST message + case Format =:= StoredCT of + true -> + {ok, Ret} = + case StoredMaxAge =:= MaxAge of + true -> + emqx_coap_pubsub_topics:reset_topic_info(Topic, Payload); + false -> + emqx_coap_pubsub_topics:reset_topic_info(Topic, MaxAge, Payload) + end, + {changed, Ret}; + false -> + ?LOG(debug, "ct values of topic=~p do not match, stored ct=~p, new ct=~p, ignore the PUBLISH", [Topic, StoredCT, Format]), + {changed, false} + end; + [] -> + ?LOG(debug, "publish topic=~p will be created", [Topic]), + {ok, Ret} = emqx_coap_pubsub_topics:add_topic_info(Topic, MaxAge, Format, Payload), + {created, Ret} + end; + +add_topic_info(Topic, _MaxAge, _Format, _Payload) -> + ?LOG(debug, "create topic=~p info failed", [Topic]), + {badarg, false}. + +format_string_to_int(<<"application/octet-stream">>) -> + <<"42">>; +format_string_to_int(<<"application/exi">>) -> + <<"47">>; +format_string_to_int(<<"application/json">>) -> + <<"50">>; +format_string_to_int(_) -> + <<"42">>. + +handle_received_publish(Msg, Topic, MaxAge, Format, Payload, Cfg) -> + case add_topic_info(Topic, MaxAge, format_string_to_int(Format), Payload) of + {_, true} -> + response(publish(Msg, Topic, Cfg), Msg); + {_, false} -> + ?LOG(debug, "add_topic_info failed, will return bad_request", []), + response({error, bad_request}, Msg) + end. + +handle_received_create(Msg, Topic, MaxAge, Format, Payload) -> + case add_topic_info(Topic, MaxAge, format_string_to_int(Format), Payload) of + {Ret, true} -> + response({ok, Ret}, Msg); + {_, false} -> + ?LOG(debug, "add_topic_info failed, will return bad_request", []), + response({error, bad_request}, Msg) + end. + +return_resource(Msg, Topic, Payload, MaxAge, TimeStamp, Content) -> + TimeElapsed = trunc((erlang:system_time(millisecond) - TimeStamp) / 1000), + case TimeElapsed < MaxAge of + true -> + LeftTime = (MaxAge - TimeElapsed), + ?LOG(debug, "topic=~p has max age left time is ~p", [Topic, LeftTime]), + set_content(Content#coap_content{max_age = LeftTime, payload = Payload}, + response({ok, content}, Msg)); + false -> + ?LOG(debug, "topic=~p has been timeout, will return empty content", [Topic]), + response({ok, nocontent}, Msg) + end. + +read_last_publish_message(false, Msg, Topic, Content=#coap_content{format = QueryFormat}) when is_binary(QueryFormat)-> + ?LOG(debug, "the QueryFormat=~p", [QueryFormat]), + case emqx_coap_pubsub_topics:lookup_topic_info(Topic) of + [] -> + response({error, not_found}, Msg); + [{_, MaxAge, CT, Payload, TimeStamp}] -> + case CT =:= format_string_to_int(QueryFormat) of + true -> + return_resource(Msg, Topic, Payload, MaxAge, TimeStamp, Content); + false -> + ?LOG(debug, "format value does not match, the queried format=~p, the stored format=~p", [QueryFormat, CT]), + response({error, bad_request}, Msg) + end + end; + +read_last_publish_message(false, Msg, Topic, Content) -> + case emqx_coap_pubsub_topics:lookup_topic_info(Topic) of + [] -> + response({error, not_found}, Msg); + [{_, MaxAge, _, Payload, TimeStamp}] -> + return_resource(Msg, Topic, Payload, MaxAge, TimeStamp, Content) + end; + +read_last_publish_message(true, Msg, Topic, _Content) -> + ?LOG(debug, "the topic=~p is illegal wildcard topic", [Topic]), + response({error, bad_request}, Msg). + +delete_topic_info(Msg, Topic) -> + case emqx_coap_pubsub_topics:lookup_topic_info(Topic) of + [] -> + response({error, not_found}, Msg); + [{_, _, _, _, _}] -> + emqx_coap_pubsub_topics:delete_sub_topics(Topic), + response({ok, deleted}, Msg) + end. diff --git a/apps/emqx_coap/src/emqx_coap_pubsub_topics.erl b/apps/emqx_gateway/src/coap/resources/emqx_coap_pubsub_topics.erl similarity index 98% rename from apps/emqx_coap/src/emqx_coap_pubsub_topics.erl rename to apps/emqx_gateway/src/coap/resources/emqx_coap_pubsub_topics.erl index 57d0c3ae6..4ebada566 100644 --- a/apps/emqx_coap/src/emqx_coap_pubsub_topics.erl +++ b/apps/emqx_gateway/src/coap/resources/emqx_coap_pubsub_topics.erl @@ -18,10 +18,8 @@ -behaviour(gen_server). --include("emqx_coap.hrl"). --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). -logger_header("[CoAP-PS-TOPICS]"). diff --git a/apps/emqx_gateway/src/coap/test/emqx_coap_SUITE.erl b/apps/emqx_gateway/src/coap/test/emqx_coap_SUITE.erl new file mode 100644 index 000000000..3f55aa716 --- /dev/null +++ b/apps/emqx_gateway/src/coap/test/emqx_coap_SUITE.erl @@ -0,0 +1,319 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_coap_SUITE). + +% -compile(export_all). +% -compile(nowarn_export_all). + +% -include_lib("gen_coap/include/coap.hrl"). +% -include_lib("eunit/include/eunit.hrl"). +% -include_lib("emqx/include/emqx.hrl"). + +% -define(LOGT(Format, Args), ct:pal(Format, Args)). + +% all() -> emqx_ct:all(?MODULE). + +% init_per_suite(Config) -> +% emqx_ct_helpers:start_apps([emqx_coap], fun set_special_cfg/1), +% Config. + +% set_special_cfg(emqx_coap) -> +% Opts = application:get_env(emqx_coap, dtls_opts,[]), +% Opts2 = [{keyfile, emqx_ct_helpers:deps_path(emqx, "etc/certs/key.pem")}, +% {certfile, emqx_ct_helpers:deps_path(emqx, "etc/certs/cert.pem")}], +% application:set_env(emqx_coap, dtls_opts, emqx_misc:merge_opts(Opts, Opts2)), +% application:set_env(emqx_coap, enable_stats, true); +% set_special_cfg(_) -> +% ok. + +% end_per_suite(Config) -> +% emqx_ct_helpers:stop_apps([emqx_coap]), +% Config. + +% %%-------------------------------------------------------------------- +% %% Test Cases +% %%-------------------------------------------------------------------- + +% t_publish(_Config) -> +% Topic = <<"abc">>, Payload = <<"123">>, +% TopicStr = binary_to_list(Topic), +% URI = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", + +% %% Sub topic first +% emqx:subscribe(Topic), + +% Reply = er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), +% {ok, changed, _} = Reply, + +% receive +% {deliver, Topic, Msg} -> +% ?assertEqual(Topic, Msg#message.topic), +% ?assertEqual(Payload, Msg#message.payload) +% after +% 500 -> +% ?assert(false) +% end. + +% t_publish_authz_deny(_Config) -> +% Topic = <<"abc">>, Payload = <<"123">>, +% TopicStr = binary_to_list(Topic), +% URI = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", + +% %% Sub topic first +% emqx:subscribe(Topic), + +% ok = meck:new(emqx_access_control, [non_strict, passthrough, no_history]), +% ok = meck:expect(emqx_access_control, authorize, 3, deny), +% Reply = er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), +% ?assertEqual({error,forbidden}, Reply), +% ok = meck:unload(emqx_access_control), +% receive +% {deliver, Topic, Msg} -> ct:fail({unexpected, {Topic, Msg}}) +% after +% 500 -> ok +% end. + +% t_observe(_Config) -> +% Topic = <<"abc">>, TopicStr = binary_to_list(Topic), +% Payload = <<"123">>, +% Uri = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", +% {ok, Pid, N, Code, Content} = er_coap_observer:observe(Uri), +% ?LOGT("observer Pid=~p, N=~p, Code=~p, Content=~p", [Pid, N, Code, Content]), + +% [SubPid] = emqx:subscribers(Topic), +% ?assert(is_pid(SubPid)), + +% %% Publish a message +% emqx:publish(emqx_message:make(Topic, Payload)), + +% Notif = receive_notification(), +% ?LOGT("observer get Notif=~p", [Notif]), +% {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv}} = Notif, +% ?assertEqual(Payload, PayloadRecv), + +% er_coap_observer:stop(Pid), +% timer:sleep(100), + +% [] = emqx:subscribers(Topic). + +% t_observe_authz_deny(_Config) -> +% Topic = <<"abc">>, TopicStr = binary_to_list(Topic), +% Uri = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", +% ok = meck:new(emqx_access_control, [non_strict, passthrough, no_history]), +% ok = meck:expect(emqx_access_control, authorize, 3, deny), +% ?assertEqual({error,forbidden}, er_coap_observer:observe(Uri)), +% [] = emqx:subscribers(Topic), +% ok = meck:unload(emqx_access_control). + +% t_observe_wildcard(_Config) -> +% Topic = <<"+/b">>, TopicStr = emqx_http_lib:uri_encode(binary_to_list(Topic)), +% Payload = <<"123">>, +% Uri = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", +% {ok, Pid, N, Code, Content} = er_coap_observer:observe(Uri), +% ?LOGT("observer Uri=~p, Pid=~p, N=~p, Code=~p, Content=~p", [Uri, Pid, N, Code, Content]), + +% [SubPid] = emqx:subscribers(Topic), +% ?assert(is_pid(SubPid)), + +% %% Publish a message +% emqx:publish(emqx_message:make(<<"a/b">>, Payload)), + +% Notif = receive_notification(), +% ?LOGT("observer get Notif=~p", [Notif]), +% {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv}} = Notif, +% ?assertEqual(Payload, PayloadRecv), + +% er_coap_observer:stop(Pid), +% timer:sleep(100), + +% [] = emqx:subscribers(Topic). + +% t_observe_pub(_Config) -> +% Topic = <<"+/b">>, TopicStr = emqx_http_lib:uri_encode(binary_to_list(Topic)), +% Uri = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", +% {ok, Pid, N, Code, Content} = er_coap_observer:observe(Uri), +% ?LOGT("observer Pid=~p, N=~p, Code=~p, Content=~p", [Pid, N, Code, Content]), + +% [SubPid] = emqx:subscribers(Topic), +% ?assert(is_pid(SubPid)), + +% Topic2 = <<"a/b">>, Payload2 = <<"UFO">>, +% TopicStr2 = emqx_http_lib:uri_encode(binary_to_list(Topic2)), +% URI2 = "coap://127.0.0.1/mqtt/"++TopicStr2++"?c=client1&u=tom&p=secret", + +% Reply2 = er_coap_client:request(put, URI2, #coap_content{format = <<"application/octet-stream">>, payload = Payload2}), +% {ok,changed, _} = Reply2, + +% Notif2 = receive_notification(), +% ?LOGT("observer get Notif2=~p", [Notif2]), +% {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv2}} = Notif2, +% ?assertEqual(Payload2, PayloadRecv2), + +% Topic3 = <<"j/b">>, Payload3 = <<"ET629">>, +% TopicStr3 = emqx_http_lib:uri_encode(binary_to_list(Topic3)), +% URI3 = "coap://127.0.0.1/mqtt/"++TopicStr3++"?c=client2&u=mike&p=guess", +% Reply3 = er_coap_client:request(put, URI3, #coap_content{format = <<"application/octet-stream">>, payload = Payload3}), +% {ok,changed, _} = Reply3, + +% Notif3 = receive_notification(), +% ?LOGT("observer get Notif3=~p", [Notif3]), +% {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv3}} = Notif3, +% ?assertEqual(Payload3, PayloadRecv3), + +% er_coap_observer:stop(Pid). + +% t_one_clientid_sub_2_topics(_Config) -> +% Topic1 = <<"abc">>, TopicStr1 = binary_to_list(Topic1), +% Payload1 = <<"123">>, +% Uri1 = "coap://127.0.0.1/mqtt/"++TopicStr1++"?c=client1&u=tom&p=secret", +% {ok, Pid1, N1, Code1, Content1} = er_coap_observer:observe(Uri1), +% ?LOGT("observer 1 Pid=~p, N=~p, Code=~p, Content=~p", [Pid1, N1, Code1, Content1]), + +% [SubPid] = emqx:subscribers(Topic1), +% ?assert(is_pid(SubPid)), + +% Topic2 = <<"x/y">>, TopicStr2 = emqx_http_lib:uri_encode(binary_to_list(Topic2)), +% Payload2 = <<"456">>, +% Uri2 = "coap://127.0.0.1/mqtt/"++TopicStr2++"?c=client1&u=tom&p=secret", +% {ok, Pid2, N2, Code2, Content2} = er_coap_observer:observe(Uri2), +% ?LOGT("observer 2 Pid=~p, N=~p, Code=~p, Content=~p", [Pid2, N2, Code2, Content2]), + +% [SubPid] = emqx:subscribers(Topic2), +% ?assert(is_pid(SubPid)), + +% emqx:publish(emqx_message:make(Topic1, Payload1)), + +% Notif1 = receive_notification(), +% ?LOGT("observer 1 get Notif=~p", [Notif1]), +% {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv1}} = Notif1, +% ?assertEqual(Payload1, PayloadRecv1), + +% emqx:publish(emqx_message:make(Topic2, Payload2)), + +% Notif2 = receive_notification(), +% ?LOGT("observer 2 get Notif=~p", [Notif2]), +% {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv2}} = Notif2, +% ?assertEqual(Payload2, PayloadRecv2), + +% er_coap_observer:stop(Pid1), +% er_coap_observer:stop(Pid2). + +% t_invalid_parameter(_Config) -> +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% %% "cid=client2" is invaid +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Topic3 = <<"a/b">>, Payload3 = <<"ET629">>, +% TopicStr3 = emqx_http_lib:uri_encode(binary_to_list(Topic3)), +% URI3 = "coap://127.0.0.1/mqtt/"++TopicStr3++"?cid=client2&u=tom&p=simple", +% Reply3 = er_coap_client:request(put, URI3, #coap_content{format = <<"application/octet-stream">>, payload = Payload3}), +% ?assertMatch({error,bad_request}, Reply3), + +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% %% "what=hello" is invaid +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% URI4 = "coap://127.0.0.1/mqtt/"++TopicStr3++"?what=hello", +% Reply4 = er_coap_client:request(put, URI4, #coap_content{format = <<"application/octet-stream">>, payload = Payload3}), +% ?assertMatch({error, bad_request}, Reply4). + +% t_invalid_topic(_Config) -> +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% %% "a/b" is a valid topic string +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Topic3 = <<"a/b">>, Payload3 = <<"ET629">>, +% TopicStr3 = binary_to_list(Topic3), +% URI3 = "coap://127.0.0.1/mqtt/"++TopicStr3++"?c=client2&u=tom&p=simple", +% Reply3 = er_coap_client:request(put, URI3, #coap_content{format = <<"application/octet-stream">>, payload = Payload3}), +% ?assertMatch({ok,changed,_Content}, Reply3), + +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% %% "+?#" is invaid topic string +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% URI4 = "coap://127.0.0.1/mqtt/"++"+?#"++"?what=hello", +% Reply4 = er_coap_client:request(put, URI4, #coap_content{format = <<"application/octet-stream">>, payload = Payload3}), +% ?assertMatch({error,bad_request}, Reply4). + +% % mqtt connection kicked by coap with same client id +% t_kick_1(_Config) -> +% URI = "coap://127.0.0.1/mqtt/abc?c=clientid&u=tom&p=secret", +% % workaround: emqx:subscribe does not kick same client id. +% spawn_monitor(fun() -> +% {ok, C} = emqtt:start_link([{host, "localhost"}, +% {clientid, <<"clientid">>}, +% {username, <<"plain">>}, +% {password, <<"plain">>}]), +% {ok, _} = emqtt:connect(C) end), +% er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, +% payload = <<"123">>}), +% receive +% {'DOWN', _, _, _, _} -> ok +% after 2000 -> +% ?assert(false) +% end. + +% % mqtt connection kicked by coap with same client id +% t_authz(_Config) -> +% OldPath = emqx:get_env(plugins_etc_dir), +% application:set_env(emqx, plugins_etc_dir, +% emqx_ct_helpers:deps_path(emqx_authz, "test")), +% Conf = #{<<"authz">> => +% #{<<"rules">> => +% [#{<<"principal">> =>#{<<"username">> => <<"coap">>}, +% <<"permission">> => deny, +% <<"topics">> => [<<"abc">>], +% <<"action">> => <<"publish">>} +% ]}}, +% ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)), +% application:ensure_all_started(emqx_authz), + +% emqx:subscribe(<<"abc">>), +% URI = "coap://127.0.0.1/mqtt/adbc?c=client1&u=coap&p=secret", +% er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, +% payload = <<"123">>}), +% receive +% _Something -> ?assert(false) +% after 2000 -> +% ok +% end, + +% ok = emqx_hooks:del('client.authorize', {emqx_authz, authorize}), +% file:delete(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf')), +% application:set_env(emqx, plugins_etc_dir, OldPath), +% application:stop(emqx_authz). + +% t_stats(_) -> +% ok. + +% t_auth_failure(_) -> +% ok. + +% t_qos_supprot(_) -> +% ok. + +% %%-------------------------------------------------------------------- +% %% Helpers + +% receive_notification() -> +% receive +% {coap_notify, Pid, N2, Code2, Content2} -> +% {coap_notify, Pid, N2, Code2, Content2} +% after 2000 -> +% receive_notification_timeout +% end. + +% testdir(DataPath) -> +% Ls = filename:split(DataPath), +% filename:join(lists:sublist(Ls, 1, length(Ls) - 1)). diff --git a/apps/emqx_gateway/src/coap/test/emqx_coap_pubsub_SUITE.erl b/apps/emqx_gateway/src/coap/test/emqx_coap_pubsub_SUITE.erl new file mode 100644 index 000000000..403cd8b2b --- /dev/null +++ b/apps/emqx_gateway/src/coap/test/emqx_coap_pubsub_SUITE.erl @@ -0,0 +1,678 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_coap_pubsub_SUITE). + +% -compile(export_all). +% -compile(nowarn_export_all). + + +% -include_lib("gen_coap/include/coap.hrl"). +% -include_lib("eunit/include/eunit.hrl"). +% -include_lib("emqx/include/emqx.hrl"). + +% -define(LOGT(Format, Args), ct:pal(Format, Args)). + +% all() -> emqx_ct:all(?MODULE). + +% init_per_suite(Config) -> +% emqx_ct_helpers:start_apps([emqx_coap], fun set_special_cfg/1), +% Config. + +% set_special_cfg(emqx_coap) -> +% application:set_env(emqx_coap, enable_stats, true); +% set_special_cfg(_) -> +% ok. + +% end_per_suite(Config) -> +% emqx_ct_helpers:stop_apps([emqx_coap]), +% Config. + +% %%-------------------------------------------------------------------- +% %% Test Cases +% %%-------------------------------------------------------------------- + +% t_update_max_age(_Config) -> +% TopicInPayload = <<"topic1">>, +% Payload = <<";ct=42">>, +% Payload1 = <<";ct=50">>, +% URI = "coap://127.0.0.1/ps/"++"?c=client1&u=tom&p=secret", +% URI2 = "coap://127.0.0.1/ps/topic1"++"?c=client1&u=tom&p=secret", +% Reply = er_coap_client:request(post, URI, #coap_content{format = <<"application/link-format">>, payload = Payload}), +% ?LOGT("Reply =~p", [Reply]), +% {ok,created, #coap_content{location_path = LocPath}} = Reply, +% ?assertEqual([<<"/ps/topic1">>] ,LocPath), +% TopicInfo = [{TopicInPayload, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(TopicInPayload), +% ?LOGT("lookup topic info=~p", [TopicInfo]), +% ?assertEqual(60, MaxAge1), +% ?assertEqual(<<"42">>, CT1), + +% timer:sleep(50), + +% %% post to create the same topic but with different max age and ct value in payload +% Reply1 = er_coap_client:request(post, URI, #coap_content{max_age = 70, format = <<"application/link-format">>, payload = Payload1}), +% {ok,created, #coap_content{location_path = LocPath}} = Reply1, +% ?assertEqual([<<"/ps/topic1">>] ,LocPath), +% [{TopicInPayload, MaxAge2, CT2, _ResPayload, _TimeStamp1}] = emqx_coap_pubsub_topics:lookup_topic_info(TopicInPayload), +% ?assertEqual(70, MaxAge2), +% ?assertEqual(<<"50">>, CT2), + +% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI2). + +% t_create_subtopic(_Config) -> +% TopicInPayload = <<"topic1">>, +% TopicInPayloadStr = "topic1", +% Payload = <<";ct=42">>, +% URI = "coap://127.0.0.1/ps/"++"?c=client1&u=tom&p=secret", +% RealURI = "coap://127.0.0.1/ps/topic1"++"?c=client1&u=tom&p=secret", + +% Reply = er_coap_client:request(post, URI, #coap_content{format = <<"application/link-format">>, payload = Payload}), +% ?LOGT("Reply =~p", [Reply]), +% {ok,created, #coap_content{location_path = LocPath}} = Reply, +% ?assertEqual([<<"/ps/topic1">>] ,LocPath), +% TopicInfo = [{TopicInPayload, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(TopicInPayload), +% ?LOGT("lookup topic info=~p", [TopicInfo]), +% ?assertEqual(60, MaxAge1), +% ?assertEqual(<<"42">>, CT1), + +% timer:sleep(50), + +% %% post to create the a sub topic +% SubPayload = <<";ct=42">>, +% SubTopicInPayloadStr = "subtopic", +% SubURI = "coap://127.0.0.1/ps/"++TopicInPayloadStr++"?c=client1&u=tom&p=secret", +% SubRealURI = "coap://127.0.0.1/ps/"++TopicInPayloadStr++"/"++SubTopicInPayloadStr++"?c=client1&u=tom&p=secret", +% FullTopic = list_to_binary(TopicInPayloadStr++"/"++SubTopicInPayloadStr), +% Reply1 = er_coap_client:request(post, SubURI, #coap_content{format = <<"application/link-format">>, payload = SubPayload}), +% ?LOGT("Reply =~p", [Reply1]), +% {ok,created, #coap_content{location_path = LocPath1}} = Reply1, +% ?assertEqual([<<"/ps/topic1/subtopic">>] ,LocPath1), +% [{FullTopic, MaxAge2, CT2, _ResPayload, _}] = emqx_coap_pubsub_topics:lookup_topic_info(FullTopic), +% ?assertEqual(60, MaxAge2), +% ?assertEqual(<<"42">>, CT2), + +% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, SubRealURI), +% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, RealURI). + +% t_over_max_age(_Config) -> +% TopicInPayload = <<"topic1">>, +% Payload = <<";ct=42">>, +% URI = "coap://127.0.0.1/ps/"++"?c=client1&u=tom&p=secret", +% Reply = er_coap_client:request(post, URI, #coap_content{max_age = 2, format = <<"application/link-format">>, payload = Payload}), +% ?LOGT("Reply =~p", [Reply]), +% {ok,created, #coap_content{location_path = LocPath}} = Reply, +% ?assertEqual([<<"/ps/topic1">>] ,LocPath), +% TopicInfo = [{TopicInPayload, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(TopicInPayload), +% ?LOGT("lookup topic info=~p", [TopicInfo]), +% ?assertEqual(2, MaxAge1), +% ?assertEqual(<<"42">>, CT1), + +% timer:sleep(3000), +% ?assertEqual(true, emqx_coap_pubsub_topics:is_topic_timeout(TopicInPayload)). + +% t_refreash_max_age(_Config) -> +% TopicInPayload = <<"topic1">>, +% Payload = <<";ct=42">>, +% Payload1 = <<";ct=50">>, +% URI = "coap://127.0.0.1/ps/"++"?c=client1&u=tom&p=secret", +% RealURI = "coap://127.0.0.1/ps/topic1"++"?c=client1&u=tom&p=secret", +% Reply = er_coap_client:request(post, URI, #coap_content{max_age = 5, format = <<"application/link-format">>, payload = Payload}), +% ?LOGT("Reply =~p", [Reply]), +% {ok,created, #coap_content{location_path = LocPath}} = Reply, +% ?assertEqual([<<"/ps/topic1">>] ,LocPath), +% TopicInfo = [{TopicInPayload, MaxAge1, CT1, _ResPayload, TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(TopicInPayload), +% ?LOGT("lookup topic info=~p", [TopicInfo]), +% ?LOGT("TimeStamp=~p", [TimeStamp]), +% ?assertEqual(5, MaxAge1), +% ?assertEqual(<<"42">>, CT1), + +% timer:sleep(3000), + +% %% post to create the same topic, the max age timer will be restarted with the new max age value +% Reply1 = er_coap_client:request(post, URI, #coap_content{max_age = 5, format = <<"application/link-format">>, payload = Payload1}), +% {ok,created, #coap_content{location_path = LocPath}} = Reply1, +% ?assertEqual([<<"/ps/topic1">>] ,LocPath), +% [{TopicInPayload, MaxAge2, CT2, _ResPayload, TimeStamp1}] = emqx_coap_pubsub_topics:lookup_topic_info(TopicInPayload), +% ?LOGT("TimeStamp1=~p", [TimeStamp1]), +% ?assertEqual(5, MaxAge2), +% ?assertEqual(<<"50">>, CT2), + +% timer:sleep(3000), +% ?assertEqual(false, emqx_coap_pubsub_topics:is_topic_timeout(TopicInPayload)), + +% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, RealURI). + +% t_case01_publish_post(_Config) -> +% timer:sleep(100), +% MainTopic = <<"maintopic">>, +% TopicInPayload = <<"topic1">>, +% Payload = <<";ct=42">>, +% MainTopicStr = binary_to_list(MainTopic), + +% %% post to create topic maintopic/topic1 +% URI1 = "coap://127.0.0.1/ps/"++MainTopicStr++"?c=client1&u=tom&p=secret", +% FullTopic = list_to_binary(MainTopicStr++"/"++binary_to_list(TopicInPayload)), +% Reply1 = er_coap_client:request(post, URI1, #coap_content{format = <<"application/link-format">>, payload = Payload}), +% ?LOGT("Reply =~p", [Reply1]), +% {ok,created, #coap_content{location_path = LocPath1}} = Reply1, +% ?assertEqual([<<"/ps/maintopic/topic1">>] ,LocPath1), +% [{FullTopic, MaxAge, CT2, <<>>, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(FullTopic), +% ?assertEqual(60, MaxAge), +% ?assertEqual(<<"42">>, CT2), + +% %% post to publish message to topic maintopic/topic1 +% FullTopicStr = emqx_http_lib:uri_encode(binary_to_list(FullTopic)), +% URI2 = "coap://127.0.0.1/ps/"++FullTopicStr++"?c=client1&u=tom&p=secret", +% PubPayload = <<"PUBLISH">>, + +% %% Sub topic first +% emqx:subscribe(FullTopic), + +% Reply2 = er_coap_client:request(post, URI2, #coap_content{format = <<"application/octet-stream">>, payload = PubPayload}), +% ?LOGT("Reply =~p", [Reply2]), +% {ok,changed, _} = Reply2, +% TopicInfo = [{FullTopic, MaxAge, CT2, PubPayload, _TimeStamp1}] = emqx_coap_pubsub_topics:lookup_topic_info(FullTopic), +% ?LOGT("the topic info =~p", [TopicInfo]), + +% assert_recv(FullTopic, PubPayload), +% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI2). + +% t_case02_publish_post(_Config) -> +% Topic = <<"topic1">>, +% TopicStr = binary_to_list(Topic), +% Payload = <<"payload">>, + +% %% Sub topic first +% emqx:subscribe(Topic), + +% %% post to publish a new topic "topic1", and the topic is created +% URI = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", +% Reply = er_coap_client:request(post, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), +% ?LOGT("Reply =~p", [Reply]), +% {ok,created, #coap_content{location_path = LocPath}} = Reply, +% ?assertEqual([<<"/ps/topic1">>] ,LocPath), +% [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), +% ?assertEqual(60, MaxAge), +% ?assertEqual(<<"42">>, CT), + +% assert_recv(Topic, Payload), + +% %% post to publish a new message to the same topic "topic1" with different payload +% NewPayload = <<"newpayload">>, +% Reply1 = er_coap_client:request(post, URI, #coap_content{format = <<"application/octet-stream">>, payload = NewPayload}), +% ?LOGT("Reply =~p", [Reply1]), +% {ok,changed, _} = Reply1, +% [{Topic, MaxAge, CT, NewPayload, _TimeStamp1}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), + +% assert_recv(Topic, NewPayload), +% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). + +% t_case03_publish_post(_Config) -> +% Topic = <<"topic1">>, +% TopicStr = binary_to_list(Topic), +% Payload = <<"payload">>, + +% %% Sub topic first +% emqx:subscribe(Topic), + +% %% post to publish a new topic "topic1", and the topic is created +% URI = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", +% Reply = er_coap_client:request(post, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), +% ?LOGT("Reply =~p", [Reply]), +% {ok,created, #coap_content{location_path = LocPath}} = Reply, +% ?assertEqual([<<"/ps/topic1">>] ,LocPath), +% [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), +% ?assertEqual(60, MaxAge), +% ?assertEqual(<<"42">>, CT), + +% assert_recv(Topic, Payload), + +% %% post to publish a new message to the same topic "topic1", but the ct is not same as created +% NewPayload = <<"newpayload">>, +% Reply1 = er_coap_client:request(post, URI, #coap_content{format = <<"application/exi">>, payload = NewPayload}), +% ?LOGT("Reply =~p", [Reply1]), +% ?assertEqual({error,bad_request}, Reply1), + +% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). + +% t_case04_publish_post(_Config) -> +% Topic = <<"topic1">>, +% TopicStr = binary_to_list(Topic), +% Payload = <<"payload">>, + +% %% post to publish a new topic "topic1", and the topic is created +% URI = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", +% Reply = er_coap_client:request(post, URI, #coap_content{max_age = 5, format = <<"application/octet-stream">>, payload = Payload}), +% ?LOGT("Reply =~p", [Reply]), +% {ok,created, #coap_content{location_path = LocPath}} = Reply, +% ?assertEqual([<<"/ps/topic1">>] ,LocPath), +% [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), +% ?assertEqual(5, MaxAge), +% ?assertEqual(<<"42">>, CT), + +% %% after max age timeout, the topic still exists but the status is timeout +% timer:sleep(6000), +% ?assertEqual(true, emqx_coap_pubsub_topics:is_topic_timeout(Topic)), + +% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). + +% t_case01_publish_put(_Config) -> +% MainTopic = <<"maintopic">>, +% TopicInPayload = <<"topic1">>, +% Payload = <<";ct=42">>, +% MainTopicStr = binary_to_list(MainTopic), + +% %% post to create topic maintopic/topic1 +% URI1 = "coap://127.0.0.1/ps/"++MainTopicStr++"?c=client1&u=tom&p=secret", +% FullTopic = list_to_binary(MainTopicStr++"/"++binary_to_list(TopicInPayload)), +% Reply1 = er_coap_client:request(post, URI1, #coap_content{format = <<"application/link-format">>, payload = Payload}), +% ?LOGT("Reply =~p", [Reply1]), +% {ok,created, #coap_content{location_path = LocPath1}} = Reply1, +% ?assertEqual([<<"/ps/maintopic/topic1">>] ,LocPath1), +% [{FullTopic, MaxAge, CT2, <<>>, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(FullTopic), +% ?assertEqual(60, MaxAge), +% ?assertEqual(<<"42">>, CT2), + +% %% put to publish message to topic maintopic/topic1 +% FullTopicStr = emqx_http_lib:uri_encode(binary_to_list(FullTopic)), +% URI2 = "coap://127.0.0.1/ps/"++FullTopicStr++"?c=client1&u=tom&p=secret", +% PubPayload = <<"PUBLISH">>, + +% %% Sub topic first +% emqx:subscribe(FullTopic), + +% Reply2 = er_coap_client:request(put, URI2, #coap_content{format = <<"application/octet-stream">>, payload = PubPayload}), +% ?LOGT("Reply =~p", [Reply2]), +% {ok,changed, _} = Reply2, +% [{FullTopic, MaxAge, CT2, PubPayload, _TimeStamp1}] = emqx_coap_pubsub_topics:lookup_topic_info(FullTopic), + +% assert_recv(FullTopic, PubPayload), + +% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI2). + +% t_case02_publish_put(_Config) -> +% Topic = <<"topic1">>, +% TopicStr = binary_to_list(Topic), +% Payload = <<"payload">>, + +% %% Sub topic first +% emqx:subscribe(Topic), + +% %% put to publish a new topic "topic1", and the topic is created +% URI = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", +% Reply = er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), +% ?LOGT("Reply =~p", [Reply]), +% {ok,created, #coap_content{location_path = LocPath}} = Reply, +% ?assertEqual([<<"/ps/topic1">>] ,LocPath), +% [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), +% ?assertEqual(60, MaxAge), +% ?assertEqual(<<"42">>, CT), + +% assert_recv(Topic, Payload), + +% %% put to publish a new message to the same topic "topic1" with different payload +% NewPayload = <<"newpayload">>, +% Reply1 = er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, payload = NewPayload}), +% ?LOGT("Reply =~p", [Reply1]), +% {ok,changed, _} = Reply1, +% [{Topic, MaxAge, CT, NewPayload, _TimeStamp1}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), + +% assert_recv(Topic, NewPayload), + +% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). + +% t_case03_publish_put(_Config) -> +% Topic = <<"topic1">>, +% TopicStr = binary_to_list(Topic), +% Payload = <<"payload">>, + +% %% Sub topic first +% emqx:subscribe(Topic), + +% %% put to publish a new topic "topic1", and the topic is created +% URI = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", +% Reply = er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), +% ?LOGT("Reply =~p", [Reply]), +% {ok,created, #coap_content{location_path = LocPath}} = Reply, +% ?assertEqual([<<"/ps/topic1">>] ,LocPath), +% [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), +% ?assertEqual(60, MaxAge), +% ?assertEqual(<<"42">>, CT), + +% assert_recv(Topic, Payload), + +% %% put to publish a new message to the same topic "topic1", but the ct is not same as created +% NewPayload = <<"newpayload">>, +% Reply1 = er_coap_client:request(put, URI, #coap_content{format = <<"application/exi">>, payload = NewPayload}), +% ?LOGT("Reply =~p", [Reply1]), +% ?assertEqual({error,bad_request}, Reply1), + +% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). + +% t_case04_publish_put(_Config) -> +% Topic = <<"topic1">>, +% TopicStr = binary_to_list(Topic), +% Payload = <<"payload">>, + +% %% put to publish a new topic "topic1", and the topic is created +% URI = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", +% Reply = er_coap_client:request(put, URI, #coap_content{max_age = 5, format = <<"application/octet-stream">>, payload = Payload}), +% ?LOGT("Reply =~p", [Reply]), +% {ok,created, #coap_content{location_path = LocPath}} = Reply, +% ?assertEqual([<<"/ps/topic1">>] ,LocPath), +% [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), +% ?assertEqual(5, MaxAge), +% ?assertEqual(<<"42">>, CT), + +% %% after max age timeout, no publish message to the same topic, the topic info will be deleted +% %%%%%%%%%%%%%%%%%%%%%%%%%% +% % but there is one thing to do is we don't count in the publish message received from emqx(from other node).TBD!!!!!!!!!!!!! +% %%%%%%%%%%%%%%%%%%%%%%%%%% +% timer:sleep(6000), +% ?assertEqual(true, emqx_coap_pubsub_topics:is_topic_timeout(Topic)), + +% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). + +% t_case01_subscribe(_Config) -> +% Topic = <<"topic1">>, +% Payload1 = <<";ct=42">>, +% timer:sleep(100), + +% %% First post to create a topic "topic1" +% Uri = "coap://127.0.0.1/ps/"++"?c=client1&u=tom&p=secret", +% Reply = er_coap_client:request(post, Uri, #coap_content{format = <<"application/link-format">>, payload = Payload1}), +% ?LOGT("Reply =~p", [Reply]), +% {ok,created, #coap_content{location_path = [LocPath]}} = Reply, +% ?assertEqual(<<"/ps/topic1">> ,LocPath), +% TopicInfo = [{Topic, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), +% ?LOGT("lookup topic info=~p", [TopicInfo]), +% ?assertEqual(60, MaxAge1), +% ?assertEqual(<<"42">>, CT1), + +% %% Subscribe the topic +% Uri1 = "coap://127.0.0.1"++binary_to_list(LocPath)++"?c=client1&u=tom&p=secret", +% {ok, Pid, N, Code, Content} = er_coap_observer:observe(Uri1), +% ?LOGT("observer Pid=~p, N=~p, Code=~p, Content=~p", [Pid, N, Code, Content]), + +% [SubPid] = emqx:subscribers(Topic), +% ?assert(is_pid(SubPid)), + +% %% Publish a message +% Payload = <<"123">>, +% emqx:publish(emqx_message:make(Topic, Payload)), + +% Notif = receive_notification(), +% ?LOGT("observer get Notif=~p", [Notif]), +% {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv}} = Notif, + +% ?assertEqual(Payload, PayloadRecv), + +% %% GET to read the publish message of the topic +% Reply1 = er_coap_client:request(get, Uri1), +% ?LOGT("Reply=~p", [Reply1]), +% {ok,content, #coap_content{payload = <<"123">>}} = Reply1, + +% er_coap_observer:stop(Pid), +% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, Uri1). + +% t_case02_subscribe(_Config) -> +% Topic = <<"a/b">>, +% TopicStr = binary_to_list(Topic), +% PercentEncodedTopic = emqx_http_lib:uri_encode(TopicStr), +% Payload = <<"payload">>, + +% %% post to publish a new topic "a/b", and the topic is created +% URI = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", +% Reply = er_coap_client:request(post, URI, #coap_content{max_age = 5, format = <<"application/octet-stream">>, payload = Payload}), +% ?LOGT("Reply =~p", [Reply]), +% {ok,created, #coap_content{location_path = LocPath}} = Reply, +% ?assertEqual([<<"/ps/a/b">>] ,LocPath), +% [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), +% ?assertEqual(5, MaxAge), +% ?assertEqual(<<"42">>, CT), + +% %% Wait for the max age of the timer expires +% timer:sleep(6000), +% ?assertEqual(true, emqx_coap_pubsub_topics:is_topic_timeout(Topic)), + +% %% Subscribe to the timeout topic "a/b", still successfully，got {ok, nocontent} Method +% Uri = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", +% Reply1 = {ok, Pid, _N, nocontent, _} = er_coap_observer:observe(Uri), +% ?LOGT("Subscribe Reply=~p", [Reply1]), + +% [SubPid] = emqx:subscribers(Topic), +% ?assert(is_pid(SubPid)), + +% %% put to publish to topic "a/b" +% Reply2 = er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), +% {ok,changed, #coap_content{}} = Reply2, +% [{Topic, MaxAge1, CT, Payload, TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), +% ?assertEqual(60, MaxAge1), +% ?assertEqual(<<"42">>, CT), +% ?assertEqual(false, TimeStamp =:= timeout), + +% %% Publish a message +% emqx:publish(emqx_message:make(Topic, Payload)), + +% Notif = receive_notification(), +% ?LOGT("observer get Notif=~p", [Notif]), +% {coap_notify, _, _, {ok,content}, #coap_content{payload = Payload}} = Notif, + +% er_coap_observer:stop(Pid), +% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). + +% t_case03_subscribe(_Config) -> +% %% Subscribe to the unexisted topic "a/b", got not_found +% Topic = <<"a/b">>, +% TopicStr = binary_to_list(Topic), +% PercentEncodedTopic = emqx_http_lib:uri_encode(TopicStr), +% Uri = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", +% {error, not_found} = er_coap_observer:observe(Uri), + +% [] = emqx:subscribers(Topic). + +% t_case04_subscribe(_Config) -> +% %% Subscribe to the wildcad topic "+/b", got bad_request +% Topic = <<"+/b">>, +% TopicStr = binary_to_list(Topic), +% PercentEncodedTopic = emqx_http_lib:uri_encode(TopicStr), +% Uri = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", +% {error, bad_request} = er_coap_observer:observe(Uri), + +% [] = emqx:subscribers(Topic). + +% t_case01_read(_Config) -> +% Topic = <<"topic1">>, +% TopicStr = binary_to_list(Topic), +% Payload = <<"PubPayload">>, +% timer:sleep(100), + +% %% First post to create a topic "topic1" +% Uri = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", +% Reply = er_coap_client:request(post, Uri, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), +% ?LOGT("Reply =~p", [Reply]), +% {ok,created, #coap_content{location_path = [LocPath]}} = Reply, +% ?assertEqual(<<"/ps/topic1">> ,LocPath), +% TopicInfo = [{Topic, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), +% ?LOGT("lookup topic info=~p", [TopicInfo]), +% ?assertEqual(60, MaxAge1), +% ?assertEqual(<<"42">>, CT1), + +% %% GET to read the publish message of the topic +% timer:sleep(1000), +% Reply1 = er_coap_client:request(get, Uri), +% ?LOGT("Reply=~p", [Reply1]), +% {ok,content, #coap_content{payload = Payload}} = Reply1, + +% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, Uri). + +% t_case02_read(_Config) -> +% Topic = <<"topic1">>, +% TopicStr = binary_to_list(Topic), +% Payload = <<"PubPayload">>, +% timer:sleep(100), + +% %% First post to publish a topic "topic1" +% Uri = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", +% Reply = er_coap_client:request(post, Uri, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), +% ?LOGT("Reply =~p", [Reply]), +% {ok,created, #coap_content{location_path = [LocPath]}} = Reply, +% ?assertEqual(<<"/ps/topic1">> ,LocPath), +% TopicInfo = [{Topic, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), +% ?LOGT("lookup topic info=~p", [TopicInfo]), +% ?assertEqual(60, MaxAge1), +% ?assertEqual(<<"42">>, CT1), + +% %% GET to read the publish message of unmatched format, got bad_request +% Reply1 = er_coap_client:request(get, Uri, #coap_content{format = <<"application/json">>}), +% ?LOGT("Reply=~p", [Reply1]), +% {error, bad_request} = Reply1, + +% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, Uri). + +% t_case03_read(_Config) -> +% Topic = <<"topic1">>, +% TopicStr = binary_to_list(Topic), +% Uri = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", +% timer:sleep(100), + +% %% GET to read the nexisted topic "topic1", got not_found +% Reply = er_coap_client:request(get, Uri), +% ?LOGT("Reply=~p", [Reply]), +% {error, not_found} = Reply. + +% t_case04_read(_Config) -> +% Topic = <<"topic1">>, +% TopicStr = binary_to_list(Topic), +% Payload = <<"PubPayload">>, +% timer:sleep(100), + +% %% First post to publish a topic "topic1" +% Uri = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", +% Reply = er_coap_client:request(post, Uri, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), +% ?LOGT("Reply =~p", [Reply]), +% {ok,created, #coap_content{location_path = [LocPath]}} = Reply, +% ?assertEqual(<<"/ps/topic1">> ,LocPath), +% TopicInfo = [{Topic, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), +% ?LOGT("lookup topic info=~p", [TopicInfo]), +% ?assertEqual(60, MaxAge1), +% ?assertEqual(<<"42">>, CT1), + +% %% GET to read the publish message of wildcard topic, got bad_request +% WildTopic = binary_to_list(<<"+/topic1">>), +% Uri1 = "coap://127.0.0.1/ps/"++WildTopic++"?c=client1&u=tom&p=secret", +% Reply1 = er_coap_client:request(get, Uri1, #coap_content{format = <<"application/json">>}), +% ?LOGT("Reply=~p", [Reply1]), +% {error, bad_request} = Reply1, + +% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, Uri). + +% t_case05_read(_Config) -> +% Topic = <<"a/b">>, +% TopicStr = binary_to_list(Topic), +% PercentEncodedTopic = emqx_http_lib:uri_encode(TopicStr), +% Payload = <<"payload">>, + +% %% post to publish a new topic "a/b", and the topic is created +% URI = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", +% Reply = er_coap_client:request(post, URI, #coap_content{max_age = 5, format = <<"application/octet-stream">>, payload = Payload}), +% ?LOGT("Reply =~p", [Reply]), +% {ok,created, #coap_content{location_path = LocPath}} = Reply, +% ?assertEqual([<<"/ps/a/b">>] ,LocPath), +% [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), +% ?assertEqual(5, MaxAge), +% ?assertEqual(<<"42">>, CT), + +% %% Wait for the max age of the timer expires +% timer:sleep(6000), +% ?assertEqual(true, emqx_coap_pubsub_topics:is_topic_timeout(Topic)), + +% %% GET to read the expired publish message, supposed to get {ok, nocontent}, but now got {ok, content} +% Reply1 = er_coap_client:request(get, URI), +% ?LOGT("Reply=~p", [Reply1]), +% {ok, content, #coap_content{payload = <<>>}}= Reply1, + +% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). + +% t_case01_delete(_Config) -> +% TopicInPayload = <<"a/b">>, +% TopicStr = binary_to_list(TopicInPayload), +% PercentEncodedTopic = emqx_http_lib:uri_encode(TopicStr), +% Payload = list_to_binary("<"++PercentEncodedTopic++">;ct=42"), +% URI = "coap://127.0.0.1/ps/"++"?c=client1&u=tom&p=secret", + +% %% Client post to CREATE topic "a/b" +% Reply = er_coap_client:request(post, URI, #coap_content{format = <<"application/link-format">>, payload = Payload}), +% ?LOGT("Reply =~p", [Reply]), +% {ok,created, #coap_content{location_path = LocPath}} = Reply, +% ?assertEqual([<<"/ps/a/b">>] ,LocPath), + +% %% Client post to CREATE topic "a/b/c" +% TopicInPayload1 = <<"a/b/c">>, +% PercentEncodedTopic1 = emqx_http_lib:uri_encode(binary_to_list(TopicInPayload1)), +% Payload1 = list_to_binary("<"++PercentEncodedTopic1++">;ct=42"), +% Reply1 = er_coap_client:request(post, URI, #coap_content{format = <<"application/link-format">>, payload = Payload1}), +% ?LOGT("Reply =~p", [Reply1]), +% {ok,created, #coap_content{location_path = LocPath1}} = Reply1, +% ?assertEqual([<<"/ps/a/b/c">>] ,LocPath1), + +% timer:sleep(50), + +% %% DELETE the topic "a/b" +% UriD = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", +% ReplyD = er_coap_client:request(delete, UriD), +% ?LOGT("Reply=~p", [ReplyD]), +% {ok, deleted, #coap_content{}}= ReplyD, + +% timer:sleep(300), %% Waiting gen_server:cast/2 for deleting operation +% ?assertEqual(false, emqx_coap_pubsub_topics:is_topic_existed(TopicInPayload)), +% ?assertEqual(false, emqx_coap_pubsub_topics:is_topic_existed(TopicInPayload1)). + +% t_case02_delete(_Config) -> +% TopicInPayload = <<"a/b">>, +% TopicStr = binary_to_list(TopicInPayload), +% PercentEncodedTopic = emqx_http_lib:uri_encode(TopicStr), + +% %% DELETE the unexisted topic "a/b" +% Uri1 = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", +% Reply1 = er_coap_client:request(delete, Uri1), +% ?LOGT("Reply=~p", [Reply1]), +% {error, not_found} = Reply1. + +% t_case13_emit_stats_test(_Config) -> +% ok. + +% %%-------------------------------------------------------------------- +% %% Internal functions + +% receive_notification() -> +% receive +% {coap_notify, Pid, N2, Code2, Content2} -> +% {coap_notify, Pid, N2, Code2, Content2} +% after 2000 -> +% receive_notification_timeout +% end. + +% assert_recv(Topic, Payload) -> +% receive +% {deliver, _, Msg} -> +% ?assertEqual(Topic, Msg#message.topic), +% ?assertEqual(Payload, Msg#message.payload) +% after +% 500 -> +% ?assert(false) +% end. + diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src new file mode 100644 index 000000000..541f31a23 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -0,0 +1,11 @@ +{application, emqx_gateway, + [{description, "The Gateway management application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_gateway_app, []}}, + {applications, [kernel, stdlib, grpc]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/apps/emqx_gateway/src/emqx_gateway.erl b/apps/emqx_gateway/src/emqx_gateway.erl new file mode 100644 index 000000000..f6c12ad53 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway.erl @@ -0,0 +1,84 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway). + +-include("include/emqx_gateway.hrl"). + +%% APIs +-export([ registered_gateway/0 + , create/4 + , remove/1 + , lookup/1 + , update/1 + , start/1 + , stop/1 + , list/0 + ]). + +-spec registered_gateway() -> + [{gateway_type(), emqx_gateway_registry:descriptor()}]. +registered_gateway() -> + emqx_gateway_registry:list(). + +%%-------------------------------------------------------------------- +%% Gateway Instace APIs + +-spec list() -> [instance()]. +list() -> + lists:append(lists:map( + fun({_, Insta}) -> Insta end, + emqx_gateway_sup:list_gateway_insta() + )). + +-spec create(gateway_type(), binary(), binary(), map()) + -> {ok, pid()} + | {error, any()}. +create(Type, Name, Descr, RawConf) -> + Insta = #{ id => clacu_insta_id(Type, Name) + , type => Type + , name => Name + , descr => Descr + , rawconf => RawConf + }, + emqx_gateway_sup:create_gateway_insta(Insta). + +-spec remove(instance_id()) -> ok | {error, any()}. +remove(InstaId) -> + emqx_gateway_sup:remove_gateway_insta(InstaId). + +-spec lookup(instance_id()) -> instance() | undefined. +lookup(InstaId) -> + emqx_gateway_sup:lookup_gateway_insta(InstaId). + +-spec update(instance()) -> ok | {error, any()}. +update(NewInsta) -> + emqx_gateway_sup:update_gateway_insta(NewInsta). + +-spec start(instance_id()) -> ok | {error, any()}. +start(InstaId) -> + emqx_gateway_sup:start_gateway_insta(InstaId). + +-spec stop(instance_id()) -> ok | {error, any()}. +stop(InstaId) -> + emqx_gateway_sup:stop_gateway_insta(InstaId). + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +clacu_insta_id(Type, Name) when is_binary(Name) -> + list_to_atom(lists:concat([Type, "#", binary_to_list(Name)])). diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl new file mode 100644 index 000000000..a25640c9b --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -0,0 +1,91 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway_app). + +-behaviour(application). + +-include_lib("emqx/include/logger.hrl"). + +-logger_header("[Gateway]"). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + {ok, Sup} = emqx_gateway_sup:start_link(), + emqx_gateway_cli:load(), + load_default_gateway_applications(), + create_gateway_by_default(), + {ok, Sup}. + +stop(_State) -> + emqx_gateway_cli:unload(), + ok. + +%%-------------------------------------------------------------------- +%% Internal funcs + +load_default_gateway_applications() -> + Apps = gateway_type_searching(), + ?LOG(info, "Starting the default gateway types: ~p", [Apps]), + lists:foreach(fun load/1, Apps). + +gateway_type_searching() -> + %% FIXME: Hardcoded apps + [emqx_stomp_impl, emqx_sn_impl, emqx_exproto_impl, emqx_coap_impl]. + +load(Mod) -> + try + Mod:load(), + ?LOG(info, "Load ~s gateway application successfully!", [Mod]) + catch + Class : Reason -> + ?LOG(error, "Load ~s gateway application failed: {~p, ~p}", + [Mod, Class, Reason]) + end. + +create_gateway_by_default() -> + create_gateway_by_default(zipped_confs()). + +create_gateway_by_default([]) -> + ok; +create_gateway_by_default([{Type, Name, Confs}|More]) -> + case emqx_gateway_registry:lookup(Type) of + undefined -> + ?LOG(error, "Skip to start ~s#~s: not_registred_type", + [Type, Name]); + _ -> + case emqx_gateway:create(Type, + atom_to_binary(Name, utf8), + <<>>, + Confs) of + {ok, _} -> + ?LOG(debug, "Start ~s#~s successfully!", [Type, Name]); + {error, Reason} -> + ?LOG(error, "Start ~s#~s failed: ~0p", + [Type, Name, Reason]) + end + end, + create_gateway_by_default(More). + +zipped_confs() -> + All = maps:to_list(emqx_config:get([gateway])), + lists:append(lists:foldr( + fun({Type, Gws}, Acc) -> + {Names, Confs} = lists:unzip(maps:to_list(Gws)), + Types = [ Type || _ <- lists:seq(1, length(Names))], + [lists:zip3(Types, Names, Confs) | Acc] + end, [], All)). diff --git a/apps/emqx_gateway/src/emqx_gateway_cli.erl b/apps/emqx_gateway/src/emqx_gateway_cli.erl new file mode 100644 index 000000000..fa2363370 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_cli.erl @@ -0,0 +1,212 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The Command-Line-Interface module for Gateway Application +-module(emqx_gateway_cli). + +-export([ load/0 + , unload/0 + ]). + +-export([ gateway/1 + , 'gateway-registry'/1 + , 'gateway-clients'/1 + , 'gateway-metrics'/1 + %, 'gateway-banned'/1 + ]). + +-spec load() -> ok. +load() -> + Cmds = [Fun || {Fun, _} <- ?MODULE:module_info(exports), is_cmd(Fun)], + lists:foreach(fun(Cmd) -> + emqx_ctl:register_command(Cmd, {?MODULE, Cmd}, []) + end, Cmds). + +-spec unload() -> ok. +unload() -> + Cmds = [Fun || {Fun, _} <- ?MODULE:module_info(exports), is_cmd(Fun)], + lists:foreach(fun(Cmd) -> emqx_ctl:unregister_command(Cmd) end, Cmds). + +is_cmd(Fun) -> + case atom_to_list(Fun) of + "gateway" ++ _ -> true; + _ -> false + end. + +%%-------------------------------------------------------------------- +%% Cmds + +gateway(["list"]) -> + lists:foreach(fun(#{id := InstaId, name := Name, type := Type}) -> + %% FIXME: Get the real running status + emqx_ctl:print("Gateway(~s, name=~s, type=~s, status=running~n", + [InstaId, Name, Type]) + end, emqx_gateway:list()); + +gateway(["lookup", GatewayInstaId]) -> + case emqx_gateway:lookup(atom(GatewayInstaId)) of + undefined -> + emqx_ctl:print("undefined~n"); + Info -> + emqx_ctl:print("~p~n", [Info]) + end; + +gateway(["stop", GatewayInstaId]) -> + case emqx_gateway:stop(atom(GatewayInstaId)) of + ok -> + emqx_ctl:print("ok~n"); + {error, Reason} -> + emqx_ctl:print("Error: ~p~n", [Reason]) + end; + +gateway(["start", GatewayInstaId]) -> + case emqx_gateway:start(atom(GatewayInstaId)) of + ok -> + emqx_ctl:print("ok~n"); + {error, Reason} -> + emqx_ctl:print("Error: ~p~n", [Reason]) + end; + +gateway(_) -> + %% TODO: create/remove APIs + emqx_ctl:usage([ {"gateway list", + "List all created gateway instances"} + , {"gateway lookup ", + "Looup a gateway detailed informations"} + , {"gateway stop ", + "Stop a gateway instance and release all resources"} + , {"gateway start ", + "Start a gateway instance"} + ]). + +'gateway-registry'(["list"]) -> + lists:foreach( + fun({GwType, #{cbkmod := CbMod}}) -> + emqx_ctl:print("Registered Type: ~s, Callback Module: ~s~n", [GwType, CbMod]) + end, + emqx_gateway_registry:list()); + +'gateway-registry'(_) -> + emqx_ctl:usage([ {"gateway-registry list", + "List all registered gateway types"} + ]). + +'gateway-clients'(["list", Type]) -> + InfoTab = emqx_gateway_cm:tabname(info, Type), + dump(InfoTab, client); + +'gateway-clients'(["lookup", Type, ClientId]) -> + ChanTab = emqx_gateway_cm:tabname(chan, Type), + case ets:lookup(ChanTab, bin(ClientId)) of + [] -> emqx_ctl:print("Not Found.~n"); + [Chann] -> + InfoTab = emqx_gateway_cm:tabname(info, Type), + [ChannInfo] = ets:lookup(InfoTab, Chann), + print({client, ChannInfo}) + end; + +'gateway-clients'(["kick", Type, ClientId]) -> + case emqx_gateway_cm:kick_session(Type, bin(ClientId)) of + ok -> emqx_ctl:print("ok~n"); + _ -> emqx_ctl:print("Not Found.~n") + end; + +'gateway-clients'(_) -> + emqx_ctl:usage([ {"gateway-clients list ", + "List all clients for a type of gateway"} + , {"gateway-clients lookup ", + "Lookup the Client Info for specified client"} + , {"gateway-clients kick ", + "Kick out a client"} + ]). + +'gateway-metrics'([GatewayType]) -> + Tab = emqx_gateway_metrics:tabname(GatewayType), + case ets:info(Tab) of + undefined -> + emqx_ctl:print("Bad Gateway Type.~n"); + _ -> + lists:foreach( + fun({K, V}) -> + emqx_ctl:print("~-30s: ~w~n", [K, V]) + end, lists:sort(ets:tab2list(Tab))) + end; + +'gateway-metrics'(_) -> + emqx_ctl:usage([ {"gateway-metrics ", + "List all metrics for a type of gateway"} + ]). + +atom(Id) -> + try + list_to_existing_atom(Id) + catch + _ : _ -> undefined + end. + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +bin(S) -> iolist_to_binary(S). + +dump(Table, Tag) -> + dump(Table, Tag, ets:first(Table), []). + +dump(_Table, _, '$end_of_table', Result) -> + lists:reverse(Result); + +dump(Table, Tag, Key, Result) -> + PrintValue = [print({Tag, Record}) || Record <- ets:lookup(Table, Key)], + dump(Table, Tag, ets:next(Table, Key), [PrintValue | Result]). + +print({client, {_, Infos, Stats}}) -> + ClientInfo = maps:get(clientinfo, Infos, #{}), + ConnInfo = maps:get(conninfo, Infos, #{}), + _Session = maps:get(session, Infos, #{}), + SafeGet = fun(K, M) -> maps:get(K, M, undefined) end, + StatsGet = fun(K) -> proplists:get_value(K, Stats, 0) end, + + ConnectedAt = SafeGet(connected_at, ConnInfo), + InfoKeys = [clientid, username, peername, clean_start, keepalive, + subscriptions_cnt, send_msg, connected, created_at, connected_at], + Info = #{ clientid => SafeGet(clientid, ClientInfo), + username => SafeGet(username, ClientInfo), + peername => SafeGet(peername, ConnInfo), + clean_start => SafeGet(clean_start, ConnInfo), + keepalive => SafeGet(keepalive, ConnInfo), + subscriptions_cnt => StatsGet(subscriptions_cnt), + send_msg => StatsGet(send_msg), + connected => SafeGet(conn_state, ClientInfo) == connected, + created_at => ConnectedAt, + connected_at => ConnectedAt + }, + + emqx_ctl:print("Client(~s, username=~s, peername=~s, " + "clean_start=~s, keepalive=~w, " + "subscriptions=~w, delivered_msgs=~w, " + "connected=~s, created_at=~w, connected_at=~w)~n", + [format(K, maps:get(K, Info)) || K <- InfoKeys]). + +format(_, undefined) -> + undefined; + +format(peername, {IPAddr, Port}) -> + IPStr = emqx_mgmt_util:ntoa(IPAddr), + io_lib:format("~s:~p", [IPStr, Port]); + +format(_, Val) -> + Val. diff --git a/apps/emqx_gateway/src/emqx_gateway_cm.erl b/apps/emqx_gateway/src/emqx_gateway_cm.erl new file mode 100644 index 000000000..e4d21e1a5 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_cm.erl @@ -0,0 +1,468 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The Gateway Connection-Manager +%% +%% For a certain type of protocol, this is a single instance of the manager. +%% It means that no matter how many instances of the stomp gateway are created, +%% they all share a single this Connection-Manager +-module(emqx_gateway_cm). + +-behaviour(gen_server). + +-include("include/emqx_gateway.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-logger_header("[PGW-CM]"). + +%% APIs +-export([start_link/1]). + +-export([ open_session/5 + , kick_session/2 + , kick_session/3 + , register_channel/4 + , unregister_channel/2 + , insert_channel_info/4 + , set_chan_info/3 + , set_chan_info/4 + , get_chan_info/2 + , get_chan_info/3 + , set_chan_stats/3 + , set_chan_stats/4 + , get_chan_stats/2 + , get_chan_stats/3 + , connection_closed/2 + ]). + +%% Internal funcs for getting tabname by GatewayId +-export([cmtabs/1, tabname/2]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-record(state, { + type :: atom(), %% Gateway Type + locker :: pid(), %% ClientId Locker for CM + registry :: pid(), %% ClientId Registry server + chan_pmon :: emqx_pmon:pmon() + }). + +-type option() :: {type, gateway_type()}. +-type options() :: list(option()). + +-define(T_TAKEOVER, 15000). +-define(DEFAULT_BATCH_SIZE, 10000). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +-spec start_link(options()) -> {ok, pid()} | ignore | {error, any()}. +start_link(Options) -> + Type = proplists:get_value(type, Options), + gen_server:start_link({local, procname(Type)}, ?MODULE, Options, []). + +procname(Type) -> + list_to_atom(lists:concat([emqx_gateway_, Type, '_cm'])). + +-spec cmtabs(Type :: atom()) -> {ChanTab :: atom(), + ConnTab :: atom(), + ChannInfoTab :: atom()}. +cmtabs(Type) -> + { tabname(chan, Type) %% Client Tabname; Record: {ClientId, Pid} + , tabname(conn, Type) %% Client ConnMod; Recrod: {{ClientId, Pid}, ConnMod} + , tabname(info, Type) %% ClientInfo Tabname; Record: {{ClientId, Pid}, ClientInfo, ClientStats} + }. + +tabname(chan, Type) -> + list_to_atom(lists:concat([emqx_gateway_, Type, '_channel'])); +tabname(conn, Type) -> + list_to_atom(lists:concat([emqx_gateway_, Type, '_channel_conn'])); +tabname(info, Type) -> + list_to_atom(lists:concat([emqx_gateway_, Type, '_channel_info'])). + +lockername(Type) -> + list_to_atom(lists:concat([emqx_gateway_, Type, '_locker'])). + +-spec register_channel(atom(), binary(), pid(), emqx_types:conninfo()) -> ok. +register_channel(Type, ClientId, ChanPid, #{conn_mod := ConnMod}) when is_pid(ChanPid) -> + Chan = {ClientId, ChanPid}, + true = ets:insert(tabname(chan, Type), Chan), + true = ets:insert(tabname(conn, Type), {Chan, ConnMod}), + ok = emqx_gateway_cm_registry:register_channel(Type, Chan), + cast(procname(Type), {registered, Chan}). + +%% @doc Unregister a channel. +-spec unregister_channel(atom(), emqx_types:clientid()) -> ok. +unregister_channel(Type, ClientId) when is_binary(ClientId) -> + true = do_unregister_channel(Type, {ClientId, self()}, cmtabs(Type)), + ok. + +%% @doc Insert/Update the channel info and stats +-spec insert_channel_info(atom(), + emqx_types:clientid(), + emqx_types:infos(), + emqx_types:stats()) -> ok. +insert_channel_info(Type, ClientId, Info, Stats) -> + Chan = {ClientId, self()}, + true = ets:insert(tabname(info, Type), {Chan, Info, Stats}), + %%?tp(debug, insert_channel_info, #{client_id => ClientId}), + ok. + +%% @doc Get info of a channel. +-spec get_chan_info(gateway_type(), emqx_types:clientid()) + -> emqx_types:infos() | undefined. +get_chan_info(Type, ClientId) -> + with_channel(Type, ClientId, + fun(ChanPid) -> + get_chan_info(Type, ClientId, ChanPid) + end). + +-spec get_chan_info(gateway_type(), emqx_types:clientid(), pid()) + -> emqx_types:infos() | undefined. +get_chan_info(Type, ClientId, ChanPid) when node(ChanPid) == node() -> + Chan = {ClientId, ChanPid}, + try ets:lookup_element(tabname(info, Type), Chan, 2) + catch + error:badarg -> undefined + end; +get_chan_info(Type, ClientId, ChanPid) -> + rpc_call(node(ChanPid), get_chan_info, [Type, ClientId, ChanPid]). + +%% @doc Update infos of the channel. +-spec set_chan_info(gateway_type(), + emqx_types:clientid(), + emqx_types:infos()) -> boolean(). +set_chan_info(Type, ClientId, Infos) -> + set_chan_info(Type, ClientId, self(), Infos). + +-spec set_chan_info(gateway_type(), + emqx_types:clientid(), + pid(), + emqx_types:infos()) -> boolean(). +set_chan_info(Type, ClientId, ChanPid, Infos) when node(ChanPid) == node() -> + Chan = {ClientId, ChanPid}, + try ets:update_element(tabname(info, Type), Chan, {2, Infos}) + catch + error:badarg -> false + end; +set_chan_info(Type, ClientId, ChanPid, Infos) -> + rpc_call(node(ChanPid), set_chan_info, [Type, ClientId, ChanPid, Infos]). + +%% @doc Get channel's stats. +-spec get_chan_stats(gateway_type(), emqx_types:clientid()) + -> emqx_types:stats() | undefined. +get_chan_stats(Type, ClientId) -> + with_channel(Type, ClientId, + fun(ChanPid) -> + get_chan_stats(Type, ClientId, ChanPid) + end). + +-spec get_chan_stats(gateway_type(), emqx_types:clientid(), pid()) + -> emqx_types:stats() | undefined. +get_chan_stats(Type, ClientId, ChanPid) when node(ChanPid) == node() -> + Chan = {ClientId, ChanPid}, + try ets:lookup_element(tabname(info, Type), Chan, 3) + catch + error:badarg -> undefined + end; +get_chan_stats(Type, ClientId, ChanPid) -> + rpc_call(node(ChanPid), get_chan_stats, [Type, ClientId, ChanPid]). + +-spec set_chan_stats(gateway_type(), + emqx_types:clientid(), + emqx_types:stats()) -> boolean(). +set_chan_stats(Type, ClientId, Stats) -> + set_chan_stats(Type, ClientId, self(), Stats). + +-spec set_chan_stats(gateway_type(), + emqx_types:clientid(), + pid(), + emqx_types:stats()) -> boolean(). +set_chan_stats(Type, ClientId, ChanPid, Stats) when node(ChanPid) == node() -> + Chan = {ClientId, self()}, + try ets:update_element(tabname(info, Type), Chan, {3, Stats}) + catch + error:badarg -> false + end; +set_chan_stats(Type, ClientId, ChanPid, Stats) -> + rpc_call(node(ChanPid), set_chan_stats, [Type, ClientId, ChanPid, Stats]). + +-spec connection_closed(gateway_type(), emqx_types:clientid()) -> true. +connection_closed(Type, ClientId) -> + %% XXX: Why we need to delete conn_mod tab ??? + Chan = {ClientId, self()}, + ets:delete_object(tabname(conn, Type), Chan). + +-spec open_session(Type :: atom(), CleanStart :: boolean(), + ClientInfo :: emqx_types:clientinfo(), + ConnInfo :: emqx_types:conninfo(), + CreateSessionFun :: fun((emqx_types:clientinfo(), + emqx_types:conninfo()) -> Session + )) + -> {ok, #{session := Session, + present := boolean(), + pendings => list() + }} + | {error, any()}. + +open_session(Type, true = _CleanStart, ClientInfo, ConnInfo, CreateSessionFun) -> + Self = self(), + ClientId = maps:get(clientid, ClientInfo), + Fun = fun(_) -> + ok = discard_session(Type, ClientId), + Session = create_session(Type, + ClientInfo, + ConnInfo, + CreateSessionFun + ), + register_channel(Type, ClientId, Self, ConnInfo), + {ok, #{session => Session, present => false}} + end, + locker_trans(Type, ClientId, Fun); + +open_session(_Type, false = _CleanStart, + _ClientInfo, _ConnInfo, _CreateSessionFun) -> + %% TODO: + {error, not_supported_now}. + +%% @private +create_session(Type, ClientInfo, ConnInfo, CreateSessionFun) -> + try + Session = emqx_gateway_utils:apply( + CreateSessionFun, + [ClientInfo, ConnInfo] + ), + ok = emqx_gateway_metrics:inc(Type, 'session.created'), + SessionInfo = case is_tuple(Session) + andalso element(1, Session) == session of + true -> emqx_session:info(Session); + _ -> + case is_map(Session) of + false -> + throw(session_structure_should_be_map); + _ -> + Session + end + end, + ok = emqx_hooks:run('session.created', [ClientInfo, SessionInfo]), + Session + catch + Class : Reason : Stk -> + ?LOG(error, "Failed to create a session: ~p, ~p " + "Stacktrace:~0p", [Class, Reason, Stk]), + throw(Reason) + end. + +%% @doc Discard all the sessions identified by the ClientId. +-spec discard_session(Type :: atom(), binary()) -> ok. +discard_session(Type, ClientId) when is_binary(ClientId) -> + case lookup_channels(Type, ClientId) of + [] -> ok; + ChanPids -> lists:foreach(fun(Pid) -> do_discard_session(Type, ClientId, Pid) end, ChanPids) + end. + +%% @private +do_discard_session(Type, ClientId, Pid) -> + try + discard_session(Type, ClientId, Pid) + catch + _ : noproc -> % emqx_ws_connection: call + %?tp(debug, "session_already_gone", #{pid => Pid}), + ok; + _ : {noproc, _} -> % emqx_connection: gen_server:call + %?tp(debug, "session_already_gone", #{pid => Pid}), + ok; + _ : {{shutdown, _}, _} -> + %?tp(debug, "session_already_shutdown", #{pid => Pid}), + ok; + _ : _Error : _St -> + %?tp(error, "failed_to_discard_session", + % #{pid => Pid, reason => Error, stacktrace=>St}) + ok + end. + +%% @private +discard_session(Type, ClientId, ChanPid) when node(ChanPid) == node() -> + case get_chann_conn_mod(Type, ClientId, ChanPid) of + undefined -> ok; + ConnMod when is_atom(ConnMod) -> + ConnMod:call(ChanPid, discard, ?T_TAKEOVER) + end; + +%% @private +discard_session(Type, ClientId, ChanPid) -> + rpc_call(node(ChanPid), discard_session, [Type, ClientId, ChanPid]). + +-spec kick_session(gateway_type(), emqx_types:clientid()) + -> {error, any()} + | ok. +kick_session(Type, ClientId) -> + case lookup_channels(Type, ClientId) of + [] -> {error, not_found}; + [ChanPid] -> + kick_session(Type, ClientId, ChanPid); + ChanPids -> + [ChanPid|StalePids] = lists:reverse(ChanPids), + ?LOG(error, "More than one channel found: ~p", [ChanPids]), + lists:foreach(fun(StalePid) -> + catch discard_session(Type, ClientId, StalePid) + end, StalePids), + kick_session(Type, ClientId, ChanPid) + end. + +kick_session(Type, ClientId, ChanPid) when node(ChanPid) == node() -> + case get_chan_info(Type, ClientId, ChanPid) of + #{conninfo := #{conn_mod := ConnMod}} -> + ConnMod:call(ChanPid, kick, ?T_TAKEOVER); + undefined -> + {error, not_found} + end; + +kick_session(Type, ClientId, ChanPid) -> + rpc_call(node(ChanPid), kick_session, [Type, ClientId, ChanPid]). + +with_channel(Type, ClientId, Fun) -> + case lookup_channels(Type, ClientId) of + [] -> undefined; + [Pid] -> Fun(Pid); + Pids -> Fun(lists:last(Pids)) + end. + +%% @doc Lookup channels. +-spec(lookup_channels(atom(), emqx_types:clientid()) -> list(pid())). +lookup_channels(Type, ClientId) -> + emqx_gateway_cm_registry:lookup_channels(Type, ClientId). + +get_chann_conn_mod(Type, ClientId, ChanPid) when node(ChanPid) == node() -> + Chan = {ClientId, ChanPid}, + try [ConnMod] = ets:lookup_element(tabname(conn, Type), Chan, 2), ConnMod + catch + error:badarg -> undefined + end; +get_chann_conn_mod(Type, ClientId, ChanPid) -> + rpc_call(node(ChanPid), get_chann_conn_mod, [Type, ClientId, ChanPid]). + +%% Locker + +locker_trans(_Type, undefined, Fun) -> + Fun([]); +locker_trans(Type, ClientId, Fun) -> + Locker = lockername(Type), + case locker_lock(Locker, ClientId) of + {true, Nodes} -> + try Fun(Nodes) after locker_unlock(Locker, ClientId) end; + {false, _Nodes} -> + {error, client_id_unavailable} + end. + +locker_lock(Locker, ClientId) -> + ekka_locker:acquire(Locker, ClientId, quorum). + +locker_unlock(Locker, ClientId) -> + ekka_locker:release(Locker, ClientId, quorum). + +%% @private +rpc_call(Node, Fun, Args) -> + case rpc:call(Node, ?MODULE, Fun, Args) of + {badrpc, Reason} -> error(Reason); + Res -> Res + end. + +cast(Name, Msg) -> + gen_server:cast(Name, Msg). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init(Options) -> + Type = proplists:get_value(type, Options), + + TabOpts = [public, {write_concurrency, true}], + + {ChanTab, ConnTab, InfoTab} = cmtabs(Type), + ok = emqx_tables:new(ChanTab, [bag, {read_concurrency, true}|TabOpts]), + ok = emqx_tables:new(ConnTab, [bag | TabOpts]), + ok = emqx_tables:new(InfoTab, [set, compressed | TabOpts]), + + %% Start link cm-registry process + %% XXX: Should I hang it under a higher level supervisor? + {ok, Registry} = emqx_gateway_cm_registry:start_link(Type), + + %% Start locker process + {ok, Locker} = ekka_locker:start_link(lockername(Type)), + + %% Interval update stats + %% TODO: v0.2 + %ok = emqx_stats:update_interval(chan_stats, fun ?MODULE:stats_fun/0), + + {ok, #state{type = Type, + locker = Locker, + registry = Registry, + chan_pmon = emqx_pmon:new()}}. + +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +handle_cast({registered, {ClientId, ChanPid}}, State = #state{chan_pmon = PMon}) -> + PMon1 = emqx_pmon:monitor(ChanPid, ClientId, PMon), + {noreply, State#state{chan_pmon = PMon1}}; + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info({'DOWN', _MRef, process, Pid, _Reason}, + State = #state{type = Type, chan_pmon = PMon}) -> + ChanPids = [Pid | emqx_misc:drain_down(?DEFAULT_BATCH_SIZE)], + {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), + + CmTabs = cmtabs(Type), + ok = emqx_pool:async_submit(fun do_unregister_channel_task/3, [Items, Type, CmTabs]), + {noreply, State#state{chan_pmon = PMon1}}; + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +do_unregister_channel_task(Items, Type, CmTabs) -> + lists:foreach( + fun({ChanPid, ClientId}) -> + do_unregister_channel(Type, {ClientId, ChanPid}, CmTabs) + end, Items). + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +do_unregister_channel(Type, Chan, {ChanTab, ConnTab, InfoTab}) -> + ok = emqx_gateway_cm_registry:unregister_channel(Type, Chan), + true = ets:delete(ConnTab, Chan), + true = ets:delete(InfoTab, Chan), + ets:delete_object(ChanTab, Chan). diff --git a/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl b/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl new file mode 100644 index 000000000..4275fdf3e --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl @@ -0,0 +1,141 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The gateway connection registry +-module(emqx_gateway_cm_registry). + +-behaviour(gen_server). + +-logger_header("[PGW-CM-Registy]"). + +-export([start_link/1]). + +%% XXX: needless +%-export([is_enabled/0]). + +-export([ register_channel/2 + , unregister_channel/2 + ]). + +-export([lookup_channels/2]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-define(LOCK, {?MODULE, cleanup_down}). + +-record(channel, {chid, pid}). + +%% @doc Start the global channel registry. +-spec(start_link(atom()) -> gen_server:startlink_ret()). +start_link(Type) -> + gen_server:start_link(?MODULE, [Type], []). + +-spec tabname(atom()) -> atom(). +tabname(Type) -> + list_to_atom(lists:concat([emqx_gateway_, Type, '_channel_registry'])). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +%% @doc Register a global channel. +-spec register_channel(atom(), binary() | {binary(), pid()}) -> ok. +register_channel(Type, ClientId) when is_binary(ClientId) -> + register_channel(Type, {ClientId, self()}); + +register_channel(Type, {ClientId, ChanPid}) when is_binary(ClientId), is_pid(ChanPid) -> + ekka_mnesia:dirty_write(tabname(Type), record(ClientId, ChanPid)). + +%% @doc Unregister a global channel. +-spec unregister_channel(atom(), binary() | {binary(), pid()}) -> ok. +unregister_channel(Type, ClientId) when is_binary(ClientId) -> + unregister_channel(Type, {ClientId, self()}); + +unregister_channel(Type, {ClientId, ChanPid}) when is_binary(ClientId), is_pid(ChanPid) -> + ekka_mnesia:dirty_delete_object(tabname(Type), record(ClientId, ChanPid)). + +%% @doc Lookup the global channels. +-spec lookup_channels(atom(), binary()) -> list(pid()). +lookup_channels(Type, ClientId) -> + [ChanPid || #channel{pid = ChanPid} <- mnesia:dirty_read(tabname(Type), ClientId)]. + +record(ClientId, ChanPid) -> + #channel{chid = ClientId, pid = ChanPid}. + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([Type]) -> + Tab = tabname(Type), + ok = ekka_mnesia:create_table(Tab, [ + {type, bag}, + {ram_copies, [node()]}, + {record_name, channel}, + {attributes, record_info(fields, channel)}, + {storage_properties, [{ets, [{read_concurrency, true}, + {write_concurrency, true}]}]}]), + ok = ekka_mnesia:copy_table(Tab, ram_copies), + %%ok = ekka_rlog:wait_for_shards([?CM_SHARD], infinity), + ok = ekka:monitor(membership), + {ok, #{type => Type}}. + +handle_call(Req, _From, State) -> + logger:error("Unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + logger:error("Unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({membership, {mnesia, down, Node}}, State = #{type := Type}) -> + Tab = tabname(Type), + global:trans({?LOCK, self()}, + fun() -> + %% FIXME: The shard name should be fixed later + ekka_mnesia:transaction(?MODULE, fun cleanup_channels/2, [Node, Tab]) + end), + {noreply, State}; + +handle_info({membership, _Event}, State) -> + {noreply, State}; + +handle_info(Info, State) -> + logger:error("Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +cleanup_channels(Node, Tab) -> + Pat = [{#channel{pid = '$1', _ = '_'}, [{'==', {node, '$1'}, Node}], ['$_']}], + lists:foreach(fun(Chan) -> + mnesia:delete_object(Tab, Chan, write) + end, mnesia:select(Tab, Pat, write)). diff --git a/apps/emqx_gateway/src/emqx_gateway_ctx.erl b/apps/emqx_gateway/src/emqx_gateway_ctx.erl new file mode 100644 index 000000000..722d4e549 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_ctx.erl @@ -0,0 +1,152 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The gateway instance context +-module(emqx_gateway_ctx). + +-include("include/emqx_gateway.hrl"). + +-logger_header(["PGW-Ctx"]). + +%% @doc The running context for a Connection/Channel process. +%% +%% The `Context` encapsulates a complex structure of contextual information. +%% It is convenient to use it directly in Channel/Connection to read +%% configuration, register devices and other common operations. +%% +-type context() :: + #{ %% Gateway Instance ID + instid := instance_id() + %% Gateway ID + , type := gateway_type() + %% Autenticator + , auth := emqx_authn:chain_id() + %% The ConnectionManager PID + , cm := pid() + }. + +%% Authentication circle +-export([ authenticate/2 + , open_session/5 + , insert_channel_info/4 + , set_chan_info/3 + , set_chan_stats/3 + , connection_closed/2 + ]). + +%% Message circle +-export([ authorize/4 + % Needless for pub/sub + %, publish/3 + %, subscribe/4 + ]). + +%% Metrics & Stats +-export([ metrics_inc/2 + , metrics_inc/3 + ]). + +%%-------------------------------------------------------------------- +%% Authentication circle + +%% @doc Authenticate whether the client has access to the Broker. +-spec authenticate(context(), emqx_types:clientinfo()) + -> {ok, emqx_types:clientinfo()} + | {error, any()}. +authenticate(_Ctx = #{auth := ChainId}, ClientInfo0) -> + ClientInfo = ClientInfo0#{ + zone => default, + listener => mqtt_tcp, + chain_id => ChainId + }, + case emqx_access_control:authenticate(ClientInfo) of + ok -> + {ok, mountpoint(ClientInfo)}; + {error, Reason} -> + {error, Reason} + end; +authenticate(_Ctx, ClientInfo) -> + {ok, ClientInfo}. + +%% @doc Register the session to the cluster. +%% +%% This function should be called after the client has authenticated +%% successfully so that the client can be managed in the cluster. +-spec open_session(context(), boolean(), emqx_types:clientinfo(), + emqx_types:conninfo(), + fun((emqx_types:clientinfo(), + emqx_types:conninfo()) -> Session) + ) + -> {ok, #{session := Session, + present := boolean(), + pendings => list() + }} + | {error, any()}. +open_session(Ctx, false, ClientInfo, ConnInfo, CreateSessionFun) -> + logger:warning("clean_start=false is not supported now, " + "fallback to clean_start mode"), + open_session(Ctx, true, ClientInfo, ConnInfo, CreateSessionFun); + +open_session(_Ctx = #{type := Type}, + CleanStart, ClientInfo, ConnInfo, CreateSessionFun) -> + emqx_gateway_cm:open_session(Type, CleanStart, + ClientInfo, ConnInfo, CreateSessionFun). + +-spec insert_channel_info(context(), + emqx_types:clientid(), + emqx_types:infos(), + emqx_types:stats()) -> ok. +insert_channel_info(_Ctx = #{type := Type}, ClientId, Infos, Stats) -> + emqx_gateway_cm:insert_channel_info(Type, ClientId, Infos, Stats). + +%% @doc Set the Channel Info to the ConnectionManager for this client +-spec set_chan_info(context(), + emqx_types:clientid(), + emqx_types:infos()) -> boolean(). +set_chan_info(_Ctx = #{type := Type}, ClientId, Infos) -> + emqx_gateway_cm:set_chan_info(Type, ClientId, Infos). + +-spec set_chan_stats(context(), + emqx_types:clientid(), + emqx_types:stats()) -> boolean(). +set_chan_stats(_Ctx = #{type := Type}, ClientId, Stats) -> + emqx_gateway_cm:set_chan_stats(Type, ClientId, Stats). + +-spec connection_closed(context(), emqx_types:clientid()) -> boolean(). +connection_closed(_Ctx = #{type := Type}, ClientId) -> + emqx_gateway_cm:connection_closed(Type, ClientId). + +-spec authorize(context(), emqx_types:clientinfo(), + emqx_types:pubsub(), emqx_types:topic()) + -> allow | deny. +authorize(_Ctx, ClientInfo, PubSub, Topic) -> + emqx_access_control:authorize(ClientInfo, PubSub, Topic). + +metrics_inc(_Ctx = #{type := Type}, Name) -> + emqx_gateway_metrics:inc(Type, Name). + +metrics_inc(_Ctx = #{type := Type}, Name, Oct) -> + emqx_gateway_metrics:inc(Type, Name, Oct). + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +mountpoint(ClientInfo = #{mountpoint := undefined}) -> + ClientInfo; +mountpoint(ClientInfo = #{mountpoint := MountPoint}) -> + MountPoint1 = emqx_mountpoint:replvar(MountPoint, ClientInfo), + ClientInfo#{mountpoint := MountPoint1}. diff --git a/apps/emqx_gateway/src/emqx_gateway_gw_sup.erl b/apps/emqx_gateway/src/emqx_gateway_gw_sup.erl new file mode 100644 index 000000000..21ad30c0d --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_gw_sup.erl @@ -0,0 +1,135 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The Gateway Top supervisor. +-module(emqx_gateway_gw_sup). + +-behaviour(supervisor). + +-include("include/emqx_gateway.hrl"). + +-export([start_link/1]). + +-export([ create_insta/3 + , remove_insta/2 + , update_insta/2 + , start_insta/2 + , stop_insta/2 + , list_insta/1 + ]). + +%% Supervisor callbacks +-export([init/1]). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +start_link(Type) -> + supervisor:start_link({local, Type}, ?MODULE, [Type]). + +-spec create_insta(pid(), instance(), map()) -> {ok, GwInstaPid :: pid()} | {error, any()}. +create_insta(Sup, Insta = #{id := InstaId}, GwDscrptr) -> + case emqx_gateway_utils:find_sup_child(Sup, InstaId) of + {ok, _GwInstaPid} -> {error, alredy_existed}; + false -> + %% XXX: More instances options to it? + %% + Ctx = ctx(Sup, InstaId), + %% + ChildSpec = emqx_gateway_utils:childspec( + InstaId, + worker, + emqx_gateway_insta_sup, + [Insta, Ctx, GwDscrptr] + ), + emqx_gateway_utils:supervisor_ret( + supervisor:start_child(Sup, ChildSpec) + ) + end. + +-spec remove_insta(pid(), InstaId :: atom()) -> ok | {error, any()}. +remove_insta(Sup, InstaId) -> + case emqx_gateway_utils:find_sup_child(Sup, InstaId) of + false -> ok; + {ok, _GwInstaPid} -> + ok = supervisor:terminate_child(Sup, InstaId), + ok = supervisor:delete_child(Sup, InstaId) + end. + +-spec update_insta(pid(), NewInsta :: instance()) -> ok | {error, any()}. +update_insta(Sup, NewInsta = #{id := InstaId}) -> + case emqx_gateway_utils:find_sup_child(Sup, InstaId) of + false -> {error, not_found}; + {ok, GwInstaPid} -> + emqx_gateway_insta_sup:update(GwInstaPid, NewInsta) + end. + +-spec start_insta(pid(), atom()) -> ok | {error, any()}. +start_insta(Sup, InstaId) -> + case emqx_gateway_utils:find_sup_child(Sup, InstaId) of + false -> {error, not_found}; + {ok, GwInstaPid} -> + emqx_gateway_insta_sup:enable(GwInstaPid) + end. + +-spec stop_insta(pid(), atom()) -> ok | {error, any()}. +stop_insta(Sup, InstaId) -> + case emqx_gateway_utils:find_sup_child(Sup, InstaId) of + false -> {error, not_found}; + {ok, GwInstaPid} -> + emqx_gateway_insta_sup:disable(GwInstaPid) + end. + +-spec list_insta(pid()) -> [instance()]. +list_insta(Sup) -> + lists:filtermap( + fun({InstaId, GwInstaPid, _Type, _Mods}) -> + is_gateway_insta_id(InstaId) + andalso {true, emqx_gateway_insta_sup:info(GwInstaPid)} + end, supervisor:which_children(Sup)). + +%% Supervisor callback + +%% @doc Initialize Top Supervisor for a Protocol +init([Type]) -> + SupFlags = #{ strategy => one_for_one + , intensity => 10 + , period => 60 + }, + CmOpts = [{type, Type}], + CM = emqx_gateway_utils:childspec(worker, emqx_gateway_cm, [CmOpts]), + Metrics = emqx_gateway_utils:childspec(worker, emqx_gateway_metrics, [Type]), + {ok, {SupFlags, [CM, Metrics]}}. + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +ctx(Sup, InstaId) -> + {_, Type} = erlang:process_info(Sup, registered_name), + {ok, CM} = emqx_gateway_utils:find_sup_child(Sup, emqx_gateway_cm), + #{ instid => InstaId + , type => Type + , cm => CM + }. + +is_gateway_insta_id(emqx_gateway_cm) -> + false; +is_gateway_insta_id(emqx_gateway_metrics) -> + false; +is_gateway_insta_id(_Id) -> + true. diff --git a/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl b/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl new file mode 100644 index 000000000..7994a6cea --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl @@ -0,0 +1,311 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The gateway instance management +-module(emqx_gateway_insta_sup). + +-behaviour(gen_server). + +-include("include/emqx_gateway.hrl"). + +-logger_header("[PGW-Insta-Sup]"). + +%% APIs +-export([ start_link/3 + , info/1 + , disable/1 + , enable/1 + , update/2 + ]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-record(state, { + insta :: instance(), + ctx :: emqx_gateway_ctx:context(), + status :: stopped | running, + child_pids :: [pid()], + insta_state :: emqx_gateway_impl:state() | undefined + }). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +start_link(Insta, Ctx, GwDscrptr) -> + gen_server:start_link( + ?MODULE, + [Insta, Ctx, GwDscrptr], + [] + ). + +-spec info(pid()) -> instance(). +info(Pid) -> + gen_server:call(Pid, info). + +%% @doc Stop instance +-spec disable(pid()) -> ok | {error, any()}. +disable(Pid) -> + call(Pid, disable). + +%% @doc Start instance +-spec enable(pid()) -> ok | {error, any()}. +enable(Pid) -> + call(Pid, enable). + +%% @doc Update the gateway instance configurations +-spec update(pid(), instance()) -> ok | {error, any()}. +update(Pid, NewInsta) -> + call(Pid, {update, NewInsta}). + +call(Pid, Req) -> + gen_server:call(Pid, Req, 5000). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([Insta, Ctx0, _GwDscrptr]) -> + process_flag(trap_exit, true), + #{rawconf := RawConf} = Insta, + Ctx = do_init_context(RawConf, Ctx0), + State = #state{ + insta = Insta, + ctx = Ctx, + child_pids = [], + status = stopped + }, + case cb_insta_create(State) of + {error, _Reason} -> + do_deinit_context(Ctx), + %% XXX: Return Reason?? + {stop, create_gateway_instance_failed}; + {ok, NState} -> + {ok, NState} + end. + +do_init_context(RawConf, Ctx) -> + Auth = case maps:get(authenticator, RawConf, allow_anonymous) of + allow_anonymous -> allow_anonymous; + Funcs when is_list(Funcs) -> + create_authenticator_for_gateway_insta(Funcs) + end, + Ctx#{auth => Auth}. + +do_deinit_context(Ctx) -> + cleanup_authenticator_for_gateway_insta(maps:get(auth, Ctx)), + ok. + +handle_call(info, _From, State = #state{insta = Insta}) -> + {reply, Insta, State}; + +handle_call(disable, _From, State = #state{status = Status}) -> + case Status of + running -> + case cb_insta_destroy(State) of + {ok, NState} -> + {reply, ok, NState}; + {error, Reason} -> + {reply, {error, Reason}, State} + end; + _ -> + {reply, {error, already_stopped}, State} + end; + +handle_call(enable, _From, State = #state{status = Status}) -> + case Status of + stopped -> + case cb_insta_create(State) of + {error, Reason} -> + {reply, {error, Reason}, State}; + {ok, NState} -> + {reply, ok, NState} + end; + _ -> + {reply, {error, already_started}, State} + end; + +%% Stopped -> update +handle_call({update, NewInsta}, _From, State = #state{insta = Insta, + status = stopped}) -> + case maps:get(id, NewInsta, undefined) == maps:get(id, Insta, undefined) of + true -> + {reply, ok, State#state{insta = NewInsta}}; + false -> + {reply, {error, bad_instan_id}, State} + end; + +%% Running -> update +handle_call({update, NewInsta}, _From, State = #state{insta = Insta, + status = running}) -> + case maps:get(id, NewInsta, undefined) == maps:get(id, Insta, undefined) of + true -> + case cb_insta_update(NewInsta, State) of + {ok, NState} -> + {reply, ok, NState}; + {error, Reason} -> + {reply, {error, Reason}, State} + end; + false -> + {reply, {error, bad_instan_id}, State} + end; + +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info({'EXIT', Pid, Reason}, State = #state{child_pids = Pids}) -> + case lists:member(Pid, Pids) of + true -> + logger:error("Child process ~p exited: ~0p.", [Pid, Reason]), + case Pids -- [Pid]of + [] -> + logger:error("All child process exited!"), + {noreply, State#state{status = stopped, + child_pids = [], + insta_state = undefined}}; + RemainPids -> + {noreply, State#state{child_pids = RemainPids}} + end; + _ -> + logger:error("Unknown process exited ~p:~0p", [Pid, Reason]), + {noreply, State} + end; + +handle_info(Info, State) -> + logger:warning("Unexcepted info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, State = #state{ctx = Ctx, child_pids = Pids}) -> + %% Cleanup instances + %% Step1. Destory instance + Pids /= [] andalso (_ = cb_insta_destroy(State)), + %% Step2. Delete authenticator resources + _ = do_deinit_context(Ctx), + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +create_authenticator_for_gateway_insta(_Funcs) -> + todo. + +cleanup_authenticator_for_gateway_insta(allow_anonymouse) -> + ok; +cleanup_authenticator_for_gateway_insta(_ChainId) -> + todo. + +cb_insta_destroy(State = #state{insta = Insta = #{type := Type}, + insta_state = InstaState}) -> + try + #{cbkmod := CbMod, + state := GwState} = emqx_gateway_registry:lookup(Type), + CbMod:on_insta_destroy(Insta, InstaState, GwState), + {ok, State#state{child_pids = [], + insta_state = undefined, + status = stopped}} + catch + Class : Reason : Stk -> + logger:error("Destroy instance (~0p, ~0p, _) crashed: " + "{~p, ~p}, stacktrace: ~0p", + [Insta, InstaState, + Class, Reason, Stk]), + {error, {Class, Reason, Stk}} + end. + +cb_insta_create(State = #state{insta = Insta = #{type := Type}, + ctx = Ctx}) -> + try + #{cbkmod := CbMod, + state := GwState} = emqx_gateway_registry:lookup(Type), + case CbMod:on_insta_create(Insta, Ctx, GwState) of + {error, Reason} -> throw({callback_return_error, Reason}); + {ok, InstaPidOrSpecs, InstaState} -> + ChildPids = start_child_process(InstaPidOrSpecs), + {ok, State#state{ + status = running, + child_pids = ChildPids, + insta_state = InstaState + }} + end + catch + Class : Reason1 : Stk -> + logger:error("Create instance (~0p, ~0p, _) crashed: " + "{~p, ~p}, stacktrace: ~0p", + [Insta, Ctx, + Class, Reason1, Stk]), + {error, {Class, Reason1, Stk}} + end. + +cb_insta_update(NewInsta, + State = #state{insta = Insta = #{type := Type}, + ctx = Ctx, + insta_state = GwInstaState}) -> + try + #{cbkmod := CbMod, + state := GwState} = emqx_gateway_registry:lookup(Type), + case CbMod:on_insta_update(NewInsta, Insta, GwInstaState, GwState) of + {error, Reason} -> throw({callback_return_error, Reason}); + {ok, InstaPidOrSpecs, InstaState} -> + %% XXX: Hot-upgrade ??? + ChildPids = start_child_process(InstaPidOrSpecs), + {ok, State#state{ + status = running, + child_pids = ChildPids, + insta_state = InstaState + }} + end + catch + Class : Reason1 : Stk -> + logger:error("Update instance (~0p, ~0p, ~0p, _) crashed: " + "{~p, ~p}, stacktrace: ~0p", + [NewInsta, Insta, Ctx, + Class, Reason1, Stk]), + {error, {Class, Reason1, Stk}} + end. + +start_child_process([Indictor|_] = InstaPidOrSpecs) -> + case erlang:is_pid(Indictor) of + true -> + InstaPidOrSpecs; + _ -> + do_start_child_process(InstaPidOrSpecs) + end. + +do_start_child_process(ChildSpecs) when is_list(ChildSpecs) -> + lists:map(fun do_start_child_process/1, ChildSpecs); + +do_start_child_process(_ChildSpec = #{start := {M, F, A}}) -> + case erlang:apply(M, F, A) of + {ok, Pid} -> + Pid; + {error, Reason} -> + throw({start_child_process, Reason}) + end. diff --git a/apps/emqx_gateway/src/emqx_gateway_metrics.erl b/apps/emqx_gateway/src/emqx_gateway_metrics.erl new file mode 100644 index 000000000..461eb3344 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_metrics.erl @@ -0,0 +1,101 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway_metrics). + +-behaviour(gen_server). + +-include("include/emqx_gateway.hrl"). + +-logger_header("[PGW-Metrics]"). + +%% APIs +-export([start_link/1]). + +-export([ inc/2 + , inc/3 + , dec/2 + , dec/3 + ]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-export([tabname/1]). + +-record(state, {}). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +start_link(Type) -> + gen_server:start_link(?MODULE, [Type], []). + +-spec inc(gateway_type(), atom()) -> ok. +inc(Type, Name) -> + inc(Type, Name, 1). + +-spec inc(gateway_type(), atom(), integer()) -> ok. +inc(Type, Name, Oct) -> + ets:update_counter(tabname(Type), Name, {2, Oct}, {Name, 0}), + ok. + +-spec dec(gateway_type(), atom()) -> ok. +dec(Type, Name) -> + inc(Type, Name, -1). + +-spec dec(gateway_type(), atom(), non_neg_integer()) -> ok. +dec(Type, Name, Oct) -> + inc(Type, Name, -Oct). + +tabname(Type) -> + list_to_atom(lists:concat([emqx_gateway_, Type, '_metrics'])). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([Type]) -> + TabOpts = [public, {write_concurrency, true}], + ok = emqx_tables:new(tabname(Type), [set|TabOpts]), + {ok, #state{}}. + +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/src/emqx_gateway_registry.erl b/apps/emqx_gateway/src/emqx_gateway_registry.erl new file mode 100644 index 000000000..dde440756 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_registry.erl @@ -0,0 +1,165 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The Registry Centre of Gateway Type +-module(emqx_gateway_registry). + +-include("include/emqx_gateway.hrl"). + +-logger_header("[PGW-Registry]"). + +-behavior(gen_server). + +%% APIs for Impl. +-export([ load/3 + , unload/1 + ]). + +-export([ list/0 + , lookup/1 + ]). + +%% APIs +-export([start_link/0]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-record(state, { + loaded = #{} :: #{ gateway_type() => descriptor() } + }). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +%%-------------------------------------------------------------------- +%% Mgmt +%%-------------------------------------------------------------------- + +-type registry_options() :: [registry_option()]. + +-type registry_option() :: {cbkmod, atom()}. + +-type gateway_options() :: list(). + +-type descriptor() :: #{ cbkmod := atom() + , rgopts := registry_options() + , gwopts := gateway_options() + , state => any() + }. + +-spec load(gateway_type(), registry_options(), gateway_options()) + -> ok + | {error, any()}. +load(Type, RgOpts, GwOpts) -> + CbMod = proplists:get_value(cbkmod, RgOpts, Type), + Dscrptr = #{ cbkmod => CbMod + , rgopts => RgOpts + , gwopts => GwOpts + }, + call({load, Type, Dscrptr}). + +-spec unload(gateway_type()) -> ok | {error, any()}. +unload(Type) -> + %% TODO: Checking ALL INSTACE HAS STOPPED + call({unload, Type}). + +%% TODO: +%unload(Type, Force) -> +% call({unload, Type, Froce}). + +%% @doc Return all registered protocol gateway implementation +-spec list() -> [{gateway_type(), descriptor()}]. +list() -> + call(all). + +-spec lookup(gateway_type()) -> undefined | descriptor(). +lookup(Type) -> + call({lookup, Type}). + +call(Req) -> + gen_server:call(?MODULE, Req, 5000). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([]) -> + %% TODO: Metrics ??? + process_flag(trap_exit, true), + {ok, #state{loaded = #{}}}. + +handle_call({load, Type, Dscrptr}, _From, State = #state{loaded = Gateways}) -> + case maps:get(Type, Gateways, notfound) of + notfound -> + try + GwOpts = maps:get(gwopts, Dscrptr), + CbMod = maps:get(cbkmod, Dscrptr), + {ok, GwState} = CbMod:init(GwOpts), + NDscrptr = maps:put(state, GwState, Dscrptr), + NGateways = maps:put(Type, NDscrptr, Gateways), + {reply, ok, State#state{loaded = NGateways}} + catch + Class : Reason : Stk -> + logger:error("Load ~s crashed {~p, ~p}; stacktrace: ~0p", + [Type, Class, Reason, Stk]), + {reply, {error, {Class, Reason}}, State} + end; + _ -> + {reply, {error, already_existed}, State} + end; + +handle_call({unload, Type}, _From, State = #state{loaded = Gateways}) -> + case maps:get(Type, Gateways, undefined) of + undefined -> + {reply, ok, State}; + _ -> + emqx_gateway_sup:stop_all_suptree(Type), + {reply, ok, State#state{loaded = maps:remove(Type, Gateways)}} + end; + +handle_call(all, _From, State = #state{loaded = Gateways}) -> + {reply, maps:to_list(Gateways), State}; + +handle_call({lookup, Type}, _From, State = #state{loaded = Gateways}) -> + Reply = maps:get(Type, Gateways, undefined), + {reply, Reply, State}; + +handle_call(Req, _From, State) -> + logger:error("Unexpected call: ~0p", [Req]), + {reply, ok, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl new file mode 100644 index 000000000..f5f2b9ab0 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -0,0 +1,265 @@ +-module(emqx_gateway_schema). + +-dialyzer(no_return). +-dialyzer(no_match). +-dialyzer(no_contracts). +-dialyzer(no_unused). +-dialyzer(no_fail_call). + +-include_lib("typerefl/include/types.hrl"). + +-type duration() :: integer(). +-type bytesize() :: integer(). +-type comma_separated_list() :: list(). +-type ip_port() :: tuple(). + +-typerefl_from_string({duration/0, emqx_schema, to_duration}). +-typerefl_from_string({bytesize/0, emqx_schema, to_bytesize}). +-typerefl_from_string({comma_separated_list/0, emqx_schema, to_comma_separated_list}). +-typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}). + +-behaviour(hocon_schema). + +-reflect_type([ duration/0 + , bytesize/0 + , comma_separated_list/0 + , ip_port/0 + ]). + +-export([structs/0 , fields/1]). +-export([t/1, t/3, t/4, ref/1]). + +structs() -> ["gateway"]. + +fields("gateway") -> + [{stomp, t(ref(stomp))}, + {mqttsn, t(ref(mqttsn))}, + {exproto, t(ref(exproto))}, + {coap, t(ref(coap))} + ]; + +fields(stomp) -> + [{"$id", t(ref(stomp_structs))}]; + +fields(stomp_structs) -> + [ {frame, t(ref(stomp_frame))} + , {clientinfo_override, t(ref(clientinfo_override))} + , {authenticator, t(union([allow_anonymous]))} + , {listener, t(ref(tcp_listener_group))} + ]; + +fields(stomp_frame) -> + [ {max_headers, t(integer(), undefined, 10)} + , {max_headers_length, t(integer(), undefined, 1024)} + , {max_body_length, t(integer(), undefined, 8192)} + ]; + +fields(mqttsn) -> + [{"$id", t(ref(mqttsn_structs))}]; + +fields(mqttsn_structs) -> + [ {gateway_id, t(integer())} + , {broadcast, t(boolean())} + , {enable_stats, t(boolean())} + , {enable_qos3, t(boolean())} + , {idle_timeout, t(duration())} + , {predefined, hoconsc:array(ref(mqttsn_predefined))} + , {clientinfo_override, t(ref(clientinfo_override))} + , {listener, t(ref(udp_listener_group))} + ]; + +fields(mqttsn_predefined) -> + %% FIXME: How to check the $id is a integer ??? + [ {id, t(integer())} + , {topic, t(string())} + ]; + +fields(exproto) -> + [{"$id", t(ref(exproto_structs))}]; + +fields(exproto_structs) -> + [ {server, t(ref(exproto_grpc_server))} + , {handler, t(ref(exproto_grpc_handler))} + , {authenticator, t(union([allow_anonymous]))} + , {listener, t(ref(udp_tcp_listener_group))} + ]; + +fields(exproto_grpc_server) -> + [ {bind, t(integer())} + %% TODO: ssl options + ]; + +fields(exproto_grpc_handler) -> + [ {address, t(string())} + %% TODO: ssl + ]; + +fields(clientinfo_override) -> + [ {username, t(string())} + , {password, t(string())} + , {clientid, t(string())} + ]; + +fields(udp_listener_group) -> + [ {udp, t(ref(udp_listener))} + , {dtls, t(ref(dtls_listener))} + ]; + +fields(tcp_listener_group) -> + [ {tcp, t(ref(tcp_listener))} + , {ssl, t(ref(ssl_listener))} + ]; + +fields(udp_tcp_listener_group) -> + [ {udp, t(ref(udp_listener))} + , {dtls, t(ref(dtls_listener))} + , {tcp, t(ref(tcp_listener))} + , {ssl, t(ref(ssl_listener))} + ]; + +fields(tcp_listener) -> + [ {"$name", t(ref(tcp_listener_settings))}]; + +fields(ssl_listener) -> + [ {"$name", t(ref(ssl_listener_settings))}]; + +fields(udp_listener) -> + [ {"$name", t(ref(udp_listener_settings))}]; + +fields(dtls_listener) -> + [ {"$name", t(ref(dtls_listener_settings))}]; + +fields(listener_settings) -> + % FIXME: + %[ {"bind", t(union(ip_port(), integer()))} + [ {bind, t(integer())} + , {acceptors, t(integer(), undefined, 8)} + , {max_connections, t(integer(), undefined, 1024)} + , {max_conn_rate, t(integer())} + , {active_n, t(integer(), undefined, 100)} + %, {zone, t(string())} + %, {rate_limit, t(comma_separated_list())} + , {access, t(ref(access))} + , {proxy_protocol, t(boolean())} + , {proxy_protocol_timeout, t(duration())} + , {backlog, t(integer(), undefined, 1024)} + , {send_timeout, t(duration(), undefined, "15s")} %% FIXME: mapping it + , {send_timeout_close, t(boolean(), undefined, true)} + , {recbuf, t(bytesize())} + , {sndbuf, t(bytesize())} + , {buffer, t(bytesize())} + , {high_watermark, t(bytesize(), undefined, "1MB")} + , {tune_buffer, t(boolean())} + , {nodelay, t(boolean())} + , {reuseaddr, t(boolean())} + ]; + +fields(tcp_listener_settings) -> + [ + %% some special confs for tcp listener + ] ++ fields(listener_settings); + +fields(ssl_listener_settings) -> + [ + %% some special confs for ssl listener + ] ++ + ssl(undefined, #{handshake_timeout => "15s" + , depth => 10 + , reuse_sessions => true}) ++ fields(listener_settings); + +fields(udp_listener_settings) -> + [ + %% some special confs for udp listener + ] ++ fields(listener_settings); + +fields(dtls_listener_settings) -> + [ + %% some special confs for dtls listener + ] ++ + ssl(undefined, #{handshake_timeout => "15s" + , depth => 10 + , reuse_sessions => true}) ++ fields(listener_settings); + +fields(access) -> + [ {"$id", #{type => string(), + nullable => true}}]; + +fields(coap) -> + [{"$id", t(ref(coap_structs))}]; + +fields(coap_structs) -> + [ {enable_stats, t(boolean(), undefined, true)} + , {authenticator, t(union([allow_anonymous]))} + , {heartbeat, t(duration(), undefined, "15s")} + , {resource, t(union([mqtt, pubsub]), undefined, mqtt)} + , {notify_type, t(union([non, con, qos]), undefined, qos)} + , {subscribe_qos, t(union([qos0, qos1, qos2, coap]), undefined, coap)} + , {publish_qos, t(union([qos0, qos1, qos2, coap]), undefined, coap)} + , {listener, t(ref(udp_listener_group))} + ]; + +fields(ExtraField) -> + Mod = list_to_atom(ExtraField++"_schema"), + Mod:fields(ExtraField). + +%translations() -> []. +% +%translations(_) -> []. + +%%-------------------------------------------------------------------- +%% Helpers + +%% types + +t(Type) -> #{type => Type}. + +t(Type, Mapping, Default) -> + hoconsc:t(Type, #{mapping => Mapping, default => Default}). + +t(Type, Mapping, Default, OverrideEnv) -> + hoconsc:t(Type, #{ mapping => Mapping + , default => Default + , override_env => OverrideEnv + }). + +ref(Field) -> + hoconsc:ref(?MODULE, Field). + +%% utils + +%% generate a ssl field. +%% ssl("emqx", #{"verify" => verify_peer}) will return +%% [ {"cacertfile", t(string(), "emqx.cacertfile", undefined)} +%% , {"certfile", t(string(), "emqx.certfile", undefined)} +%% , {"keyfile", t(string(), "emqx.keyfile", undefined)} +%% , {"verify", t(union(verify_peer, verify_none), "emqx.verify", verify_peer)} +%% , {"server_name_indication", "emqx.server_name_indication", undefined)} +%% ... +ssl(Mapping, Defaults) -> + M = fun (Field) -> + case (Mapping) of + undefined -> undefined; + _ -> Mapping ++ "." ++ Field + end end, + D = fun (Field) -> maps:get(list_to_atom(Field), Defaults, undefined) end, + [ {"enable", t(boolean(), M("enable"), D("enable"))} + , {"cacertfile", t(string(), M("cacertfile"), D("cacertfile"))} + , {"certfile", t(string(), M("certfile"), D("certfile"))} + , {"keyfile", t(string(), M("keyfile"), D("keyfile"))} + , {"verify", t(union(verify_peer, verify_none), M("verify"), D("verify"))} + , {"fail_if_no_peer_cert", t(boolean(), M("fail_if_no_peer_cert"), D("fail_if_no_peer_cert"))} + , {"secure_renegotiate", t(boolean(), M("secure_renegotiate"), D("secure_renegotiate"))} + , {"reuse_sessions", t(boolean(), M("reuse_sessions"), D("reuse_sessions"))} + , {"honor_cipher_order", t(boolean(), M("honor_cipher_order"), D("honor_cipher_order"))} + , {"handshake_timeout", t(duration(), M("handshake_timeout"), D("handshake_timeout"))} + , {"depth", t(integer(), M("depth"), D("depth"))} + , {"password", hoconsc:t(string(), #{mapping => M("key_password"), + default => D("key_password"), + sensitive => true + })} + , {"dhfile", t(string(), M("dhfile"), D("dhfile"))} + , {"server_name_indication", t(union(disable, string()), M("server_name_indication"), + D("server_name_indication"))} + , {"tls_versions", t(comma_separated_list(), M("tls_versions"), D("tls_versions"))} + , {"ciphers", t(comma_separated_list(), M("ciphers"), D("ciphers"))} + , {"psk_ciphers", t(comma_separated_list(), M("ciphers"), D("ciphers"))}]. diff --git a/apps/emqx_gateway/src/emqx_gateway_sup.erl b/apps/emqx_gateway/src/emqx_gateway_sup.erl new file mode 100644 index 000000000..c974060d0 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_sup.erl @@ -0,0 +1,193 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway_sup). + +-behaviour(supervisor). + +-include("include/emqx_gateway.hrl"). + +-export([start_link/0]). + +%% Gateway Instance APIs +-export([ create_gateway_insta/1 + , remove_gateway_insta/1 + , lookup_gateway_insta/1 + , update_gateway_insta/1 + , start_gateway_insta/1 + , stop_gateway_insta/1 + , list_gateway_insta/1 + , list_gateway_insta/0 + ]). + +%% Gateway APs +-export([ list_started_gateway/0 + , stop_all_suptree/1 + ]). + +%% supervisor callbacks +-export([init/1]). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +-spec create_gateway_insta(instance()) -> {ok, pid()} | {error, any()}. +create_gateway_insta(Insta = #{type := Type}) -> + case emqx_gateway_registry:lookup(Type) of + undefined -> {error, {unknown_gateway_id, Type}}; + GwDscrptr -> + {ok, GwSup} = ensure_gateway_suptree_ready(gatewayid(Type)), + emqx_gateway_gw_sup:create_insta(GwSup, Insta, GwDscrptr) + end. + +-spec remove_gateway_insta(instance_id()) -> ok | {error, any()}. +remove_gateway_insta(InstaId) -> + case search_gateway_insta_proc(InstaId) of + {ok, {GwSup, _}} -> + emqx_gateway_gw_sup:remove_insta(GwSup, InstaId); + _ -> + ok + end. + +-spec lookup_gateway_insta(instance_id()) -> instance() | undefined. +lookup_gateway_insta(InstaId) -> + case search_gateway_insta_proc(InstaId) of + {ok, {_, GwInstaPid}} -> + emqx_gateway_insta_sup:info(GwInstaPid); + _ -> + undefined + end. + +-spec update_gateway_insta(instance()) + -> ok + | {error, any()}. +update_gateway_insta(NewInsta = #{type := Type}) -> + case emqx_gateway_utils:find_sup_child(?MODULE, gatewayid(Type)) of + {ok, GwSup} -> + emqx_gateway_gw_sup:update_insta(GwSup, NewInsta); + _ -> {error, not_found} + end. + +start_gateway_insta(InstaId) -> + case search_gateway_insta_proc(InstaId) of + {ok, {GwSup, _}} -> + emqx_gateway_gw_sup:start_insta(GwSup, InstaId); + _ -> {error, not_found} + end. + +-spec stop_gateway_insta(instance_id()) -> ok | {error, any()}. +stop_gateway_insta(InstaId) -> + case search_gateway_insta_proc(InstaId) of + {ok, {GwSup, _}} -> + emqx_gateway_gw_sup:stop_insta(GwSup, InstaId); + _ -> {error, not_found} + end. + +-spec list_gateway_insta(gateway_type()) -> {ok, [instance()]} | {error, any()}. +list_gateway_insta(Type) -> + case emqx_gateway_utils:find_sup_child(?MODULE, gatewayid(Type)) of + {ok, GwSup} -> + {ok, emqx_gateway_gw_sup:list_insta(GwSup)}; + _ -> {error, not_found} + end. + +-spec list_gateway_insta() -> [{gateway_type(), instance()}]. +list_gateway_insta() -> + lists:map( + fun(SupId) -> + Instas = emqx_gateway_gw_sup:list_insta(SupId), + {SupId, Instas} + end, list_started_gateway()). + +-spec list_started_gateway() -> [gateway_type()]. +list_started_gateway() -> + started_gateway_type(). + +-spec stop_all_suptree(atom()) -> ok. +stop_all_suptree(Type) -> + case lists:keyfind(Type, 1, supervisor:which_children(?MODULE)) of + false -> ok; + _ -> + _ = supervisor:terminate_child(?MODULE, Type), + _ = supervisor:delete_child(?MODULE, Type), + ok + end. + +%% Supervisor callback + +init([]) -> + SupFlags = #{ strategy => one_for_one + , intensity => 10 + , period => 60 + }, + ChildSpecs = [ emqx_gateway_utils:childspec(worker, emqx_gateway_registry) + ], + {ok, {SupFlags, ChildSpecs}}. + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +gatewayid(Type) -> + list_to_atom(lists:concat([Type])). + +ensure_gateway_suptree_ready(Type) -> + case lists:keyfind(Type, 1, supervisor:which_children(?MODULE)) of + false -> + ChildSpec = emqx_gateway_utils:childspec( + Type, + supervisor, + emqx_gateway_gw_sup, + [Type] + ), + emqx_gateway_utils:supervisor_ret( + supervisor:start_child(?MODULE, ChildSpec) + ); + {_Id, Pid, _Type, _Mods} -> + {ok, Pid} + end. + +search_gateway_insta_proc(InstaId) -> + search_gateway_insta_proc(InstaId, started_gateway_pid()). + +search_gateway_insta_proc(_InstaId, []) -> + {error, not_found}; +search_gateway_insta_proc(InstaId, [SupPid|More]) -> + case emqx_gateway_utils:find_sup_child(SupPid, InstaId) of + {ok, InstaPid} -> {ok, {SupPid, InstaPid}}; + _ -> + search_gateway_insta_proc(InstaId, More) + end. + +started_gateway_type() -> + lists:filtermap( + fun({Id, _, _, _}) -> + is_a_gateway_id(Id) andalso {true, Id} + end, supervisor:which_children(?MODULE)). + +started_gateway_pid() -> + lists:filtermap( + fun({Id, Pid, _, _}) -> + is_a_gateway_id(Id) andalso {true, Pid} + end, supervisor:which_children(?MODULE)). + +is_a_gateway_id(Id) -> + Id /= emqx_gateway_registry. + diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl new file mode 100644 index 000000000..74bc3a8ce --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -0,0 +1,196 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc Utils funcs for emqx-gateway +-module(emqx_gateway_utils). + +-export([ childspec/2 + , childspec/3 + , childspec/4 + , supervisor_ret/1 + , find_sup_child/2 + ]). + +-export([ apply/2 + , format_listenon/1 + ]). + +-export([ normalize_rawconf/1 + ]). + +%% Common Envs +-export([ active_n/1 + , ratelimit/1 + , frame_options/1 + , init_gc_state/1 + , stats_timer/1 + , idle_timeout/1 + , oom_policy/1 + ]). + +-export([ default_tcp_options/0 + , default_udp_options/0 + , default_subopts/0 + ]). + +-define(ACTIVE_N, 100). +-define(DEFAULT_IDLE_TIMEOUT, 30000). +-define(DEFAULT_GC_OPTS, #{count => 1000, bytes => 1024*1024}). +-define(DEFAULT_OOM_POLICY, #{max_heap_size => 4194304, + message_queue_len => 32000}). + +-spec childspec(supervisor:worker(), Mod :: atom()) + -> supervisor:child_spec(). +childspec(Type, Mod) -> + childspec(Mod, Type, Mod, []). + +-spec childspec(supervisor:worker(), Mod :: atom(), Args :: list()) + -> supervisor:child_spec(). +childspec(Type, Mod, Args) -> + childspec(Mod, Type, Mod, Args). + +-spec childspec(atom(), supervisor:worker(), Mod :: atom(), Args :: list()) + -> supervisor:child_spec(). +childspec(Id, Type, Mod, Args) -> + #{ id => Id + , start => {Mod, start_link, Args} + , type => Type + }. + +-spec supervisor_ret(supervisor:startchild_ret()) + -> {ok, pid()} + | {error, supervisor:startchild_err()}. +supervisor_ret({ok, Pid, _Info}) -> {ok, Pid}; +supervisor_ret(Ret) -> Ret. + +-spec find_sup_child(Sup :: pid() | atom(), ChildId :: supervisor:child_id()) + -> false + | {ok, pid()}. +find_sup_child(Sup, ChildId) -> + case lists:keyfind(ChildId, 1, supervisor:which_children(Sup)) of + false -> false; + {_Id, Pid, _Type, _Mods} -> {ok, Pid} + end. + +apply({M, F, A}, A2) when is_atom(M), + is_atom(M), + is_list(A), + is_list(A2) -> + erlang:apply(M, F, A ++ A2); +apply({F, A}, A2) when is_function(F), + is_list(A), + is_list(A2) -> + erlang:apply(F, A ++ A2); +apply(F, A2) when is_function(F), + is_list(A2) -> + erlang:apply(F, A2). + +format_listenon(Port) when is_integer(Port) -> + io_lib:format("0.0.0.0:~w", [Port]); +format_listenon({Addr, Port}) when is_list(Addr) -> + io_lib:format("~s:~w", [Addr, Port]); +format_listenon({Addr, Port}) when is_tuple(Addr) -> + io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). + +-type listener() :: #{}. + +-type rawconf() :: + #{ clientinfo_override => #{} + , authenticators := #{} + , listeners => listener() + , atom() => any() + }. + +-spec normalize_rawconf(rawconf()) + -> list({ Type :: udp | tcp | ssl | dtls + , ListenOn :: esockd:listen_on() + , SocketOpts :: esockd:option() + , Cfg :: map() + }). +normalize_rawconf(RawConf = #{listener := LisMap}) -> + Cfg0 = maps:without([listener], RawConf), + lists:append(maps:fold(fun(Type, Liss, AccIn1) -> + Listeners = + maps:fold(fun(_Name, Confs, AccIn2) -> + ListenOn = maps:get(bind, Confs), + SocketOpts = esockd:parse_opt(maps:to_list(Confs)), + RemainCfgs = maps:without( + [bind] ++ proplists:get_keys(SocketOpts), + Confs), + Cfg = maps:merge(Cfg0, RemainCfgs), + [{Type, ListenOn, SocketOpts, Cfg}|AccIn2] + end, [], Liss), + [Listeners|AccIn1] + end, [], LisMap)). + +%%-------------------------------------------------------------------- +%% Envs + +active_n(Options) -> + maps:get( + active_n, + maps:get(listener, Options, #{active_n => ?ACTIVE_N}), + ?ACTIVE_N + ). + +-spec idle_timeout(map()) -> pos_integer(). +idle_timeout(Options) -> + maps:get(idle_timeout, Options, ?DEFAULT_IDLE_TIMEOUT). + +-spec ratelimit(map()) -> esockd_rate_limit:config() | undefined. +ratelimit(Options) -> + maps:get(ratelimit, Options, undefined). + +-spec frame_options(map()) -> map(). +frame_options(Options) -> + maps:get(frame, Options, #{}). + +-spec init_gc_state(map()) -> emqx_gc:gc_state() | undefined. +init_gc_state(Options) -> + emqx_misc:maybe_apply(fun emqx_gc:init/1, force_gc_policy(Options)). + +-spec force_gc_policy(map()) -> emqx_gc:opts() | undefined. +force_gc_policy(Options) -> + maps:get(force_gc_policy, Options, ?DEFAULT_GC_OPTS). + +-spec oom_policy(map()) -> emqx_types:oom_policy(). +oom_policy(Options) -> + maps:get(force_shutdown_policy, Options, ?DEFAULT_OOM_POLICY). + +-spec stats_timer(map()) -> undefined | disabled. +stats_timer(Options) -> + case enable_stats(Options) of true -> undefined; false -> disabled end. + +-spec enable_stats(map()) -> boolean(). +enable_stats(Options) -> + maps:get(enable_stats, Options, true). + +%%-------------------------------------------------------------------- +%% Envs2 + +default_tcp_options() -> + [binary, {packet, raw}, {reuseaddr, true}, + {nodelay, true}, {backlog, 512}]. + +default_udp_options() -> + [binary]. + +default_subopts() -> + #{rh => 0, %% Retain Handling + rap => 0, %% Retain as Publish + nl => 0, %% No Local + qos => 0 %% QoS + }. diff --git a/apps/emqx_exhook/README.md b/apps/emqx_gateway/src/exhook/README.md similarity index 100% rename from apps/emqx_exhook/README.md rename to apps/emqx_gateway/src/exhook/README.md diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_gateway/src/exhook/emqx_exhook.app.src similarity index 100% rename from apps/emqx_exhook/src/emqx_exhook.app.src rename to apps/emqx_gateway/src/exhook/emqx_exhook.app.src diff --git a/apps/emqx_exhook/src/emqx_exhook.erl b/apps/emqx_gateway/src/exhook/emqx_exhook.erl similarity index 98% rename from apps/emqx_exhook/src/emqx_exhook.erl rename to apps/emqx_gateway/src/exhook/emqx_exhook.erl index 032d7f91a..b3b3057b6 100644 --- a/apps/emqx_exhook/src/emqx_exhook.erl +++ b/apps/emqx_gateway/src/exhook/emqx_exhook.erl @@ -16,7 +16,7 @@ -module(emqx_exhook). --include("emqx_exhook.hrl"). +-include("src/exhook/include/emqx_exhook.hrl"). -include_lib("emqx/include/logger.hrl"). -logger_header("[ExHook]"). diff --git a/apps/emqx_exhook/src/emqx_exhook_app.erl b/apps/emqx_gateway/src/exhook/emqx_exhook_app.erl similarity index 97% rename from apps/emqx_exhook/src/emqx_exhook_app.erl rename to apps/emqx_gateway/src/exhook/emqx_exhook_app.erl index 4e00340d8..d4621f85e 100644 --- a/apps/emqx_exhook/src/emqx_exhook_app.erl +++ b/apps/emqx_gateway/src/exhook/emqx_exhook_app.erl @@ -18,7 +18,7 @@ -behaviour(application). --include("emqx_exhook.hrl"). +-include("src/exhook/include/emqx_exhook.hrl"). -emqx_plugin(extension). @@ -88,7 +88,7 @@ init_hooks_cnter() -> try _ = ets:new(?CNTER, [named_table, public]), ok catch - exit:badarg:_ -> + error:badarg:_ -> ok end. diff --git a/apps/emqx_exhook/src/emqx_exhook_cli.erl b/apps/emqx_gateway/src/exhook/emqx_exhook_cli.erl similarity index 98% rename from apps/emqx_exhook/src/emqx_exhook_cli.erl rename to apps/emqx_gateway/src/exhook/emqx_exhook_cli.erl index a8dc43b16..efce962d9 100644 --- a/apps/emqx_exhook/src/emqx_exhook_cli.erl +++ b/apps/emqx_gateway/src/exhook/emqx_exhook_cli.erl @@ -16,7 +16,7 @@ -module(emqx_exhook_cli). --include("emqx_exhook.hrl"). +-include("src/exhook/include/emqx_exhook.hrl"). -export([cli/1]). diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_gateway/src/exhook/emqx_exhook_handler.erl similarity index 98% rename from apps/emqx_exhook/src/emqx_exhook_handler.erl rename to apps/emqx_gateway/src/exhook/emqx_exhook_handler.erl index f3964dc42..9033fdacc 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_gateway/src/exhook/emqx_exhook_handler.erl @@ -16,7 +16,7 @@ -module(emqx_exhook_handler). --include("emqx_exhook.hrl"). +-include("src/exhook/include/emqx_exhook.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -27,7 +27,7 @@ , on_client_connected/2 , on_client_disconnected/3 , on_client_authenticate/2 - , on_client_check_acl/4 + , on_client_authorize/4 , on_client_subscribe/3 , on_client_unsubscribe/3 ]). @@ -109,7 +109,7 @@ on_client_authenticate(ClientInfo, AuthResult) -> {ok, AuthResult} end. -on_client_check_acl(ClientInfo, PubSub, Topic, Result) -> +on_client_authorize(ClientInfo, PubSub, Topic, Result) -> Bool = Result == allow, Type = case PubSub of publish -> 'PUBLISH'; @@ -120,7 +120,7 @@ on_client_check_acl(ClientInfo, PubSub, Topic, Result) -> topic => Topic, result => Bool }, - case call_fold('client.check_acl', Req, + case call_fold('client.authorize', Req, fun merge_responsed_bool/2) of {StopOrOk, #{result := Result0}} when is_boolean(Result0) -> NResult = case Result0 of true -> allow; _ -> deny end, diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_gateway/src/exhook/emqx_exhook_server.erl similarity index 98% rename from apps/emqx_exhook/src/emqx_exhook_server.erl rename to apps/emqx_gateway/src/exhook/emqx_exhook_server.erl index f4965e4ca..79bc52b4e 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_gateway/src/exhook/emqx_exhook_server.erl @@ -16,7 +16,7 @@ -module(emqx_exhook_server). --include("emqx_exhook.hrl"). +-include("src/exhook/include/emqx_exhook.hrl"). -include_lib("emqx/include/logger.hrl"). -logger_header("[ExHook Svr]"). @@ -58,7 +58,7 @@ | 'client.connected' | 'client.disconnected' | 'client.authenticate' - | 'client.check_acl' + | 'client.authorize' | 'client.subscribe' | 'client.unsubscribe' | 'session.created' @@ -297,7 +297,7 @@ hk2func('client.connack') -> 'on_client_connack'; hk2func('client.connected') -> 'on_client_connected'; hk2func('client.disconnected') -> 'on_client_disconnected'; hk2func('client.authenticate') -> 'on_client_authenticate'; -hk2func('client.check_acl') -> 'on_client_check_acl'; +hk2func('client.authorize') -> 'on_client_authorize'; hk2func('client.subscribe') -> 'on_client_subscribe'; hk2func('client.unsubscribe') -> 'on_client_unsubscribe'; hk2func('session.created') -> 'on_session_created'; @@ -320,7 +320,7 @@ message_hooks() -> -compile({inline, [available_hooks/0]}). available_hooks() -> ['client.connect', 'client.connack', 'client.connected', - 'client.disconnected', 'client.authenticate', 'client.check_acl', + 'client.disconnected', 'client.authenticate', 'client.authorize', 'client.subscribe', 'client.unsubscribe', 'session.created', 'session.subscribed', 'session.unsubscribed', 'session.resumed', 'session.discarded', 'session.takeovered', diff --git a/apps/emqx_exhook/src/emqx_exhook_sup.erl b/apps/emqx_gateway/src/exhook/emqx_exhook_sup.erl similarity index 100% rename from apps/emqx_exhook/src/emqx_exhook_sup.erl rename to apps/emqx_gateway/src/exhook/emqx_exhook_sup.erl diff --git a/apps/emqx_exhook/include/emqx_exhook.hrl b/apps/emqx_gateway/src/exhook/include/emqx_exhook.hrl similarity index 96% rename from apps/emqx_exhook/include/emqx_exhook.hrl rename to apps/emqx_gateway/src/exhook/include/emqx_exhook.hrl index 7301fdcbb..64131735e 100644 --- a/apps/emqx_exhook/include/emqx_exhook.hrl +++ b/apps/emqx_gateway/src/exhook/include/emqx_exhook.hrl @@ -25,7 +25,7 @@ , {'client.connected', {emqx_exhook_handler, on_client_connected, []}} , {'client.disconnected', {emqx_exhook_handler, on_client_disconnected, []}} , {'client.authenticate', {emqx_exhook_handler, on_client_authenticate, []}} - , {'client.check_acl', {emqx_exhook_handler, on_client_check_acl, []}} + , {'client.authorize', {emqx_exhook_handler, on_client_authorize, []}} , {'client.subscribe', {emqx_exhook_handler, on_client_subscribe, []}} , {'client.unsubscribe', {emqx_exhook_handler, on_client_unsubscribe, []}} , {'session.created', {emqx_exhook_handler, on_session_created, []}} diff --git a/apps/emqx_gateway/src/exhook/prop_exhook_hooks.erl b/apps/emqx_gateway/src/exhook/prop_exhook_hooks.erl new file mode 100644 index 000000000..cb2ab8d11 --- /dev/null +++ b/apps/emqx_gateway/src/exhook/prop_exhook_hooks.erl @@ -0,0 +1,531 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(prop_exhook_hooks). + +% -include_lib("proper/include/proper.hrl"). +% -include_lib("eunit/include/eunit.hrl"). + +% -import(emqx_ct_proper_types, +% [ conninfo/0 +% , clientinfo/0 +% , sessioninfo/0 +% , message/0 +% , connack_return_code/0 +% , topictab/0 +% , topic/0 +% , subopts/0 +% ]). + +% -define(ALL(Vars, Types, Exprs), +% ?SETUP(fun() -> +% State = do_setup(), +% fun() -> do_teardown(State) end +% end, ?FORALL(Vars, Types, Exprs))). + +% %%-------------------------------------------------------------------- +% %% Properties +% %%-------------------------------------------------------------------- + +% prop_client_connect() -> +% ?ALL({ConnInfo, ConnProps}, +% {conninfo(), conn_properties()}, +% begin +% ok = emqx_hooks:run('client.connect', [ConnInfo, ConnProps]), +% {'on_client_connect', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{props => properties(ConnProps), +% conninfo => from_conninfo(ConnInfo) +% }, +% ?assertEqual(Expected, Resp), +% true +% end). + +% prop_client_connack() -> +% ?ALL({ConnInfo, Rc, AckProps}, +% {conninfo(), connack_return_code(), ack_properties()}, +% begin +% ok = emqx_hooks:run('client.connack', [ConnInfo, Rc, AckProps]), +% {'on_client_connack', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{props => properties(AckProps), +% result_code => atom_to_binary(Rc, utf8), +% conninfo => from_conninfo(ConnInfo) +% }, +% ?assertEqual(Expected, Resp), +% true +% end). + +% prop_client_authenticate() -> +% ?ALL({ClientInfo0, AuthResult}, +% {clientinfo(), authresult()}, +% begin +% ClientInfo = inject_magic_into(username, ClientInfo0), +% OutAuthResult = emqx_hooks:run_fold('client.authenticate', [ClientInfo], AuthResult), +% ExpectedAuthResult = case maps:get(username, ClientInfo) of +% <<"baduser">> -> +% AuthResult#{ +% auth_result => not_authorized, +% anonymous => false}; +% <<"gooduser">> -> +% AuthResult#{ +% auth_result => success, +% anonymous => false}; +% <<"normaluser">> -> +% AuthResult#{ +% auth_result => success, +% anonymous => false}; +% _ -> +% case maps:get(auth_result, AuthResult) of +% success -> +% #{auth_result => success, +% anonymous => false}; +% _ -> +% #{auth_result => not_authorized, +% anonymous => false} +% end +% end, +% ?assertEqual(ExpectedAuthResult, OutAuthResult), + +% {'on_client_authenticate', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{result => authresult_to_bool(AuthResult), +% clientinfo => from_clientinfo(ClientInfo) +% }, +% ?assertEqual(Expected, Resp), +% true +% end). + +% prop_client_authorize() -> +% ?ALL({ClientInfo0, PubSub, Topic, Result}, +% {clientinfo(), oneof([publish, subscribe]), +% topic(), oneof([allow, deny])}, +% begin +% ClientInfo = inject_magic_into(username, ClientInfo0), +% OutResult = emqx_hooks:run_fold( +% 'client.authorize', +% [ClientInfo, PubSub, Topic], +% Result), +% ExpectedOutResult = case maps:get(username, ClientInfo) of +% <<"baduser">> -> deny; +% <<"gooduser">> -> allow; +% <<"normaluser">> -> allow; +% _ -> Result +% end, +% ?assertEqual(ExpectedOutResult, OutResult), + +% {'on_client_authorize', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{result => authzresult_to_bool(Result), +% type => pubsub_to_enum(PubSub), +% topic => Topic, +% clientinfo => from_clientinfo(ClientInfo) +% }, +% ?assertEqual(Expected, Resp), +% true +% end). + +% prop_client_connected() -> +% ?ALL({ClientInfo, ConnInfo}, +% {clientinfo(), conninfo()}, +% begin +% ok = emqx_hooks:run('client.connected', [ClientInfo, ConnInfo]), +% {'on_client_connected', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{clientinfo => from_clientinfo(ClientInfo) +% }, +% ?assertEqual(Expected, Resp), +% true +% end). + +% prop_client_disconnected() -> +% ?ALL({ClientInfo, Reason, ConnInfo}, +% {clientinfo(), shutdown_reason(), conninfo()}, +% begin +% ok = emqx_hooks:run('client.disconnected', [ClientInfo, Reason, ConnInfo]), +% {'on_client_disconnected', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{reason => stringfy(Reason), +% clientinfo => from_clientinfo(ClientInfo) +% }, +% ?assertEqual(Expected, Resp), +% true +% end). + +% prop_client_subscribe() -> +% ?ALL({ClientInfo, SubProps, TopicTab}, +% {clientinfo(), sub_properties(), topictab()}, +% begin +% ok = emqx_hooks:run('client.subscribe', [ClientInfo, SubProps, TopicTab]), +% {'on_client_subscribe', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{props => properties(SubProps), +% topic_filters => topicfilters(TopicTab), +% clientinfo => from_clientinfo(ClientInfo) +% }, +% ?assertEqual(Expected, Resp), +% true +% end). + +% prop_client_unsubscribe() -> +% ?ALL({ClientInfo, UnSubProps, TopicTab}, +% {clientinfo(), unsub_properties(), topictab()}, +% begin +% ok = emqx_hooks:run('client.unsubscribe', [ClientInfo, UnSubProps, TopicTab]), +% {'on_client_unsubscribe', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{props => properties(UnSubProps), +% topic_filters => topicfilters(TopicTab), +% clientinfo => from_clientinfo(ClientInfo) +% }, +% ?assertEqual(Expected, Resp), +% true +% end). + +% prop_session_created() -> +% ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()}, +% begin +% ok = emqx_hooks:run('session.created', [ClientInfo, SessInfo]), +% {'on_session_created', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{clientinfo => from_clientinfo(ClientInfo) +% }, +% ?assertEqual(Expected, Resp), +% true +% end). + +% prop_session_subscribed() -> +% ?ALL({ClientInfo, Topic, SubOpts}, +% {clientinfo(), topic(), subopts()}, +% begin +% ok = emqx_hooks:run('session.subscribed', [ClientInfo, Topic, SubOpts]), +% {'on_session_subscribed', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{topic => Topic, +% subopts => subopts(SubOpts), +% clientinfo => from_clientinfo(ClientInfo) +% }, +% ?assertEqual(Expected, Resp), +% true +% end). + +% prop_session_unsubscribed() -> +% ?ALL({ClientInfo, Topic, SubOpts}, +% {clientinfo(), topic(), subopts()}, +% begin +% ok = emqx_hooks:run('session.unsubscribed', [ClientInfo, Topic, SubOpts]), +% {'on_session_unsubscribed', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{topic => Topic, +% clientinfo => from_clientinfo(ClientInfo) +% }, +% ?assertEqual(Expected, Resp), +% true +% end). + +% prop_session_resumed() -> +% ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()}, +% begin +% ok = emqx_hooks:run('session.resumed', [ClientInfo, SessInfo]), +% {'on_session_resumed', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{clientinfo => from_clientinfo(ClientInfo) +% }, +% ?assertEqual(Expected, Resp), +% true +% end). + +% prop_session_discared() -> +% ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()}, +% begin +% ok = emqx_hooks:run('session.discarded', [ClientInfo, SessInfo]), +% {'on_session_discarded', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{clientinfo => from_clientinfo(ClientInfo) +% }, +% ?assertEqual(Expected, Resp), +% true +% end). + +% prop_session_takeovered() -> +% ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()}, +% begin +% ok = emqx_hooks:run('session.takeovered', [ClientInfo, SessInfo]), +% {'on_session_takeovered', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{clientinfo => from_clientinfo(ClientInfo) +% }, +% ?assertEqual(Expected, Resp), +% true +% end). + +% prop_session_terminated() -> +% ?ALL({ClientInfo, Reason, SessInfo}, +% {clientinfo(), shutdown_reason(), sessioninfo()}, +% begin +% ok = emqx_hooks:run('session.terminated', [ClientInfo, Reason, SessInfo]), +% {'on_session_terminated', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{reason => stringfy(Reason), +% clientinfo => from_clientinfo(ClientInfo) +% }, +% ?assertEqual(Expected, Resp), +% true +% end). + +% prop_message_publish() -> +% ?ALL(Msg0, message(), +% begin +% Msg = emqx_message:from_map( +% inject_magic_into(from, emqx_message:to_map(Msg0))), +% OutMsg= emqx_hooks:run_fold('message.publish', [], Msg), +% case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of +% true -> +% ?assertEqual(Msg, OutMsg), +% skip; +% _ -> +% ExpectedOutMsg = case emqx_message:from(Msg) of +% <<"baduser">> -> +% MsgMap = emqx_message:to_map(Msg), +% emqx_message:from_map( +% MsgMap#{qos => 0, +% topic => <<"">>, +% payload => <<"">> +% }); +% <<"gooduser">> = From -> +% MsgMap = emqx_message:to_map(Msg), +% emqx_message:from_map( +% MsgMap#{topic => From, +% payload => From +% }); +% _ -> Msg +% end, +% ?assertEqual(ExpectedOutMsg, OutMsg), + +% {'on_message_publish', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{message => from_message(Msg) +% }, +% ?assertEqual(Expected, Resp) +% end, +% true +% end). + +% prop_message_dropped() -> +% ?ALL({Msg, By, Reason}, {message(), hardcoded, shutdown_reason()}, +% begin +% ok = emqx_hooks:run('message.dropped', [Msg, By, Reason]), +% case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of +% true -> skip; +% _ -> +% {'on_message_dropped', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{reason => stringfy(Reason), +% message => from_message(Msg) +% }, +% ?assertEqual(Expected, Resp) +% end, +% true +% end). + +% prop_message_delivered() -> +% ?ALL({ClientInfo, Msg}, {clientinfo(), message()}, +% begin +% ok = emqx_hooks:run('message.delivered', [ClientInfo, Msg]), +% case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of +% true -> skip; +% _ -> +% {'on_message_delivered', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{clientinfo => from_clientinfo(ClientInfo), +% message => from_message(Msg) +% }, +% ?assertEqual(Expected, Resp) +% end, +% true +% end). + +% prop_message_acked() -> +% ?ALL({ClientInfo, Msg}, {clientinfo(), message()}, +% begin +% ok = emqx_hooks:run('message.acked', [ClientInfo, Msg]), +% case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of +% true -> skip; +% _ -> +% {'on_message_acked', Resp} = emqx_exhook_demo_svr:take(), +% Expected = +% #{clientinfo => from_clientinfo(ClientInfo), +% message => from_message(Msg) +% }, +% ?assertEqual(Expected, Resp) +% end, +% true +% end). + +% nodestr() -> +% stringfy(node()). + +% peerhost(#{peername := {Host, _}}) -> +% ntoa(Host). + +% sockport(#{sockname := {_, Port}}) -> +% Port. + +% %% copied from emqx_exhook + +% ntoa({0,0,0,0,0,16#ffff,AB,CD}) -> +% list_to_binary(inet_parse:ntoa({AB bsr 8, AB rem 256, CD bsr 8, CD rem 256})); +% ntoa(IP) -> +% list_to_binary(inet_parse:ntoa(IP)). + +% maybe(undefined) -> <<>>; +% maybe(B) -> B. + +% properties(undefined) -> []; +% properties(M) when is_map(M) -> +% maps:fold(fun(K, V, Acc) -> +% [#{name => stringfy(K), +% value => stringfy(V)} | Acc] +% end, [], M). + +% topicfilters(Tfs) when is_list(Tfs) -> +% [#{name => Topic, qos => Qos} || {Topic, #{qos := Qos}} <- Tfs]. + +% %% @private +% stringfy(Term) when is_binary(Term) -> +% Term; +% stringfy(Term) when is_integer(Term) -> +% integer_to_binary(Term); +% stringfy(Term) when is_atom(Term) -> +% atom_to_binary(Term, utf8); +% stringfy(Term) -> +% unicode:characters_to_binary((io_lib:format("~0p", [Term]))). + +% subopts(SubOpts) -> +% #{qos => maps:get(qos, SubOpts, 0), +% rh => maps:get(rh, SubOpts, 0), +% rap => maps:get(rap, SubOpts, 0), +% nl => maps:get(nl, SubOpts, 0), +% share => maps:get(share, SubOpts, <<>>) +% }. + +% authresult_to_bool(AuthResult) -> +% maps:get(auth_result, AuthResult, undefined) == success. + +% authzresult_to_bool(Result) -> +% Result == allow. + +% pubsub_to_enum(publish) -> 'PUBLISH'; +% pubsub_to_enum(subscribe) -> 'SUBSCRIBE'. + +% from_conninfo(ConnInfo) -> +% #{node => nodestr(), +% clientid => maps:get(clientid, ConnInfo), +% username => maybe(maps:get(username, ConnInfo, <<>>)), +% peerhost => peerhost(ConnInfo), +% sockport => sockport(ConnInfo), +% proto_name => maps:get(proto_name, ConnInfo), +% proto_ver => stringfy(maps:get(proto_ver, ConnInfo)), +% keepalive => maps:get(keepalive, ConnInfo) +% }. + +% from_clientinfo(ClientInfo) -> +% #{node => nodestr(), +% clientid => maps:get(clientid, ClientInfo), +% username => maybe(maps:get(username, ClientInfo, <<>>)), +% password => maybe(maps:get(password, ClientInfo, <<>>)), +% peerhost => ntoa(maps:get(peerhost, ClientInfo)), +% sockport => maps:get(sockport, ClientInfo), +% protocol => stringfy(maps:get(protocol, ClientInfo)), +% mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)), +% is_superuser => maps:get(is_superuser, ClientInfo, false), +% anonymous => maps:get(anonymous, ClientInfo, true), +% cn => maybe(maps:get(cn, ClientInfo, <<>>)), +% dn => maybe(maps:get(dn, ClientInfo, <<>>)) +% }. + +% from_message(Msg) -> +% #{node => nodestr(), +% id => emqx_guid:to_hexstr(emqx_message:id(Msg)), +% qos => emqx_message:qos(Msg), +% from => stringfy(emqx_message:from(Msg)), +% topic => emqx_message:topic(Msg), +% payload => emqx_message:payload(Msg), +% timestamp => emqx_message:timestamp(Msg) +% }. + +% %%-------------------------------------------------------------------- +% %% Helper +% %%-------------------------------------------------------------------- + +% do_setup() -> +% logger:set_primary_config(#{level => warning}), +% _ = emqx_exhook_demo_svr:start(), +% emqx_ct_helpers:start_apps([emqx_exhook], fun set_special_cfgs/1), +% %% waiting first loaded event +% {'on_provider_loaded', _} = emqx_exhook_demo_svr:take(), +% ok. + +% do_teardown(_) -> +% emqx_ct_helpers:stop_apps([emqx_exhook]), +% %% waiting last unloaded event +% {'on_provider_unloaded', _} = emqx_exhook_demo_svr:take(), +% _ = emqx_exhook_demo_svr:stop(), +% logger:set_primary_config(#{level => notice}), +% timer:sleep(2000), +% ok. + +% set_special_cfgs(emqx) -> +% application:set_env(emqx, allow_anonymous, false), +% application:set_env(emqx, enable_authz_cache, false), +% application:set_env(emqx, modules_loaded_file, undefined), +% application:set_env(emqx, plugins_loaded_file, +% emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins")); +% set_special_cfgs(emqx_exhook) -> +% ok. + +% %%-------------------------------------------------------------------- +% %% Generators +% %%-------------------------------------------------------------------- + +% conn_properties() -> +% #{}. + +% ack_properties() -> +% #{}. + +% sub_properties() -> +% #{}. + +% unsub_properties() -> +% #{}. + +% shutdown_reason() -> +% oneof([utf8(), {shutdown, emqx_ct_proper_types:limited_atom()}]). + +% authresult() -> +% ?LET(RC, connack_return_code(), #{auth_result => RC}). + +% inject_magic_into(Key, Object) -> +% case castspell() of +% muggles -> Object; +% Spell -> +% Object#{Key => Spell} +% end. + +% castspell() -> +% L = [<<"baduser">>, <<"gooduser">>, <<"normaluser">>, muggles], +% lists:nth(rand:uniform(length(L)), L). diff --git a/apps/emqx_gateway/src/exhook/test/emqx_exhook_SUITE.erl b/apps/emqx_gateway/src/exhook/test/emqx_exhook_SUITE.erl new file mode 100644 index 000000000..5dc29e6f1 --- /dev/null +++ b/apps/emqx_gateway/src/exhook/test/emqx_exhook_SUITE.erl @@ -0,0 +1,97 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_exhook_SUITE). + +% -compile(export_all). +% -compile(nowarn_export_all). + + +% -include_lib("eunit/include/eunit.hrl"). +% -include_lib("common_test/include/ct.hrl"). + +% %%-------------------------------------------------------------------- +% %% Setups +% %%-------------------------------------------------------------------- + +% all() -> emqx_ct:all(?MODULE). + +% init_per_suite(Cfg) -> +% _ = emqx_exhook_demo_svr:start(), +% emqx_ct_helpers:start_apps([emqx_exhook], fun set_special_cfgs/1), +% Cfg. + +% end_per_suite(_Cfg) -> +% emqx_ct_helpers:stop_apps([emqx_exhook]), +% emqx_exhook_demo_svr:stop(). + +% set_special_cfgs(emqx) -> +% application:set_env(emqx, allow_anonymous, false), +% application:set_env(emqx, enable_authz_cache, false), +% application:set_env(emqx, plugins_loaded_file, undefined), +% application:set_env(emqx, modules_loaded_file, undefined); +% set_special_cfgs(emqx_exhook) -> +% ok. + +% %%-------------------------------------------------------------------- +% %% Test cases +% %%-------------------------------------------------------------------- + +% t_noserver_nohook(_) -> +% emqx_exhook:disable(default), +% ?assertEqual([], ets:tab2list(emqx_hooks)), + +% Opts = proplists:get_value( +% default, +% application:get_env(emqx_exhook, servers, []) +% ), +% ok = emqx_exhook:enable(default, Opts), +% ?assertNotEqual([], ets:tab2list(emqx_hooks)). + +% t_cli_list(_) -> +% meck_print(), +% ?assertEqual( [[emqx_exhook_server:format(Svr) || Svr <- emqx_exhook:list()]] +% , emqx_exhook_cli:cli(["server", "list"]) +% ), +% unmeck_print(). + +% t_cli_enable_disable(_) -> +% meck_print(), +% ?assertEqual([already_started], emqx_exhook_cli:cli(["server", "enable", "default"])), +% ?assertEqual(ok, emqx_exhook_cli:cli(["server", "disable", "default"])), +% ?assertEqual([], emqx_exhook_cli:cli(["server", "list"])), + +% ?assertEqual([not_running], emqx_exhook_cli:cli(["server", "disable", "default"])), +% ?assertEqual(ok, emqx_exhook_cli:cli(["server", "enable", "default"])), +% unmeck_print(). + +% t_cli_stats(_) -> +% meck_print(), +% _ = emqx_exhook_cli:cli(["server", "stats"]), +% _ = emqx_exhook_cli:cli(x), +% unmeck_print(). + +% %%-------------------------------------------------------------------- +% %% Utils +% %%-------------------------------------------------------------------- + +% meck_print() -> +% meck:new(emqx_ctl, [passthrough, no_history, no_link]), +% meck:expect(emqx_ctl, print, fun(_) -> ok end), +% meck:expect(emqx_ctl, print, fun(_, Args) -> Args end). + +% unmeck_print() -> +% meck:unload(emqx_ctl). diff --git a/apps/emqx_gateway/src/exhook/test/emqx_exhook_demo_svr.erl b/apps/emqx_gateway/src/exhook/test/emqx_exhook_demo_svr.erl new file mode 100644 index 000000000..bcc8865ab --- /dev/null +++ b/apps/emqx_gateway/src/exhook/test/emqx_exhook_demo_svr.erl @@ -0,0 +1,339 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_exhook_demo_svr). + +% -behavior(emqx_exhook_v_1_hook_provider_bhvr). + +% %% +% -export([ start/0 +% , stop/0 +% , take/0 +% , in/1 +% ]). + +% %% gRPC server HookProvider callbacks +% -export([ on_provider_loaded/2 +% , on_provider_unloaded/2 +% , on_client_connect/2 +% , on_client_connack/2 +% , on_client_connected/2 +% , on_client_disconnected/2 +% , on_client_authenticate/2 +% , on_client_authorize/2 +% , on_client_subscribe/2 +% , on_client_unsubscribe/2 +% , on_session_created/2 +% , on_session_subscribed/2 +% , on_session_unsubscribed/2 +% , on_session_resumed/2 +% , on_session_discarded/2 +% , on_session_takeovered/2 +% , on_session_terminated/2 +% , on_message_publish/2 +% , on_message_delivered/2 +% , on_message_dropped/2 +% , on_message_acked/2 +% ]). + +% -define(PORT, 9000). +% -define(NAME, ?MODULE). + +% %%-------------------------------------------------------------------- +% %% Server APIs +% %%-------------------------------------------------------------------- + +% start() -> +% Pid = spawn(fun mngr_main/0), +% register(?MODULE, Pid), +% {ok, Pid}. + +% stop() -> +% grpc:stop_server(?NAME), +% ?MODULE ! stop. + +% take() -> +% ?MODULE ! {take, self()}, +% receive {value, V} -> V +% after 5000 -> error(timeout) end. + +% in({FunName, Req}) -> +% ?MODULE ! {in, FunName, Req}. + +% mngr_main() -> +% application:ensure_all_started(grpc), +% Services = #{protos => [emqx_exhook_pb], +% services => #{'emqx.exhook.v1.HookProvider' => emqx_exhook_demo_svr} +% }, +% Options = [], +% Svr = grpc:start_server(?NAME, ?PORT, Services, Options), +% mngr_loop([Svr, queue:new(), queue:new()]). + +% mngr_loop([Svr, Q, Takes]) -> +% receive +% {in, FunName, Req} -> +% {NQ1, NQ2} = reply(queue:in({FunName, Req}, Q), Takes), +% mngr_loop([Svr, NQ1, NQ2]); +% {take, From} -> +% {NQ1, NQ2} = reply(Q, queue:in(From, Takes)), +% mngr_loop([Svr, NQ1, NQ2]); +% stop -> +% exit(normal) +% end. + +% reply(Q1, Q2) -> +% case queue:len(Q1) =:= 0 orelse +% queue:len(Q2) =:= 0 of +% true -> {Q1, Q2}; +% _ -> +% {{value, {Name, V}}, NQ1} = queue:out(Q1), +% {{value, From}, NQ2} = queue:out(Q2), +% From ! {value, {Name, V}}, +% {NQ1, NQ2} +% end. + +% %%-------------------------------------------------------------------- +% %% callbacks +% %%-------------------------------------------------------------------- + +% -spec on_provider_loaded(emqx_exhook_pb:provider_loaded_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:loaded_response(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. + +% on_provider_loaded(Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% {ok, #{hooks => [ +% #{name => <<"client.connect">>}, +% #{name => <<"client.connack">>}, +% #{name => <<"client.connected">>}, +% #{name => <<"client.disconnected">>}, +% #{name => <<"client.authenticate">>}, +% #{name => <<"client.authorize">>}, +% #{name => <<"client.subscribe">>}, +% #{name => <<"client.unsubscribe">>}, +% #{name => <<"session.created">>}, +% #{name => <<"session.subscribed">>}, +% #{name => <<"session.unsubscribed">>}, +% #{name => <<"session.resumed">>}, +% #{name => <<"session.discarded">>}, +% #{name => <<"session.takeovered">>}, +% #{name => <<"session.terminated">>}, +% #{name => <<"message.publish">>}, +% #{name => <<"message.delivered">>}, +% #{name => <<"message.acked">>}, +% #{name => <<"message.dropped">>}]}, Md}. +% -spec on_provider_unloaded(emqx_exhook_pb:provider_unloaded_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_provider_unloaded(Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% {ok, #{}, Md}. + +% -spec on_client_connect(emqx_exhook_pb:client_connect_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_client_connect(Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% {ok, #{}, Md}. + +% -spec on_client_connack(emqx_exhook_pb:client_connack_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_client_connack(Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% {ok, #{}, Md}. + +% -spec on_client_connected(emqx_exhook_pb:client_connected_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_client_connected(Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% {ok, #{}, Md}. + +% -spec on_client_disconnected(emqx_exhook_pb:client_disconnected_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_client_disconnected(Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% {ok, #{}, Md}. + +% -spec on_client_authenticate(emqx_exhook_pb:client_authenticate_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_client_authenticate(#{clientinfo := #{username := Username}} = Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% %% some cases for testing +% case Username of +% <<"baduser">> -> +% {ok, #{type => 'STOP_AND_RETURN', +% value => {bool_result, false}}, Md}; +% <<"gooduser">> -> +% {ok, #{type => 'STOP_AND_RETURN', +% value => {bool_result, true}}, Md}; +% <<"normaluser">> -> +% {ok, #{type => 'CONTINUE', +% value => {bool_result, true}}, Md}; +% _ -> +% {ok, #{type => 'IGNORE'}, Md} +% end. + +% -spec on_client_authorize(emqx_exhook_pb:client_authorize_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_client_authorize(#{clientinfo := #{username := Username}} = Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% %% some cases for testing +% case Username of +% <<"baduser">> -> +% {ok, #{type => 'STOP_AND_RETURN', +% value => {bool_result, false}}, Md}; +% <<"gooduser">> -> +% {ok, #{type => 'STOP_AND_RETURN', +% value => {bool_result, true}}, Md}; +% <<"normaluser">> -> +% {ok, #{type => 'CONTINUE', +% value => {bool_result, true}}, Md}; +% _ -> +% {ok, #{type => 'IGNORE'}, Md} +% end. + +% -spec on_client_subscribe(emqx_exhook_pb:client_subscribe_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_client_subscribe(Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% {ok, #{}, Md}. + +% -spec on_client_unsubscribe(emqx_exhook_pb:client_unsubscribe_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_client_unsubscribe(Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% {ok, #{}, Md}. + +% -spec on_session_created(emqx_exhook_pb:session_created_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_session_created(Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% {ok, #{}, Md}. + +% -spec on_session_subscribed(emqx_exhook_pb:session_subscribed_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_session_subscribed(Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% {ok, #{}, Md}. + +% -spec on_session_unsubscribed(emqx_exhook_pb:session_unsubscribed_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_session_unsubscribed(Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% {ok, #{}, Md}. + +% -spec on_session_resumed(emqx_exhook_pb:session_resumed_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_session_resumed(Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% {ok, #{}, Md}. + +% -spec on_session_discarded(emqx_exhook_pb:session_discarded_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_session_discarded(Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% {ok, #{}, Md}. + +% -spec on_session_takeovered(emqx_exhook_pb:session_takeovered_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_session_takeovered(Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% {ok, #{}, Md}. + +% -spec on_session_terminated(emqx_exhook_pb:session_terminated_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_session_terminated(Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% {ok, #{}, Md}. + +% -spec on_message_publish(emqx_exhook_pb:message_publish_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_message_publish(#{message := #{from := From} = Msg} = Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% %% some cases for testing +% case From of +% <<"baduser">> -> +% NMsg = Msg#{qos => 0, +% topic => <<"">>, +% payload => <<"">> +% }, +% {ok, #{type => 'STOP_AND_RETURN', +% value => {message, NMsg}}, Md}; +% <<"gooduser">> -> +% NMsg = Msg#{topic => From, +% payload => From}, +% {ok, #{type => 'STOP_AND_RETURN', +% value => {message, NMsg}}, Md}; +% _ -> +% {ok, #{type => 'IGNORE'}, Md} +% end. + +% -spec on_message_delivered(emqx_exhook_pb:message_delivered_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_message_delivered(Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% {ok, #{}, Md}. + +% -spec on_message_dropped(emqx_exhook_pb:message_dropped_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_message_dropped(Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% {ok, #{}, Md}. + +% -spec on_message_acked(emqx_exhook_pb:message_acked_request(), grpc:metadata()) +% -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} +% | {error, grpc_cowboy_h:error_response()}. +% on_message_acked(Req, Md) -> +% ?MODULE:in({?FUNCTION_NAME, Req}), +% %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), +% {ok, #{}, Md}. diff --git a/apps/emqx_exproto/README.md b/apps/emqx_gateway/src/exproto/README.md similarity index 100% rename from apps/emqx_exproto/README.md rename to apps/emqx_gateway/src/exproto/README.md diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl similarity index 82% rename from apps/emqx_exproto/src/emqx_exproto_channel.erl rename to apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl index 229e6f930..5301de4e4 100644 --- a/apps/emqx_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl @@ -15,8 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_exproto_channel). - --include("emqx_exproto.hrl"). +-include("src/exproto/include/emqx_exproto.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/types.hrl"). @@ -42,6 +41,8 @@ -export_type([channel/0]). -record(channel, { + %% Context + ctx :: emqx_gateway_ctx:context(), %% gRPC channel options gcli :: map(), %% Conn info @@ -81,29 +82,16 @@ -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]). --define(SESSION_STATS_KEYS, - [subscriptions_cnt, - subscriptions_max, - inflight_cnt, - inflight_max, - mqueue_len, - mqueue_max, - mqueue_dropped, - next_pkt_id, - awaiting_rel_cnt, - awaiting_rel_max - ]). - %%-------------------------------------------------------------------- %% Info, Attrs and Caps %%-------------------------------------------------------------------- %% @doc Get infos of the channel. --spec(info(channel()) -> emqx_types:infos()). +-spec info(channel()) -> emqx_types:infos(). info(Channel) -> maps:from_list(info(?INFO_KEYS, Channel)). --spec(info(list(atom())|atom(), channel()) -> term()). +-spec info(list(atom())|atom(), channel()) -> term(). info(Keys, Channel) when is_list(Keys) -> [{Key, info(Key, Channel)} || Key <- Keys]; info(conninfo, #channel{conninfo = ConnInfo}) -> @@ -122,9 +110,11 @@ info(session, #channel{subscriptions = Subs, info(conn_state, #channel{conn_state = ConnState}) -> ConnState; info(will_msg, _) -> - undefined. + undefined; +info(ctx, #channel{ctx = Ctx}) -> + Ctx. --spec(stats(channel()) -> emqx_types:stats()). +-spec stats(channel()) -> emqx_types:stats(). stats(#channel{subscriptions = Subs}) -> [{subscriptions_cnt, maps:size(Subs)}, {subscriptions_max, 0}, @@ -141,20 +131,24 @@ stats(#channel{subscriptions = Subs}) -> %% Init the channel %%-------------------------------------------------------------------- --spec(init(emqx_exproto_types:conninfo(), proplists:proplist()) -> channel()). +-spec init(emqx_exproto_types:conninfo(), map()) -> channel(). init(ConnInfo = #{socktype := Socktype, peername := Peername, sockname := Sockname, peercert := Peercert}, Options) -> - GRpcChann = proplists:get_value(handler, Options), + Ctx = maps:get(ctx, Options), + GRpcChann = maps:get(handler, Options), + PoolName = maps:get(pool_name, Options), NConnInfo = default_conninfo(ConnInfo), ClientInfo = default_clientinfo(ConnInfo), - Channel = #channel{gcli = #{channel => GRpcChann}, - conninfo = NConnInfo, - clientinfo = ClientInfo, - conn_state = connecting, - timers = #{} - }, + Channel = #channel{ + ctx = Ctx, + gcli = #{channel => GRpcChann, pool_name => PoolName}, + conninfo = NConnInfo, + clientinfo = ClientInfo, + conn_state = connecting, + timers = #{} + }, Req = #{conninfo => peercert(Peercert, @@ -164,12 +158,21 @@ init(ConnInfo = #{socktype := Socktype, try_dispatch(on_socket_created, wrap(Req), Channel). %% @private -peercert(nossl, ConnInfo) -> +peercert(NoSsl, ConnInfo) when NoSsl == nossl; + NoSsl == undefined -> ConnInfo; peercert(Peercert, ConnInfo) -> - ConnInfo#{peercert => - #{cn => esockd_peercert:common_name(Peercert), - dn => esockd_peercert:subject(Peercert)}}. + Fn = fun(_, V) -> V =/= undefined end, + Infos = maps:filter(Fn, + #{cn => esockd_peercert:common_name(Peercert), + dn => esockd_peercert:subject(Peercert)} + ), + case maps:size(Infos) of + 0 -> + ConnInfo; + _ -> + ConnInfo#{peercert => Infos} + end. %% @private socktype(tcp) -> 'TCP'; @@ -185,22 +188,23 @@ address({Host, Port}) -> %% Handle incoming packet %%-------------------------------------------------------------------- --spec(handle_in(binary(), channel()) +-spec handle_in(binary(), channel()) -> {ok, channel()} - | {shutdown, Reason :: term(), channel()}). + | {shutdown, Reason :: term(), channel()}. handle_in(Data, Channel) -> Req = #{bytes => Data}, {ok, try_dispatch(on_received_bytes, wrap(Req), Channel)}. --spec(handle_deliver(list(emqx_types:deliver()), channel()) +-spec handle_deliver(list(emqx_types:deliver()), channel()) -> {ok, channel()} - | {shutdown, Reason :: term(), channel()}). -handle_deliver(Delivers, Channel = #channel{clientinfo = ClientInfo}) -> + | {shutdown, Reason :: term(), channel()}. +handle_deliver(Delivers, Channel = #channel{ctx = Ctx, + clientinfo = ClientInfo}) -> %% XXX: ?? Nack delivers from shared subscriptions Mountpoint = maps:get(mountpoint, ClientInfo), NodeStr = atom_to_binary(node(), utf8), Msgs = lists:map(fun({_, _, Msg}) -> - ok = emqx_metrics:inc('messages.delivered'), + ok = metrics_inc(Ctx, 'messages.delivered'), Msg1 = emqx_hooks:run_fold('message.delivered', [ClientInfo], Msg), NMsg = emqx_mountpoint:unmount(Mountpoint, Msg1), @@ -216,9 +220,9 @@ handle_deliver(Delivers, Channel = #channel{clientinfo = ClientInfo}) -> Req = #{messages => Msgs}, {ok, try_dispatch(on_received_messages, wrap(Req), Channel)}. --spec(handle_timeout(reference(), Msg :: term(), channel()) +-spec handle_timeout(reference(), Msg :: term(), channel()) -> {ok, channel()} - | {shutdown, Reason :: term(), channel()}). + | {shutdown, Reason :: term(), channel()}. handle_timeout(_TRef, {keepalive, _StatVal}, Channel = #channel{keepalive = undefined}) -> {ok, Channel}; @@ -240,10 +244,10 @@ handle_timeout(_TRef, Msg, Channel) -> ?WARN("Unexpected timeout: ~p", [Msg]), {ok, Channel}. --spec(handle_call(any(), channel()) - -> {reply, Reply :: term(), channel()} - | {reply, Reply :: term(), replies(), channel()} - | {shutdown, Reason :: term(), Reply :: term(), channel()}). +-spec handle_call(any(), channel()) + -> {reply, Reply :: term(), channel()} + | {reply, Reply :: term(), replies(), channel()} + | {shutdown, Reason :: term(), Reply :: term(), channel()}. handle_call({send, Data}, Channel) -> {reply, ok, [{outgoing, Data}], Channel}; @@ -257,28 +261,35 @@ handle_call({auth, ClientInfo, _Password}, Channel = #channel{conn_state = conne ?LOG(warning, "Duplicated authorized command, dropped ~p", [ClientInfo]), {reply, {error, ?RESP_PERMISSION_DENY, <<"Duplicated authenticate command">>}, Channel}; handle_call({auth, ClientInfo0, Password}, - Channel = #channel{conninfo = ConnInfo, - clientinfo = ClientInfo}) -> + Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo}) -> ClientInfo1 = enrich_clientinfo(ClientInfo0, ClientInfo), - NConnInfo = enrich_conninfo(ClientInfo1, ConnInfo), + ConnInfo1 = enrich_conninfo(ClientInfo1, ConnInfo), - Channel1 = Channel#channel{conninfo = NConnInfo, + Channel1 = Channel#channel{conninfo = ConnInfo1, clientinfo = ClientInfo1}, #{clientid := ClientId, username := Username} = ClientInfo1, - case emqx_access_control:authenticate(ClientInfo1#{password => Password}) of - {ok, AuthResult} -> + case emqx_gateway_ctx:authenticate( + Ctx, ClientInfo1#{password => Password}) of + {ok, NClientInfo} -> + SessFun = fun(_, _) -> #{} end, emqx_logger:set_metadata_clientid(ClientId), - is_anonymous(AuthResult) andalso - emqx_metrics:inc('client.auth.anonymous'), - NClientInfo = maps:merge(ClientInfo1, AuthResult), - NChannel = Channel1#channel{clientinfo = NClientInfo}, - case emqx_cm:open_session(true, NClientInfo, NConnInfo) of + case emqx_gateway_ctx:open_session( + Ctx, + true, + NClientInfo, + ConnInfo1, + SessFun + ) of {ok, _Session} -> ?LOG(debug, "Client ~s (Username: '~s') authorized successfully!", [ClientId, Username]), - {reply, ok, [{event, connected}], ensure_connected(NChannel)}; + {reply, ok, [{event, connected}], + ensure_connected(Channel1#channel{clientinfo = NClientInfo})}; {error, Reason} -> ?LOG(warning, "Client ~s (Username: '~s') open session failed for ~0p", [ClientId, Username, Reason]), @@ -302,12 +313,12 @@ handle_call({start_timer, keepalive, Interval}, handle_call({subscribe, TopicFilter, Qos}, Channel = #channel{ + ctx = Ctx, conn_state = connected, clientinfo = ClientInfo}) -> - case is_acl_enabled(ClientInfo) andalso - emqx_access_control:check_acl(ClientInfo, subscribe, TopicFilter) of + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, TopicFilter) of deny -> - {reply, {error, ?RESP_PERMISSION_DENY, <<"ACL deny">>}, Channel}; + {reply, {error, ?RESP_PERMISSION_DENY, <<"Authorization deny">>}, Channel}; _ -> {ok, NChannel} = do_subscribe([{TopicFilter, #{qos => Qos}}], Channel), {reply, ok, NChannel} @@ -320,14 +331,14 @@ handle_call({unsubscribe, TopicFilter}, handle_call({publish, Topic, Qos, Payload}, Channel = #channel{ + ctx = Ctx, conn_state = connected, clientinfo = ClientInfo = #{clientid := From, mountpoint := Mountpoint}}) -> - case is_acl_enabled(ClientInfo) andalso - emqx_access_control:check_acl(ClientInfo, publish, Topic) of + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, publish, Topic) of deny -> - {reply, {error, ?RESP_PERMISSION_DENY, <<"ACL deny">>}, Channel}; + {reply, {error, ?RESP_PERMISSION_DENY, <<"Authorization deny">>}, Channel}; _ -> Msg = emqx_message:make(From, Qos, Topic, Payload), NMsg = emqx_mountpoint:mount(Mountpoint, Msg), @@ -342,17 +353,17 @@ handle_call(Req, Channel) -> ?LOG(warning, "Unexpected call: ~p", [Req]), {reply, {error, unexpected_call}, Channel}. --spec(handle_cast(any(), channel()) - -> {ok, channel()} - | {ok, replies(), channel()} - | {shutdown, Reason :: term(), channel()}). +-spec handle_cast(any(), channel()) + -> {ok, channel()} + | {ok, replies(), channel()} + | {shutdown, Reason :: term(), channel()}. handle_cast(Req, Channel) -> ?WARN("Unexpected call: ~p", [Req]), {ok, Channel}. --spec(handle_info(any(), channel()) - -> {ok, channel()} - | {shutdown, Reason :: term(), channel()}). +-spec handle_info(any(), channel()) + -> {ok, channel()} + | {shutdown, Reason :: term(), channel()}. handle_info({subscribe, TopicFilters}, Channel) -> do_subscribe(TopicFilters, Channel); @@ -388,14 +399,11 @@ handle_info(Info, Channel) -> ?LOG(warning, "Unexpected info: ~p", [Info]), {ok, Channel}. --spec(terminate(any(), channel()) -> channel()). +-spec terminate(any(), channel()) -> channel(). terminate(Reason, Channel) -> Req = #{reason => stringfy(Reason)}, try_dispatch(on_socket_closed, wrap(Req), Channel). -is_anonymous(#{anonymous := true}) -> true; -is_anonymous(_AuthResult) -> false. - %%-------------------------------------------------------------------- %% Sub/UnSub %%-------------------------------------------------------------------- @@ -454,36 +462,39 @@ do_unsubscribe(TopicFilter, UnSubOpts, Channel = parse_topic_filters(TopicFilters) -> lists:map(fun emqx_topic:parse/1, TopicFilters). --compile({inline, [is_acl_enabled/1]}). -is_acl_enabled(#{zone := Zone, is_superuser := IsSuperuser}) -> - (not IsSuperuser) andalso emqx_zone:enable_acl(Zone). - %%-------------------------------------------------------------------- %% Ensure & Hooks %%-------------------------------------------------------------------- -ensure_connected(Channel = #channel{conninfo = ConnInfo, - clientinfo = ClientInfo}) -> +ensure_connected(Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo}) -> NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, - ok = run_hooks('client.connected', [ClientInfo, NConnInfo]), + ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]), Channel#channel{conninfo = NConnInfo, conn_state = connected }. ensure_disconnected(Reason, Channel = #channel{ + ctx = Ctx, conn_state = connected, conninfo = ConnInfo, clientinfo = ClientInfo}) -> NConnInfo = ConnInfo#{disconnected_at => erlang:system_time(millisecond)}, - ok = run_hooks('client.disconnected', [ClientInfo, Reason, NConnInfo]), + ok = run_hooks(Ctx, 'client.disconnected', [ClientInfo, Reason, NConnInfo]), Channel#channel{conninfo = NConnInfo, conn_state = disconnected}; ensure_disconnected(_Reason, Channel = #channel{conninfo = ConnInfo}) -> NConnInfo = ConnInfo#{disconnected_at => erlang:system_time(millisecond)}, Channel#channel{conninfo = NConnInfo, conn_state = disconnected}. -run_hooks(Name, Args) -> - ok = emqx_metrics:inc(Name), emqx_hooks:run(Name, Args). +run_hooks(Ctx, Name, Args) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name), + emqx_hooks:run(Name, Args). + +metrics_inc(Ctx, Name) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name). %%-------------------------------------------------------------------- %% Enrich Keepalive diff --git a/apps/emqx_telemetry/src/emqx_telemetry_app.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_frame.erl similarity index 60% rename from apps/emqx_telemetry/src/emqx_telemetry_app.erl rename to apps/emqx_gateway/src/exproto/emqx_exproto_frame.erl index 89a30393b..ecd6273ca 100644 --- a/apps/emqx_telemetry/src/emqx_telemetry_app.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_frame.erl @@ -1,6 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% +%% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. %% You may obtain a copy of the License at @@ -14,21 +15,36 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_telemetry_app). +%% @doc The frame parser for ExProto +-module(emqx_exproto_frame). --behaviour(application). +-behavior(emqx_gateway_frame). --define(APP, emqx_telemetry). - --export([ start/2 - , stop/1 +-export([ initial_parse_state/1 + , serialize_opts/0 + , parse/2 + , serialize_pkt/2 + , format/1 + , is_message/1 + , type/1 ]). -start(_Type, _Args) -> - Enabled = emqx_config:get([?APP, enabled], true), - emqx_telemetry_sup:start_link([{enabled, Enabled}]). +initial_parse_state(_) -> + #{}. -stop(_State) -> - emqx_ctl:unregister_command(telemetry), - ok. +serialize_opts() -> + #{}. + +parse(Data, State) -> + {ok, Data, <<>>, State}. + +serialize_pkt(Data, _Opts) -> + Data. + +format(Data) -> + io_lib:format("~p", [Data]). + +is_message(_) -> true. + +type(_) -> unknown. diff --git a/apps/emqx_exproto/src/emqx_exproto_gcli.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl similarity index 94% rename from apps/emqx_exproto/src/emqx_exproto_gcli.erl rename to apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl index 650922c4b..34f0606ef 100644 --- a/apps/emqx_exproto/src/emqx_exproto_gcli.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl @@ -53,20 +53,21 @@ start_link(Pool, Id) -> gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], []). -async_call(FunName, Req = #{conn := Conn}, Options) -> - cast(pick(Conn), {rpc, FunName, Req, Options, self()}). +async_call(FunName, Req = #{conn := Conn}, + Options = #{pool_name := PoolName}) -> + cast(pick(PoolName, Conn), {rpc, FunName, Req, Options, self()}). %%-------------------------------------------------------------------- %% cast, pick %%-------------------------------------------------------------------- --compile({inline, [cast/2, pick/1]}). +-compile({inline, [cast/2, pick/2]}). cast(Deliver, Msg) -> gen_server:cast(Deliver, Msg). -pick(Conn) -> - gproc_pool:pick_worker(exproto_gcli_pool, Conn). +pick(PoolName, Conn) -> + gproc_pool:pick_worker(PoolName, Conn). %%-------------------------------------------------------------------- %% gen_server callbacks diff --git a/apps/emqx_exproto/src/emqx_exproto_gsvr.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl similarity index 97% rename from apps/emqx_exproto/src/emqx_exproto_gsvr.erl rename to apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl index a4ad5b2e4..f09a9df7b 100644 --- a/apps/emqx_exproto/src/emqx_exproto_gsvr.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl @@ -17,9 +17,9 @@ %% The gRPC server for ConnectionAdapter -module(emqx_exproto_gsvr). --behavior(emqx_exproto_v_1_connection_adapter_bhvr). +% -behavior(emqx_exproto_v_1_connection_adapter_bhvr). --include("emqx_exproto.hrl"). +-include("src/exproto/include/emqx_exproto.hrl"). -include_lib("emqx/include/logger.hrl"). -logger_header("[ExProto gServer]"). @@ -125,7 +125,7 @@ call(ConnStr, Req) -> Pid when is_pid(Pid) -> case erlang:is_process_alive(Pid) of true -> - emqx_exproto_conn:call(Pid, Req); + emqx_gateway_conn:call(Pid, Req); false -> {error, ?RESP_CONN_PROCESS_NOT_ALIVE, <<"Connection process is not alive">>} diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl new file mode 100644 index 000000000..b55d30729 --- /dev/null +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl @@ -0,0 +1,219 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The ExProto Gateway Implement interface +-module(emqx_exproto_impl). + +-behavior(emqx_gateway_impl). + +%% APIs +-export([ load/0 + , unload/0 + ]). + +-export([]). + +-export([ init/1 + , on_insta_create/3 + , on_insta_update/4 + , on_insta_destroy/3 + ]). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +load() -> + RegistryOptions = [ {cbkmod, ?MODULE} + ], + emqx_gateway_registry:load(exproto, RegistryOptions, []). + +unload() -> + emqx_gateway_registry:unload(exproto). + +init(_) -> + GwState = #{}, + {ok, GwState}. + + +%%-------------------------------------------------------------------- +%% emqx_gateway_registry callbacks +%%-------------------------------------------------------------------- + +start_grpc_server(InstaId, Options = #{bind := ListenOn}) -> + Services = #{protos => [emqx_exproto_pb], + services => #{ + 'emqx.exproto.v1.ConnectionAdapter' => emqx_exproto_gsvr} + }, + SvrOptions = case maps:to_list(maps:get(ssl, Options, #{})) of + [] -> []; + SslOpts -> + [{ssl_options, SslOpts}] + end, + _ = grpc:start_server(InstaId, ListenOn, Services, SvrOptions), + io:format("Start ~s gRPC server on ~p successfully.~n", + [InstaId, ListenOn]). + +start_grpc_client_channel(InstaId, Options = #{address := UriStr}) -> + UriMap = uri_string:parse(UriStr), + Scheme = maps:get(scheme, UriMap), + Host = maps:get(host, UriMap), + Port = maps:get(port, UriMap), + SvrAddr = lists:flatten( + io_lib:format( + "~s://~s:~w", [Scheme, Host, Port]) + ), + ClientOpts = case Scheme of + "https" -> + SslOpts = maps:to_list(maps:get(ssl, Options, #{})), + #{gun_opts => + #{transport => ssl, + transport_opts => SslOpts}}; + _ -> #{} + end, + grpc_client_sup:create_channel_pool(InstaId, SvrAddr, ClientOpts). + +on_insta_create(_Insta = #{ id := InstaId, + rawconf := RawConf + }, Ctx, _GwState) -> + %% XXX: How to monitor it ? + %% Start grpc client pool & client channel + PoolName = pool_name(InstaId), + PoolSize = emqx_vm:schedulers() * 2, + {ok, _} = emqx_pool_sup:start_link(PoolName, hash, PoolSize, + {emqx_exproto_gcli, start_link, []}), + _ = start_grpc_client_channel(InstaId, maps:get(handler, RawConf)), + + %% XXX: How to monitor it ? + _ = start_grpc_server(InstaId, maps:get(server, RawConf)), + + NRawConf = maps:without( + [server, handler], + RawConf#{pool_name => PoolName} + ), + Listeners = emqx_gateway_utils:normalize_rawconf( + NRawConf#{handler => InstaId} + ), + ListenerPids = lists:map(fun(Lis) -> + start_listener(InstaId, Ctx, Lis) + end, Listeners), + {ok, ListenerPids, _InstaState = #{ctx => Ctx}}. + +on_insta_update(NewInsta, OldInsta, GwInstaState = #{ctx := Ctx}, GwState) -> + InstaId = maps:get(id, NewInsta), + try + %% XXX: 1. How hot-upgrade the changes ??? + %% XXX: 2. Check the New confs first before destroy old instance ??? + on_insta_destroy(OldInsta, GwInstaState, GwState), + on_insta_create(NewInsta, Ctx, GwState) + catch + Class : Reason : Stk -> + logger:error("Failed to update exproto instance ~s; " + "reason: {~0p, ~0p} stacktrace: ~0p", + [InstaId, Class, Reason, Stk]), + {error, {Class, Reason}} + end. + +on_insta_destroy(_Insta = #{ id := InstaId, + rawconf := RawConf + }, _GwInstaState, _GwState) -> + Listeners = emqx_gateway_utils:normalize_rawconf(RawConf), + lists:foreach(fun(Lis) -> + stop_listener(InstaId, Lis) + end, Listeners). + +pool_name(InstaId) -> + list_to_atom(lists:concat([InstaId, "_gcli_pool"])). + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +start_listener(InstaId, Ctx, {Type, ListenOn, SocketOpts, Cfg}) -> + ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), + case start_listener(InstaId, Ctx, Type, ListenOn, SocketOpts, Cfg) of + {ok, Pid} -> + io:format("Start exproto ~s:~s listener on ~s successfully.~n", + [InstaId, Type, ListenOnStr]), + Pid; + {error, Reason} -> + io:format(standard_error, + "Failed to start exproto ~s:~s listener on ~s: ~0p~n", + [InstaId, Type, ListenOnStr, Reason]), + throw({badconf, Reason}) + end. + +start_listener(InstaId, Ctx, Type, ListenOn, SocketOpts, Cfg) -> + Name = name(InstaId, Type), + NCfg = Cfg#{ + ctx => Ctx, + frame_mod => emqx_exproto_frame, + chann_mod => emqx_exproto_channel + }, + MFA = {emqx_gateway_conn, start_link, [NCfg]}, + NSockOpts = merge_default_by_type(Type, SocketOpts), + do_start_listener(Type, Name, ListenOn, NSockOpts, MFA). + +do_start_listener(Type, Name, ListenOn, Opts, MFA) + when Type == tcp; + Type == ssl -> + esockd:open(Name, ListenOn, Opts, MFA); +do_start_listener(udp, Name, ListenOn, Opts, MFA) -> + esockd:open_udp(Name, ListenOn, Opts, MFA); +do_start_listener(dtls, Name, ListenOn, Opts, MFA) -> + esockd:open_dtls(Name, ListenOn, Opts, MFA). + +name(InstaId, Type) -> + list_to_atom(lists:concat([InstaId, ":", Type])). + +merge_default_by_type(Type, Options) when Type =:= tcp; + Type =:= ssl -> + Default = emqx_gateway_utils:default_tcp_options(), + case lists:keytake(tcp_options, 1, Options) of + {value, {tcp_options, TcpOpts}, Options1} -> + [{tcp_options, emqx_misc:merge_opts(Default, TcpOpts)} + | Options1]; + false -> + [{tcp_options, Default} | Options] + end; +merge_default_by_type(Type, Options) when Type =:= udp; + Type =:= dtls -> + Default = emqx_gateway_utils:default_udp_options(), + case lists:keytake(udp_options, 1, Options) of + {value, {udp_options, TcpOpts}, Options1} -> + [{udp_options, emqx_misc:merge_opts(Default, TcpOpts)} + | Options1]; + false -> + [{udp_options, Default} | Options] + end. + +stop_listener(InstaId, {Type, ListenOn, SocketOpts, Cfg}) -> + StopRet = stop_listener(InstaId, Type, ListenOn, SocketOpts, Cfg), + ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), + case StopRet of + ok -> io:format("Stop exproto ~s:~s listener on ~s successfully.~n", + [InstaId, Type, ListenOnStr]); + {error, Reason} -> + io:format(standard_error, + "Failed to stop exproto ~s:~s listener on ~s: ~0p~n", + [InstaId, Type, ListenOnStr, Reason] + ) + end, + StopRet. + +stop_listener(InstaId, Type, ListenOn, _SocketOpts, _Cfg) -> + Name = name(InstaId, Type), + esockd:close(Name, ListenOn). diff --git a/apps/emqx_exproto/include/emqx_exproto.hrl b/apps/emqx_gateway/src/exproto/include/emqx_exproto.hrl similarity index 99% rename from apps/emqx_exproto/include/emqx_exproto.hrl rename to apps/emqx_gateway/src/exproto/include/emqx_exproto.hrl index a599b3039..75adfb6d1 100644 --- a/apps/emqx_exproto/include/emqx_exproto.hrl +++ b/apps/emqx_gateway/src/exproto/include/emqx_exproto.hrl @@ -19,7 +19,6 @@ -define(TCP_SOCKOPTS, [binary, {packet, raw}, {reuseaddr, true}, {backlog, 512}, {nodelay, true}]). -%% TODO: -define(UDP_SOCKOPTS, []). %%-------------------------------------------------------------------- diff --git a/apps/emqx_exproto/priv/protos/exproto.proto b/apps/emqx_gateway/src/exproto/protos/exproto.proto similarity index 100% rename from apps/emqx_exproto/priv/protos/exproto.proto rename to apps/emqx_gateway/src/exproto/protos/exproto.proto diff --git a/apps/emqx_lwm2m/.gitignore b/apps/emqx_gateway/src/lwm2m/.gitignore similarity index 100% rename from apps/emqx_lwm2m/.gitignore rename to apps/emqx_gateway/src/lwm2m/.gitignore diff --git a/apps/emqx_lwm2m/README.md b/apps/emqx_gateway/src/lwm2m/README.md similarity index 100% rename from apps/emqx_lwm2m/README.md rename to apps/emqx_gateway/src/lwm2m/README.md diff --git a/apps/emqx_lwm2m/src/binary_util.erl b/apps/emqx_gateway/src/lwm2m/binary_util.erl similarity index 100% rename from apps/emqx_lwm2m/src/binary_util.erl rename to apps/emqx_gateway/src/lwm2m/binary_util.erl diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m.app.src similarity index 100% rename from apps/emqx_lwm2m/src/emqx_lwm2m.app.src rename to apps/emqx_gateway/src/lwm2m/emqx_lwm2m.app.src diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl similarity index 99% rename from apps/emqx_lwm2m/src/emqx_lwm2m_api.erl rename to apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl index 6018aa7c7..80449238c 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl @@ -16,8 +16,6 @@ -module(emqx_lwm2m_api). --import(minirest, [return/1]). - -rest_api(#{name => list, method => 'GET', path => "/lwm2m_channels/", @@ -160,3 +158,7 @@ path_list(Path) -> [ObjId, ObjInsId] -> [ObjId, ObjInsId]; [ObjId] -> [ObjId] end. + +return(_) -> +%% TODO: V5 API + ok. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_app.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_app.erl similarity index 96% rename from apps/emqx_lwm2m/src/emqx_lwm2m_app.erl rename to apps/emqx_gateway/src/lwm2m/emqx_lwm2m_app.erl index 14d042681..46f29c208 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_app.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_app.erl @@ -25,8 +25,7 @@ , prep_stop/1 ]). --include("emqx_lwm2m.hrl"). - +-include("src/lwm2m/include/emqx_lwm2m.hrl"). start(_Type, _Args) -> Pid = emqx_lwm2m_sup:start_link(), diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_cm.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cm.erl similarity index 100% rename from apps/emqx_lwm2m/src/emqx_lwm2m_cm.erl rename to apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cm.erl diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_cm_sup.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cm_sup.erl similarity index 100% rename from apps/emqx_lwm2m/src/emqx_lwm2m_cm_sup.erl rename to apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cm_sup.erl diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd_handler.erl similarity index 99% rename from apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl rename to apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd_handler.erl index b3251a275..318328e3c 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd_handler.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_cmd_handler). --include("emqx_lwm2m.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). -include_lib("lwm2m_coap/include/coap.hrl"). diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_resource.erl similarity index 99% rename from apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl rename to apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_resource.erl index f94c2bc72..8a7b41291 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_resource.erl @@ -22,7 +22,7 @@ -include_lib("lwm2m_coap/include/coap.hrl"). --behaviour(lwm2m_coap_resource). +% -behaviour(lwm2m_coap_resource). -export([ coap_discover/2 , coap_get/5 @@ -41,7 +41,7 @@ -export([parse_object_list/1]). --include("emqx_lwm2m.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). -define(PREFIX, <<"rd">>). diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_coap_server.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_server.erl similarity index 98% rename from apps/emqx_lwm2m/src/emqx_lwm2m_coap_server.erl rename to apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_server.erl index 60d5f4b85..7b6aa86af 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_coap_server.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_server.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_coap_server). --include("emqx_lwm2m.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). -export([ start/1 , stop/1 diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_json.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_json.erl similarity index 99% rename from apps/emqx_lwm2m/src/emqx_lwm2m_json.erl rename to apps/emqx_gateway/src/lwm2m/emqx_lwm2m_json.erl index 641cf7d97..295c68085 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_json.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_json.erl @@ -22,7 +22,7 @@ , opaque_to_json/2 ]). --include("emqx_lwm2m.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). -define(LOG(Level, Format, Args), logger:Level("LWM2M-JSON: " ++ Format, Args)). diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl similarity index 99% rename from apps/emqx_lwm2m/src/emqx_lwm2m_message.erl rename to apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl index 6b8bc8d50..6d155f9bd 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl @@ -23,7 +23,7 @@ , translate_json/1 ]). --include("emqx_lwm2m.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). -define(LOG(Level, Format, Args), logger:Level("LWM2M-JSON: " ++ Format, Args)). diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl similarity index 98% rename from apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl rename to apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl index 34c72dcca..e13b19e0a 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_protocol). --include("emqx_lwm2m.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). -include_lib("emqx/include/emqx.hrl"). @@ -86,15 +86,14 @@ init(CoapPid, EndpointName, Peername = {_Peerhost, _Port}, RegInfo = #{<<"lt">> ClientInfo = clientinfo(Lwm2mState), _ = run_hooks('client.connect', [conninfo(Lwm2mState)], undefined), case emqx_access_control:authenticate(ClientInfo) of - {ok, AuthResult} -> + ok -> _ = run_hooks('client.connack', [conninfo(Lwm2mState), success], undefined), - ClientInfo1 = maps:merge(ClientInfo, AuthResult), Sockport = proplists:get_value(port, lwm2m_coap_responder:options(), 5683), - ClientInfo2 = maps:put(sockport, Sockport, ClientInfo1), + ClientInfo1 = maps:put(sockport, Sockport, ClientInfo), Lwm2mState1 = Lwm2mState#lwm2m_state{started_at = time_now(), - mountpoint = maps:get(mountpoint, ClientInfo2)}, - run_hooks('client.connected', [ClientInfo2, conninfo(Lwm2mState1)]), + mountpoint = maps:get(mountpoint, ClientInfo1)}, + run_hooks('client.connected', [ClientInfo1, conninfo(Lwm2mState1)]), erlang:send(CoapPid, post_init), erlang:send_after(2000, CoapPid, auto_observe), @@ -103,7 +102,7 @@ init(CoapPid, EndpointName, Peername = {_Peerhost, _Port}, RegInfo = #{<<"lt">> emqx_cm:register_channel(EndpointName, CoapPid, conninfo(Lwm2mState1)) end), emqx_cm:insert_channel_info(EndpointName, info(Lwm2mState1), stats(Lwm2mState1)), - emqx_lwm2m_cm:register_channel(EndpointName, RegInfo, LifeTime, Ver, Peername), + emqx_lwm2m_cm:register_channel(EndpointName, RegInfo, LifeTime, Ver, Peername), {ok, Lwm2mState1#lwm2m_state{life_timer = emqx_lwm2m_timer:start_timer(LifeTime, {life_timer, expired})}}; {error, Error} -> @@ -441,7 +440,8 @@ take_place(Text, Placeholder, Value) -> clientinfo(#lwm2m_state{peername = {PeerHost, _}, endpoint_name = EndpointName, mountpoint = Mountpoint}) -> - #{zone => undefined, + #{zone => default, + listener => mqtt_tcp, %% FIXME: this won't work protocol => lwm2m, peerhost => PeerHost, sockport => 5683, %% FIXME: diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_sup.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_sup.erl similarity index 100% rename from apps/emqx_lwm2m/src/emqx_lwm2m_sup.erl rename to apps/emqx_gateway/src/lwm2m/emqx_lwm2m_sup.erl diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_timer.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_timer.erl similarity index 97% rename from apps/emqx_lwm2m/src/emqx_lwm2m_timer.erl rename to apps/emqx_gateway/src/lwm2m/emqx_lwm2m_timer.erl index 75ab2d42a..b86000292 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_timer.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_timer.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_timer). --include("emqx_lwm2m.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). -export([ cancel_timer/1 , start_timer/2 diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl similarity index 99% rename from apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl rename to apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl index 8576595f8..dd1ecddda 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl @@ -25,7 +25,7 @@ -export([binary_to_hex_string/1]). -endif. --include("emqx_lwm2m.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). -define(LOG(Level, Format, Args), logger:Level("LWM2M-TLV: " ++ Format, Args)). diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl similarity index 98% rename from apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl rename to apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl index dd9911407..96a80735f 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_xml_object). --include("emqx_lwm2m.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). -include_lib("xmerl/include/xmerl.hrl"). -export([ get_obj_def/2 diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl similarity index 94% rename from apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl rename to apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl index 012d0a649..c7e68d281 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_xml_object_db). --include("emqx_lwm2m.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). -include_lib("xmerl/include/xmerl.hrl"). % This module is for future use. Disabled now. @@ -88,10 +88,7 @@ stop() -> init([]) -> _ = ets:new(?LWM2M_OBJECT_DEF_TAB, [set, named_table, protected]), _ = ets:new(?LWM2M_OBJECT_NAME_TO_ID_TAB, [set, named_table, protected]), - PluginsEtcDir = emqx:get_env(plugins_etc_dir), - DefBaseDir = re:replace(PluginsEtcDir, "plugins", "lwm2m_xml", [{return, list}]), - BaseDir = application:get_env(emqx_lwm2m, xml_dir, DefBaseDir), - load(BaseDir), + load(emqx_config:get([emqx_lwm2m, xml_dir])), {ok, #state{}}. handle_call(_Request, _From, State) -> diff --git a/apps/emqx_lwm2m/include/emqx_lwm2m.hrl b/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl similarity index 100% rename from apps/emqx_lwm2m/include/emqx_lwm2m.hrl rename to apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl diff --git a/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml b/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml similarity index 94% rename from apps/emqx_lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml rename to apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml index 80236b632..1f929cd98 100644 --- a/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml +++ b/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml @@ -80,7 +80,7 @@ LEGAL DISCLAIMER Integer 1-65534 - + Object Instance ID @@ -93,7 +93,7 @@ LEGAL DISCLAIMER - ACL + Authorization RW Multiple Optional @@ -101,7 +101,7 @@ LEGAL DISCLAIMER 16-bit +% [ {group, test_grp_0_register} +% , {group, test_grp_1_read} +% , {group, test_grp_2_write} +% , {group, test_grp_3_execute} +% , {group, test_grp_4_discover} +% , {group, test_grp_5_write_attr} +% , {group, test_grp_6_observe} +% , {group, test_grp_8_object_19} +% ]. + +% suite() -> [{timetrap, {seconds, 90}}]. + +% groups() -> +% RepeatOpt = {repeat_until_all_ok, 1}, +% [ +% {test_grp_0_register, [RepeatOpt], [ +% case01_register, +% case01_register_additional_opts, +% case01_register_incorrect_opts, +% case01_register_report, +% case02_update_deregister, +% case03_register_wrong_version, +% case04_register_and_lifetime_timeout, +% case05_register_wrong_epn, +% case06_register_wrong_lifetime, +% case07_register_alternate_path_01, +% case07_register_alternate_path_02, +% case08_reregister +% ]}, +% {test_grp_1_read, [RepeatOpt], [ +% case10_read, +% case10_read_separate_ack, +% case11_read_object_tlv, +% case11_read_object_json, +% case12_read_resource_opaque, +% case13_read_no_xml +% ]}, +% {test_grp_2_write, [RepeatOpt], [ +% case20_write, +% case21_write_object, +% case22_write_error, +% case20_single_write +% ]}, +% {test_grp_create, [RepeatOpt], [ +% case_create_basic +% ]}, +% {test_grp_delete, [RepeatOpt], [ +% case_delete_basic +% ]}, +% {test_grp_3_execute, [RepeatOpt], [ +% case30_execute, case31_execute_error +% ]}, +% {test_grp_4_discover, [RepeatOpt], [ +% case40_discover +% ]}, +% {test_grp_5_write_attr, [RepeatOpt], [ +% case50_write_attribute +% ]}, +% {test_grp_6_observe, [RepeatOpt], [ +% case60_observe +% ]}, +% {test_grp_7_block_wize_transfer, [RepeatOpt], [ +% case70_read_large, case70_write_large +% ]}, +% {test_grp_8_object_19, [RepeatOpt], [ +% case80_specail_object_19_1_0_write, +% case80_specail_object_19_0_0_notify +% %case80_specail_object_19_0_0_response, +% %case80_normal_object_19_0_0_read +% ]}, +% {test_grp_9_psm_queue_mode, [RepeatOpt], [ +% case90_psm_mode, +% case90_queue_mode +% ]} +% ]. + +% init_per_suite(Config) -> +% emqx_ct_helpers:start_apps([emqx]), +% Config. + +% end_per_suite(Config) -> +% timer:sleep(300), +% emqx_ct_helpers:stop_apps([emqx]), +% Config. + +% init_per_testcase(_AllTestCase, Config) -> +% application:set_env(emqx_lwm2m, bind_udp, [{5683, []}]), +% application:set_env(emqx_lwm2m, bind_dtls, [{5684, []}]), +% application:set_env(emqx_lwm2m, xml_dir, emqx_ct_helpers:deps_path(emqx_lwm2m, "lwm2m_xml")), +% application:set_env(emqx_lwm2m, lifetime_max, 86400), +% application:set_env(emqx_lwm2m, lifetime_min, 1), +% application:set_env(emqx_lwm2m, mountpoint, "lwm2m/%e/"), +% {ok, _Started} = application:ensure_all_started(emqx_lwm2m), +% {ok, ClientUdpSock} = gen_udp:open(0, [binary, {active, false}]), + +% {ok, C} = emqtt:start_link([{host, "localhost"},{port, 1883},{clientid, <<"c1">>}]), +% {ok, _} = emqtt:connect(C), +% timer:sleep(100), + +% [{sock, ClientUdpSock}, {emqx_c, C} | Config]. + +% end_per_testcase(_AllTestCase, Config) -> +% timer:sleep(300), +% gen_udp:close(?config(sock, Config)), +% emqtt:disconnect(?config(emqx_c, Config)), +% ok = application:stop(emqx_lwm2m), +% ok = application:stop(lwm2m_coap). + +% %%-------------------------------------------------------------------- +% %% Cases +% %%-------------------------------------------------------------------- + +% case01_register(Config) -> +% % ---------------------------------------- +% % REGISTER command +% % ---------------------------------------- +% UdpSock = ?config(sock, Config), +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId = 12, +% SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), + +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), +% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, +% [], +% MsgId), + +% %% checkpoint 1 - response +% #coap_message{type = Type, method = Method, id = RspId, options = Opts} = +% test_recv_coap_response(UdpSock), +% ack = Type, +% {ok, created} = Method, +% RspId = MsgId, +% Location = proplists:get_value(location_path, Opts), +% ?assertNotEqual(undefined, Location), + +% %% checkpoint 2 - verify subscribed topics +% timer:sleep(50), +% ?LOGT("all topics: ~p", [test_mqtt_broker:get_subscrbied_topics()]), +% true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), + +% % ---------------------------------------- +% % DE-REGISTER command +% % ---------------------------------------- +% ?LOGT("start to send DE-REGISTER command", []), +% MsgId3 = 52, +% test_send_coap_request( UdpSock, +% delete, +% sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]), +% #coap_content{payload = <<>>}, +% [], +% MsgId3), +% #coap_message{type = ack, id = RspId3, method = Method3} = test_recv_coap_response(UdpSock), +% {ok,deleted} = Method3, +% MsgId3 = RspId3, +% timer:sleep(50), +% false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). + +% case01_register_additional_opts(Config) -> +% % ---------------------------------------- +% % REGISTER command +% % ---------------------------------------- +% UdpSock = ?config(sock, Config), +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId = 12, +% SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), + +% AddOpts = "ep=~s<=345&lwm2m=1&apn=psmA.eDRX0.ctnb&cust_opt=shawn&im=123&ct=1.4&mt=mdm9620&mv=1.2", +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b/rd?" ++ AddOpts, [?PORT, Epn]), +% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, +% [], +% MsgId), + +% %% checkpoint 1 - response +% #coap_message{type = Type, method = Method, id = RspId, options = Opts} = +% test_recv_coap_response(UdpSock), +% Type = ack, +% Method = {ok, created}, +% RspId = MsgId, +% Location = proplists:get_value(location_path, Opts), +% ?assertNotEqual(undefined, Location), + +% %% checkpoint 2 - verify subscribed topics +% timer:sleep(50), + +% true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), + +% % ---------------------------------------- +% % DE-REGISTER command +% % ---------------------------------------- +% ?LOGT("start to send DE-REGISTER command", []), +% MsgId3 = 52, +% test_send_coap_request( UdpSock, +% delete, +% sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]), +% #coap_content{payload = <<>>}, +% [], +% MsgId3), +% #coap_message{type = ack, id = RspId3, method = Method3} = test_recv_coap_response(UdpSock), +% {ok,deleted} = Method3, +% MsgId3 = RspId3, +% timer:sleep(50), +% false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). + +% case01_register_incorrect_opts(Config) -> +% % ---------------------------------------- +% % REGISTER command +% % ---------------------------------------- +% UdpSock = ?config(sock, Config), +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId = 12, + + +% AddOpts = "ep=~s<=345&lwm2m=1&incorrect_opt", +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b/rd?" ++ AddOpts, [?PORT, Epn]), +% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, +% [], +% MsgId), + +% %% checkpoint 1 - response +% #coap_message{type = ack, method = Method, id = MsgId} = +% test_recv_coap_response(UdpSock), +% ?assertEqual({error,bad_request}, Method). + +% case01_register_report(Config) -> +% % ---------------------------------------- +% % REGISTER command +% % ---------------------------------------- +% UdpSock = ?config(sock, Config), +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId = 12, +% SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), +% ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0), +% timer:sleep(200), + +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), +% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, +% [], +% MsgId), + +% #coap_message{type = Type, method = Method, id = RspId, options = Opts} = +% test_recv_coap_response(UdpSock), +% Type = ack, +% Method = {ok, created}, +% RspId = MsgId, +% Location = proplists:get_value(location_path, Opts), +% ?assertNotEqual(undefined, Location), + +% timer:sleep(50), +% true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), + +% ReadResult = emqx_json:encode(#{ +% <<"msgType">> => <<"register">>, +% <<"data">> => #{ +% <<"alternatePath">> => <<"/">>, +% <<"ep">> => list_to_binary(Epn), +% <<"lt">> => 345, +% <<"lwm2m">> => <<"1">>, +% <<"objectList">> => [<<"/1">>, <<"/2">>, <<"/3">>, <<"/4">>, <<"/5">>] +% } +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(ReportTopic)), + +% % ---------------------------------------- +% % DE-REGISTER command +% % ---------------------------------------- +% ?LOGT("start to send DE-REGISTER command", []), +% MsgId3 = 52, +% test_send_coap_request( UdpSock, +% delete, +% sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]), +% #coap_content{payload = <<>>}, +% [], +% MsgId3), +% #coap_message{type = ack, id = RspId3, method = Method3} = test_recv_coap_response(UdpSock), +% {ok,deleted} = Method3, +% MsgId3 = RspId3, +% timer:sleep(50), +% false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). + +% case02_update_deregister(Config) -> +% % ---------------------------------------- +% % REGISTER command +% % ---------------------------------------- +% UdpSock = ?config(sock, Config), +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId = 12, +% SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), +% ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0), +% timer:sleep(200), + +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), +% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, +% [], +% MsgId), +% timer:sleep(100), +% #coap_message{type = ack, method = Method, options = Opts} = test_recv_coap_response(UdpSock), +% ?assertEqual({ok,created}, Method), + +% ?LOGT("Options got: ~p", [Opts]), +% Location = proplists:get_value(location_path, Opts), +% Register = emqx_json:encode(#{ +% <<"msgType">> => <<"register">>, +% <<"data">> => #{ +% <<"alternatePath">> => <<"/">>, +% <<"ep">> => list_to_binary(Epn), +% <<"lt">> => 345, +% <<"lwm2m">> => <<"1">>, +% <<"objectList">> => [<<"/1">>, <<"/2">>, <<"/3">>, <<"/4">>, <<"/5">>] +% } +% }), +% ?assertEqual(Register, test_recv_mqtt_response(ReportTopic)), + +% % ---------------------------------------- +% % UPDATE command +% % ---------------------------------------- +% ?LOGT("start to send UPDATE command", []), +% MsgId2 = 27, +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b~s?lt=789", [?PORT, join_path(Location, <<>>)]), +% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , , ">>}, +% [], +% MsgId2), +% #coap_message{type = ack, id = RspId2, method = Method2} = test_recv_coap_response(UdpSock), +% {ok,changed} = Method2, +% MsgId2 = RspId2, +% Update = emqx_json:encode(#{ +% <<"msgType">> => <<"update">>, +% <<"data">> => #{ +% <<"alternatePath">> => <<"/">>, +% <<"ep">> => list_to_binary(Epn), +% <<"lt">> => 789, +% <<"lwm2m">> => <<"1">>, +% <<"objectList">> => [<<"/1">>, <<"/2">>, <<"/3">>, <<"/4">>, <<"/5">>, <<"/6">>] +% } +% }), +% ?assertEqual(Update, test_recv_mqtt_response(ReportTopic)), + +% % ---------------------------------------- +% % DE-REGISTER command +% % ---------------------------------------- +% ?LOGT("start to send DE-REGISTER command", []), +% MsgId3 = 52, +% test_send_coap_request( UdpSock, +% delete, +% sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]), +% #coap_content{payload = <<>>}, +% [], +% MsgId3), +% #coap_message{type = ack, id = RspId3, method = Method3} = test_recv_coap_response(UdpSock), +% {ok,deleted} = Method3, +% MsgId3 = RspId3, + +% timer:sleep(50), +% false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). + +% case03_register_wrong_version(Config) -> +% % ---------------------------------------- +% % REGISTER command +% % ---------------------------------------- +% UdpSock = ?config(sock, Config), +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId = 12, +% SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=8.3", [?PORT, Epn]), +% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, +% [], +% MsgId), +% #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock), +% ?assertEqual({error,precondition_failed}, Method), +% timer:sleep(50), + +% false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). + +% case04_register_and_lifetime_timeout(Config) -> +% % ---------------------------------------- +% % REGISTER command +% % ---------------------------------------- +% UdpSock = ?config(sock, Config), +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId = 12, +% SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), + +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b/rd?ep=~s<=2&lwm2m=1", [?PORT, Epn]), +% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, +% [], +% MsgId), +% timer:sleep(100), +% #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock), +% ?assertEqual({ok,created}, Method), + +% true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), + +% % ---------------------------------------- +% % lifetime timeout +% % ---------------------------------------- +% timer:sleep(4000), + +% false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). + +% case05_register_wrong_epn(Config) -> +% % ---------------------------------------- +% % REGISTER command +% % ---------------------------------------- +% MsgId = 12, +% UdpSock = ?config(sock, Config), + +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b/rd?lt=345&lwm2m=1.0", [?PORT]), +% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, +% [], +% MsgId), +% #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock), +% ?assertEqual({error,bad_request}, Method). + +% case06_register_wrong_lifetime(Config) -> +% % ---------------------------------------- +% % REGISTER command +% % ---------------------------------------- +% UdpSock = ?config(sock, Config), +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId = 12, + +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b/rd?ep=~s&lwm2m=1", [?PORT, Epn]), +% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, +% [], +% MsgId), +% #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock), +% ?assertEqual({error,bad_request}, Method), +% timer:sleep(50), +% ?assertEqual([], test_mqtt_broker:get_subscrbied_topics()). + +% case07_register_alternate_path_01(Config) -> +% % ---------------------------------------- +% % REGISTER command +% % ---------------------------------------- +% UdpSock = ?config(sock, Config), +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId = 12, +% SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), +% ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0), +% timer:sleep(200), + +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), +% #coap_content{content_format = <<"text/plain">>, +% payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, +% [], +% MsgId), +% timer:sleep(50), +% true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). + +% case07_register_alternate_path_02(Config) -> +% % ---------------------------------------- +% % REGISTER command +% % ---------------------------------------- +% UdpSock = ?config(sock, Config), +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId = 12, +% SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), +% ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0), +% timer:sleep(200), + +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), +% #coap_content{content_format = <<"text/plain">>, +% payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, +% [], +% MsgId), +% timer:sleep(50), +% true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). + +% case08_reregister(Config) -> +% % ---------------------------------------- +% % REGISTER command +% % ---------------------------------------- +% UdpSock = ?config(sock, Config), +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId = 12, +% SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), +% ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0), +% timer:sleep(200), + +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), +% #coap_content{content_format = <<"text/plain">>, +% payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, +% [], +% MsgId), +% timer:sleep(50), +% true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), + +% ReadResult = emqx_json:encode( +% #{ +% <<"msgType">> => <<"register">>, +% <<"data">> => #{ +% <<"alternatePath">> => <<"/lwm2m">>, +% <<"ep">> => list_to_binary(Epn), +% <<"lt">> => 345, +% <<"lwm2m">> => <<"1">>, +% <<"objectList">> => [<<"/1/0">>, <<"/2/0">>, <<"/3/0">>] +% } +% } +% ), +% ?assertEqual(ReadResult, test_recv_mqtt_response(ReportTopic)), +% timer:sleep(1000), + +% %% the same lwm2mc client registers to server again +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), +% #coap_content{content_format = <<"text/plain">>, +% payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, +% [], +% MsgId + 1), +% %% verify the lwm2m client is still online +% ?assertEqual(ReadResult, test_recv_mqtt_response(ReportTopic)). + +% case10_read(Config) -> +% UdpSock = ?config(sock, Config), +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId1 = 15, +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), +% % step 1, device register ... +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), +% #coap_content{content_format = <<"text/plain">>, +% payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, +% [], +% MsgId1), +% #coap_message{method = Method1} = test_recv_coap_response(UdpSock), +% ?assertEqual({ok,created}, Method1), +% test_recv_mqtt_response(RespTopic), + +% % step2, send a READ command to device +% CmdId = 206, +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% Command = #{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"read">>, +% <<"data">> => #{ +% <<"path">> => <<"/3/0/0">> +% } +% }, +% CommandJson = emqx_json:encode(Command), +% ?LOGT("CommandJson=~p", [CommandJson]), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, +% ?LOGT("LwM2M client got ~p", [Request2]), + +% ?assertEqual(get, Method2), +% ?assertEqual(<<"/lwm2m/3/0/0">>, get_coap_path(Options2)), +% ?assertEqual(<<>>, Payload2), +% timer:sleep(50), + +% test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request2, true), +% timer:sleep(100), + +% ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"read">>, +% <<"data">> => #{ +% <<"code">> => <<"2.05">>, +% <<"codeMsg">> => <<"content">>, +% <<"reqPath">> => <<"/3/0/0">>, +% <<"content">> => [#{ +% path => <<"/3/0/0">>, +% value => <<"EMQ">> +% }] +% } +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + +% case10_read_separate_ack(Config) -> +% UdpSock = ?config(sock, Config), +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId1 = 15, +% ObjectList = <<", , , , ">>, +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), + +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), + +% % step 1, device register ... +% std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), + +% % step2, send a READ command to device +% CmdId = 206, +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% Command = #{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"read">>, +% <<"data">> => #{ +% <<"path">> => <<"/3/0/0">> +% } +% }, +% CommandJson = emqx_json:encode(Command), +% ?LOGT("CommandJson=~p", [CommandJson]), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, +% ?LOGT("LwM2M client got ~p", [Request2]), + +% ?assertEqual(get, Method2), +% ?assertEqual(<<"/3/0/0">>, get_coap_path(Options2)), +% ?assertEqual(<<>>, Payload2), + +% test_send_empty_ack(UdpSock, "127.0.0.1", ?PORT, Request2), +% ReadResultACK = emqx_json:encode(#{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"ack">>, +% <<"data">> => #{ +% <<"path">> => <<"/3/0/0">> +% } +% }), +% ?assertEqual(ReadResultACK, test_recv_mqtt_response(RespTopic)), +% timer:sleep(100), + +% test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request2, false), +% timer:sleep(100), + +% ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"read">>, +% <<"data">> => #{ +% <<"code">> => <<"2.05">>, +% <<"codeMsg">> => <<"content">>, +% <<"reqPath">> => <<"/3/0/0">>, +% <<"content">> => [#{ +% path => <<"/3/0/0">>, +% value => <<"EMQ">> +% }] +% } +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + +% case11_read_object_tlv(Config) -> +% % step 1, device register ... +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId1 = 15, +% UdpSock = ?config(sock, Config), +% ObjectList = <<", , , , ">>, +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), + +% std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), + +% % step2, send a READ command to device +% CmdId = 207, +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% Command = #{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"read">>, +% <<"data">> => #{ +% <<"path">> => <<"/3/0">> +% } +% }, +% CommandJson = emqx_json:encode(Command), +% ?LOGT("CommandJson=~p", [CommandJson]), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2} = Request2, +% ?LOGT("LwM2M client got ~p", [Request2]), + +% ?assertEqual(get, Method2), +% timer:sleep(50), + +% Tlv = <<16#08, 16#00, 16#3C, 16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33>>, +% test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"application/vnd.oma.lwm2m+tlv">>, payload = Tlv}, Request2, true), +% timer:sleep(100), + +% ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"read">>, +% <<"data">> => #{ +% <<"code">> => <<"2.05">>, +% <<"codeMsg">> => <<"content">>, +% <<"reqPath">> => <<"/3/0">>, +% <<"content">> => [ +% #{ +% path => <<"/3/0/0">>, +% value => <<"Open Mobile Alliance">> +% }, +% #{ +% path => <<"/3/0/1">>, +% value => <<"Lightweight M2M Client">> +% }, +% #{ +% path => <<"/3/0/2">>, +% value => <<"345000123">> +% } +% ] +% } +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + +% case11_read_object_json(Config) -> +% % step 1, device register ... +% UdpSock = ?config(sock, Config), +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId1 = 15, + +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% ObjectList = <<", , , , ">>, +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), + +% std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), + +% % step2, send a READ command to device +% CmdId = 206, +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% Command = #{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"read">>, +% <<"data">> => #{ +% <<"path">> => <<"/3/0">> +% } +% }, +% CommandJson = emqx_json:encode(Command), +% ?LOGT("CommandJson=~p", [CommandJson]), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2} = Request2, +% ?LOGT("LwM2M client got ~p", [Request2]), + +% ?assertEqual(get, Method2), +% timer:sleep(50), + +% Json = <<"{\"bn\":\"/3/0\",\"e\":[{\"n\":\"0\",\"sv\":\"Open Mobile Alliance\"},{\"n\":\"1\",\"sv\":\"Lightweight M2M Client\"},{\"n\":\"2\",\"sv\":\"345000123\"}]}">>, +% test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"application/vnd.oma.lwm2m+json">>, payload = Json}, Request2, true), +% timer:sleep(100), + +% ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"read">>, +% <<"data">> => #{ +% <<"code">> => <<"2.05">>, +% <<"codeMsg">> => <<"content">>, +% <<"reqPath">> => <<"/3/0">>, +% <<"content">> => [ +% #{ +% path => <<"/3/0/0">>, +% value => <<"Open Mobile Alliance">> +% }, +% #{ +% path => <<"/3/0/1">>, +% value => <<"Lightweight M2M Client">> +% }, +% #{ +% path => <<"/3/0/2">>, +% value => <<"345000123">> +% } +% ] +% } +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + +% case12_read_resource_opaque(Config) -> +% % step 1, device register ... +% UdpSock = ?config(sock, Config), +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId1 = 15, +% ObjectList = <<", , , , ">>, +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), + +% std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), + +% % step2, send a READ command to device +% CmdId = 206, +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% Command = #{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"read">>, +% <<"data">> => #{ +% <<"path">> => <<"/3/0/8">> +% } +% }, +% CommandJson = emqx_json:encode(Command), +% ?LOGT("CommandJson=~p", [CommandJson]), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2} = Request2, +% ?LOGT("LwM2M client got ~p", [Request2]), + +% ?assertEqual(get, Method2), +% timer:sleep(50), + +% Opaque = <<20, 21, 22, 23>>, +% test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"application/octet-stream">>, payload = Opaque}, Request2, true), +% timer:sleep(100), + +% ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"read">>, +% <<"data">> => #{ +% <<"code">> => <<"2.05">>, +% <<"codeMsg">> => <<"content">>, +% <<"reqPath">> => <<"/3/0/8">>, +% <<"content">> => [ +% #{ +% path => <<"/3/0/8">>, +% value => base64:encode(Opaque) +% } +% ] +% } +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + +% case13_read_no_xml(Config) -> +% % step 1, device register ... +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId1 = 15, +% UdpSock = ?config(sock, Config), +% ObjectList = <<", , , , ">>, +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), + +% std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), + +% % step2, send a READ command to device +% CmdId = 206, +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% Command = #{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"read">>, +% <<"data">> => #{ +% <<"path">> => <<"/9723/0/0">> +% } +% }, +% CommandJson = emqx_json:encode(Command), +% ?LOGT("CommandJson=~p", [CommandJson]), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2} = Request2, +% ?LOGT("LwM2M client got ~p", [Request2]), + +% ?assertEqual(get, Method2), +% timer:sleep(50), + +% test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request2, true), +% timer:sleep(100), + +% ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"read">>, +% <<"data">> => #{ +% <<"reqPath">> => <<"/9723/0/0">>, +% <<"code">> => <<"4.00">>, +% <<"codeMsg">> => <<"bad_request">> +% } +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + +% case20_single_write(Config) -> +% % step 1, device register ... +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId1 = 15, +% UdpSock = ?config(sock, Config), +% ObjectList = <<", , , , ">>, +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), + +% std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), + +% % step2, send a WRITE command to device +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% CmdId = 307, +% Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"write">>, +% <<"data">> => #{ +% <<"path">> => <<"/3/0/13">>, +% <<"type">> => <<"Integer">>, +% <<"value">> => <<"12345">> +% } +% }, +% CommandJson = emqx_json:encode(Command), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, +% Path2 = get_coap_path(Options2), +% ?assertEqual(put, Method2), +% ?assertEqual(<<"/3/0/13">>, Path2), +% Tlv_Value = <<3:2, 0:1, 0:2, 2:3, 13, 12345:16>>, +% ?assertEqual(Tlv_Value, Payload2), +% timer:sleep(50), + +% test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true), +% timer:sleep(100), + +% ReadResult = emqx_json:encode(#{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"data">> => #{ +% <<"reqPath">> => <<"/3/0/13">>, +% <<"code">> => <<"2.04">>, +% <<"codeMsg">> => <<"changed">> +% }, +% <<"msgType">> => <<"write">> +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + +% case20_write(Config) -> +% % step 1, device register ... +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId1 = 15, +% UdpSock = ?config(sock, Config), +% ObjectList = <<", , , , ">>, +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), + +% std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), + +% % step2, send a WRITE command to device +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% CmdId = 307, +% Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"write">>, +% <<"data">> => #{ +% <<"basePath">> => <<"/3/0/13">>, +% <<"content">> => [#{ +% type => <<"Float">>, +% value => <<"12345.0">> +% }] +% } +% }, +% CommandJson = emqx_json:encode(Command), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, +% Path2 = get_coap_path(Options2), +% ?assertEqual(put, Method2), +% ?assertEqual(<<"/3/0/13">>, Path2), +% Tlv_Value = <<200, 13, 8, 64,200,28,128,0,0,0,0>>, +% ?assertEqual(Tlv_Value, Payload2), +% timer:sleep(50), + +% test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true), +% timer:sleep(100), + +% WriteResult = emqx_json:encode(#{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"data">> => #{ +% <<"reqPath">> => <<"/3/0/13">>, +% <<"code">> => <<"2.04">>, +% <<"codeMsg">> => <<"changed">> +% }, +% <<"msgType">> => <<"write">> +% }), +% ?assertEqual(WriteResult, test_recv_mqtt_response(RespTopic)). + +% case21_write_object(Config) -> +% % step 1, device register ... +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId1 = 15, +% UdpSock = ?config(sock, Config), +% ObjectList = <<", , , , ">>, +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), + +% std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), + +% % step2, send a WRITE command to device +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% CmdId = 307, +% Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"write">>, +% <<"data">> => #{ +% <<"basePath">> => <<"/3/0/">>, +% <<"content">> => [#{ +% path => <<"13">>, +% type => <<"Integer">>, +% value => <<"12345">> +% },#{ +% path => <<"14">>, +% type => <<"String">>, +% value => <<"87x">> +% }] +% } +% }, +% CommandJson = emqx_json:encode(Command), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, +% Path2 = get_coap_path(Options2), +% ?assertEqual(post, Method2), +% ?assertEqual(<<"/3/0">>, Path2), +% Tlv_Value = <<3:2, 0:1, 0:2, 2:3, 13, 12345:16, +% 3:2, 0:1, 0:2, 3:3, 14, "87x">>, +% ?assertEqual(Tlv_Value, Payload2), +% timer:sleep(50), + +% test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true), +% timer:sleep(100), + + +% ReadResult = emqx_json:encode(#{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"write">>, +% <<"data">> => #{ +% <<"reqPath">> => <<"/3/0/">>, +% <<"code">> => <<"2.04">>, +% <<"codeMsg">> => <<"changed">> +% } +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + +% case22_write_error(Config) -> +% % step 1, device register ... +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId1 = 15, +% UdpSock = ?config(sock, Config), +% ObjectList = <<", , , , ">>, +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), + +% std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), + +% % step2, send a WRITE command to device +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% CmdId = 307, +% Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"write">>, +% <<"data">> => #{ +% <<"basePath">> => <<"/3/0/1">>, +% <<"content">> => [ +% #{ +% type => <<"Integer">>, +% value => <<"12345">> +% } +% ] +% } +% }, +% CommandJson = emqx_json:encode(Command), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2, options=Options2} = Request2, +% Path2 = get_coap_path(Options2), +% ?assertEqual(put, Method2), +% ?assertEqual(<<"/3/0/1">>, Path2), +% timer:sleep(50), + +% test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {error, bad_request}, #coap_content{}, Request2, true), +% timer:sleep(100), + +% ReadResult = emqx_json:encode(#{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"data">> => #{ +% <<"reqPath">> => <<"/3/0/1">>, +% <<"code">> => <<"4.00">>, +% <<"codeMsg">> => <<"bad_request">> +% }, +% <<"msgType">> => <<"write">> +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + +% case_create_basic(Config) -> +% % step 1, device register ... +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId1 = 15, +% UdpSock = ?config(sock, Config), +% ObjectList = <<", , , ">>, +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), + +% std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), + +% % step2, send a CREATE command to device +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% CmdId = 307, +% Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"create">>, +% <<"data">> => #{ +% <<"path">> => <<"/5">> +% } +% }, +% CommandJson = emqx_json:encode(Command), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, +% Path2 = get_coap_path(Options2), +% ?assertEqual(post, Method2), +% ?assertEqual(<<"/5">>, Path2), +% ?assertEqual(<<"">>, Payload2), +% timer:sleep(50), + +% test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, created}, #coap_content{}, Request2, true), +% timer:sleep(100), + +% ReadResult = emqx_json:encode(#{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"data">> => #{ +% <<"reqPath">> => <<"/5">>, +% <<"code">> => <<"2.01">>, +% <<"codeMsg">> => <<"created">> +% }, +% <<"msgType">> => <<"create">> +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + +% case_delete_basic(Config) -> +% % step 1, device register ... +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId1 = 15, +% UdpSock = ?config(sock, Config), +% ObjectList = <<", , , , , ">>, +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), + +% std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), + +% % step2, send a CREATE command to device +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% CmdId = 307, +% Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"delete">>, +% <<"data">> => #{ +% <<"path">> => <<"/5/0">> +% } +% }, +% CommandJson = emqx_json:encode(Command), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, +% Path2 = get_coap_path(Options2), +% ?assertEqual(delete, Method2), +% ?assertEqual(<<"/5/0">>, Path2), +% ?assertEqual(<<"">>, Payload2), +% timer:sleep(50), + +% test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, deleted}, #coap_content{}, Request2, true), +% timer:sleep(100), + +% ReadResult = emqx_json:encode(#{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"data">> => #{ +% <<"reqPath">> => <<"/5/0">>, +% <<"code">> => <<"2.02">>, +% <<"codeMsg">> => <<"deleted">> +% }, +% <<"msgType">> => <<"delete">> +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + +% case30_execute(Config) -> +% % step 1, device register ... +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId1 = 15, +% UdpSock = ?config(sock, Config), +% ObjectList = <<", , , , ">>, +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), + +% std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), + +% % step2, send a WRITE command to device +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% CmdId = 307, +% Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"execute">>, +% <<"data">> => #{ +% <<"path">> => <<"/3/0/4">>, +% %% "args" should not be present for "/3/0/4", only for testing the encoding here +% <<"args">> => <<"2,7">> +% } +% }, +% CommandJson = emqx_json:encode(Command), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, +% Path2 = get_coap_path(Options2), +% ?assertEqual(post, Method2), +% ?assertEqual(<<"/3/0/4">>, Path2), +% ?assertEqual(<<"2,7">>, Payload2), +% timer:sleep(50), + +% test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true), +% timer:sleep(100), + +% ReadResult = emqx_json:encode(#{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"data">> => #{ +% <<"reqPath">> => <<"/3/0/4">>, +% <<"code">> => <<"2.04">>, +% <<"codeMsg">> => <<"changed">> +% }, +% <<"msgType">> => <<"execute">> +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + +% case31_execute_error(Config) -> +% % step 1, device register ... +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId1 = 15, +% UdpSock = ?config(sock, Config), +% ObjectList = <<", , , , ">>, +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), + +% std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), + +% % step2, send a WRITE command to device +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% CmdId = 307, +% Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"execute">>, +% <<"data">> => #{ +% <<"path">> => <<"/3/0/4">>, +% <<"args">> => <<"2,7">> +% } +% }, +% CommandJson = emqx_json:encode(Command), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, +% Path2 = get_coap_path(Options2), +% ?assertEqual(post, Method2), +% ?assertEqual(<<"/3/0/4">>, Path2), +% ?assertEqual(<<"2,7">>, Payload2), +% timer:sleep(50), + +% test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {error, uauthorized}, #coap_content{}, Request2, true), +% timer:sleep(100), + +% ReadResult = emqx_json:encode(#{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"data">> => #{ +% <<"reqPath">> => <<"/3/0/4">>, +% <<"code">> => <<"4.01">>, +% <<"codeMsg">> => <<"uauthorized">> +% }, +% <<"msgType">> => <<"execute">> +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + +% case40_discover(Config) -> +% % step 1, device register ... +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId1 = 15, +% UdpSock = ?config(sock, Config), +% ObjectList = <<", , , , ">>, +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), + +% std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), + +% % step2, send a WRITE command to device +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% CmdId = 307, +% Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"discover">>, +% <<"data">> => #{ +% <<"path">> => <<"/3/0/7">> +% } }, +% CommandJson = emqx_json:encode(Command), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, +% Path2 = get_coap_path(Options2), +% ?assertEqual(get, Method2), +% ?assertEqual(<<"/3/0/7">>, Path2), +% ?assertEqual(<<>>, Payload2), +% timer:sleep(50), + +% PayloadDiscover = <<";dim=8;pmin=10;pmax=60;gt=50;lt=42.2,">>, +% test_send_coap_response(UdpSock, +% "127.0.0.1", +% ?PORT, +% {ok, content}, +% #coap_content{content_format = <<"application/link-format">>, payload = PayloadDiscover}, +% Request2, +% true), +% timer:sleep(100), + +% ReadResult = emqx_json:encode(#{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"discover">>, +% <<"data">> => #{ +% <<"reqPath">> => <<"/3/0/7">>, +% <<"code">> => <<"2.05">>, +% <<"codeMsg">> => <<"content">>, +% <<"content">> => +% [<<";dim=8;pmin=10;pmax=60;gt=50;lt=42.2">>, <<"">>] +% } +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + +% case50_write_attribute(Config) -> +% % step 1, device register ... +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId1 = 15, +% UdpSock = ?config(sock, Config), +% ObjectList = <<", , , , ">>, +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), + +% std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), + +% % step2, send a WRITE command to device +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% CmdId = 307, +% Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"write-attr">>, +% <<"data">> => #{ +% <<"path">> => <<"/3/0/9">>, +% <<"pmin">> => <<"1">>, +% <<"pmax">> => <<"5">>, +% <<"lt">> => <<"5">> +% } }, +% CommandJson = emqx_json:encode(Command), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(100), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, +% ?LOGT("got options: ~p", [Options2]), +% Path2 = get_coap_path(Options2), +% Query2 = lists:sort(get_coap_query(Options2)), +% ?assertEqual(put, Method2), +% ?assertEqual(<<"/3/0/9">>, Path2), +% ?assertEqual(lists:sort([<<"pmax=5">>,<<"lt=5">>,<<"pmin=1">>]), Query2), +% ?assertEqual(<<>>, Payload2), +% timer:sleep(50), + +% test_send_coap_response(UdpSock, +% "127.0.0.1", +% ?PORT, +% {ok, changed}, +% #coap_content{}, +% Request2, +% true), +% timer:sleep(100), + +% ReadResult = emqx_json:encode(#{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"data">> => #{ +% <<"reqPath">> => <<"/3/0/9">>, +% <<"code">> => <<"2.04">>, +% <<"codeMsg">> => <<"changed">> +% }, +% <<"msgType">> => <<"write-attr">> +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + +% case60_observe(Config) -> +% % step 1, device register ... +% Epn = "urn:oma:lwm2m:oma:3", +% MsgId1 = 15, +% UdpSock = ?config(sock, Config), +% ObjectList = <<", , , , ">>, +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% RespTopicAD = list_to_binary("lwm2m/"++Epn++"/up/notify"), +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% emqtt:subscribe(?config(emqx_c, Config), RespTopicAD, qos0), +% timer:sleep(200), + +% std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), + +% % step2, send a OBSERVE command to device +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% CmdId = 307, +% Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"observe">>, +% <<"data">> => #{ +% <<"path">> => <<"/3/0/10">> +% } +% }, +% CommandJson = emqx_json:encode(Command), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, +% Path2 = get_coap_path(Options2), +% Observe = get_coap_observe(Options2), +% ?assertEqual(get, Method2), +% ?assertEqual(<<"/3/0/10">>, Path2), +% ?assertEqual(Observe, 0), +% ?assertEqual(<<>>, Payload2), +% timer:sleep(50), + +% test_send_coap_observe_ack( UdpSock, +% "127.0.0.1", +% ?PORT, +% {ok, content}, +% #coap_content{content_format = <<"text/plain">>, payload = <<"2048">>}, +% Request2), +% timer:sleep(100), + +% ReadResult = emqx_json:encode(#{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"observe">>, +% <<"data">> => #{ +% <<"reqPath">> => <<"/3/0/10">>, +% <<"code">> => <<"2.05">>, +% <<"codeMsg">> => <<"content">>, +% <<"content">> => [#{ +% path => <<"/3/0/10">>, +% value => 2048 +% }] +% } +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)), + +% %% step3 the notifications +% timer:sleep(200), +% ObSeq = 3, +% test_send_coap_notif( UdpSock, +% "127.0.0.1", +% ?PORT, +% #coap_content{content_format = <<"text/plain">>, payload = <<"4096">>}, +% ObSeq, +% Request2), +% timer:sleep(100), +% #coap_message{} = test_recv_coap_response(UdpSock), + +% ReadResult2 = emqx_json:encode(#{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"notify">>, +% <<"seqNum">> => ObSeq, +% <<"data">> => #{ +% <<"reqPath">> => <<"/3/0/10">>, +% <<"code">> => <<"2.05">>, +% <<"codeMsg">> => <<"content">>, +% <<"content">> => [#{ +% path => <<"/3/0/10">>, +% value => 4096 +% }] +% } +% }), +% ?assertEqual(ReadResult2, test_recv_mqtt_response(RespTopicAD)), + +% %% Step3. cancel observe +% CmdId3 = 308, +% Command3 = #{<<"requestID">> => CmdId3, <<"cacheID">> => CmdId3, +% <<"msgType">> => <<"cancel-observe">>, +% <<"data">> => #{ +% <<"path">> => <<"/3/0/10">> +% } +% }, +% CommandJson3 = emqx_json:encode(Command3), +% test_mqtt_broker:publish(CommandTopic, CommandJson3, 0), +% timer:sleep(50), +% Request3 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method3, options=Options3, payload=Payload3} = Request3, +% Path3 = get_coap_path(Options3), +% Observe3 = get_coap_observe(Options3), +% ?assertEqual(get, Method3), +% ?assertEqual(<<"/3/0/10">>, Path3), +% ?assertEqual(Observe3, 1), +% ?assertEqual(<<>>, Payload3), +% timer:sleep(50), + +% test_send_coap_observe_ack( UdpSock, +% "127.0.0.1", +% ?PORT, +% {ok, content}, +% #coap_content{content_format = <<"text/plain">>, payload = <<"1150">>}, +% Request3), +% timer:sleep(100), + +% ReadResult3 = emqx_json:encode(#{ +% <<"requestID">> => CmdId3, <<"cacheID">> => CmdId3, +% <<"msgType">> => <<"cancel-observe">>, +% <<"data">> => #{ +% <<"reqPath">> => <<"/3/0/10">>, +% <<"code">> => <<"2.05">>, +% <<"codeMsg">> => <<"content">>, +% <<"content">> => [#{ +% path => <<"/3/0/10">>, +% value => 1150 +% }] +% } +% }), +% ?assertEqual(ReadResult3, test_recv_mqtt_response(RespTopic)). + +% case80_specail_object_19_0_0_notify(Config) -> +% % step 1, device register, with extra register options +% Epn = "urn:oma:lwm2m:oma:3", +% RegOptionWangYi = "&apn=psmA.eDRX0.ctnb&im=13456&ct=2.0&mt=MDM9206&mv=4.0", +% MsgId1 = 15, +% UdpSock = ?config(sock, Config), + +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), + +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1"++RegOptionWangYi, [?PORT, Epn]), +% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, +% [], +% MsgId1), +% #coap_message{method = Method1} = test_recv_coap_response(UdpSock), +% ?assertEqual({ok,created}, Method1), +% ReadResult = emqx_json:encode(#{ +% <<"msgType">> => <<"register">>, +% <<"data">> => #{ +% <<"alternatePath">> => <<"/">>, +% <<"ep">> => list_to_binary(Epn), +% <<"lt">> => 345, +% <<"lwm2m">> => <<"1">>, +% <<"objectList">> => [<<"/1">>, <<"/2">>, <<"/3">>, <<"/4">>, <<"/5">>], +% <<"apn">> => <<"psmA.eDRX0.ctnb">>, +% <<"im">> => <<"13456">>, +% <<"ct">> => <<"2.0">>, +% <<"mt">> => <<"MDM9206">>, +% <<"mv">> => <<"4.0">> +% } +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)), + +% % step2, send a OBSERVE command to device +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% CmdId = 307, +% Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"observe">>, +% <<"data">> => #{ +% <<"path">> => <<"/19/0/0">> +% } +% }, +% CommandJson = emqx_json:encode(Command), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, +% Path2 = get_coap_path(Options2), +% Observe = get_coap_observe(Options2), +% ?assertEqual(get, Method2), +% ?assertEqual(<<"/19/0/0">>, Path2), +% ?assertEqual(Observe, 0), +% ?assertEqual(<<>>, Payload2), +% timer:sleep(50), + +% test_send_coap_observe_ack( UdpSock, +% "127.0.0.1", +% ?PORT, +% {ok, content}, +% #coap_content{content_format = <<"text/plain">>, payload = <<"2048">>}, +% Request2), +% timer:sleep(100). + +% %% step 3, device send uplink data notifications + +% case80_specail_object_19_1_0_write(Config) -> +% Epn = "urn:oma:lwm2m:oma:3", +% RegOptionWangYi = "&apn=psmA.eDRX0.ctnb&im=13456&ct=2.0&mt=MDM9206&mv=4.0", +% MsgId1 = 15, +% UdpSock = ?config(sock, Config), +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), + +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1"++RegOptionWangYi, [?PORT, Epn]), +% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, +% [], +% MsgId1), +% #coap_message{method = Method1} = test_recv_coap_response(UdpSock), +% ?assertEqual({ok,created}, Method1), +% test_recv_mqtt_response(RespTopic), + +% % step2, send a WRITE command to device +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% CmdId = 307, +% Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"write">>, +% <<"data">> => #{ +% <<"path">> => <<"/19/1/0">>, +% <<"type">> => <<"Opaque">>, +% <<"value">> => base64:encode(<<12345:32>>) +% } +% }, + +% CommandJson = emqx_json:encode(Command), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50), +% Request2 = test_recv_coap_request(UdpSock), +% #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, +% Path2 = get_coap_path(Options2), +% ?assertEqual(put, Method2), +% ?assertEqual(<<"/19/1/0">>, Path2), +% ?assertEqual(<<3:2, 0:1, 0:2, 4:3, 0, 12345:32>>, Payload2), +% timer:sleep(50), + +% test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true), +% timer:sleep(100), + +% ReadResult = emqx_json:encode(#{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"data">> => #{ +% <<"reqPath">> => <<"/19/1/0">>, +% <<"code">> => <<"2.04">>, +% <<"codeMsg">> => <<"changed">> +% }, +% <<"msgType">> => <<"write">> +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + +% case90_psm_mode(Config) -> +% server_cache_mode(Config, "ep=~s<=345&lwm2m=1&apn=psmA.eDRX0.ctnb"). + +% case90_queue_mode(Config) -> +% server_cache_mode(Config, "ep=~s<=345&lwm2m=1&b=UQ"). + +% server_cache_mode(Config, RegOption) -> +% application:set_env(?APP, qmode_time_window, 2), + +% % step 1, device register, with apn indicates "PSM" mode +% Epn = "urn:oma:lwm2m:oma:3", + +% MsgId1 = 15, +% UdpSock = ?config(sock, Config), +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +% timer:sleep(200), + +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b/rd?"++RegOption, [?PORT, Epn]), +% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, +% [], +% MsgId1), +% #coap_message{type = ack, method = Method1, options = Opts} = test_recv_coap_response(UdpSock), +% ?assertEqual({ok,created}, Method1), +% ?LOGT("Options got: ~p", [Opts]), +% Location = proplists:get_value(location_path, Opts), +% test_recv_mqtt_response(RespTopic), + +% %% server not in PSM mode +% send_read_command_1(0, UdpSock), +% verify_read_response_1(0, UdpSock), + +% %% server inters into PSM mode +% timer:sleep(2), + +% %% verify server caches downlink commands +% send_read_command_1(1, UdpSock), +% send_read_command_1(2, UdpSock), +% send_read_command_1(3, UdpSock), + +% ?assertEqual(timeout_test_recv_coap_request, test_recv_coap_request(UdpSock)), + +% device_update_1(UdpSock, Location), + +% verify_read_response_1(1, UdpSock), +% verify_read_response_1(2, UdpSock), +% verify_read_response_1(3, UdpSock). + +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% %%% Internal Functions +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% send_read_command_1(CmdId, _UdpSock) -> +% Epn = "urn:oma:lwm2m:oma:3", +% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +% Command = #{ +% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"read">>, +% <<"data">> => #{ +% <<"path">> => <<"/3/0/0">> +% } +% }, +% CommandJson = emqx_json:encode(Command), +% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +% timer:sleep(50). + +% verify_read_response_1(CmdId, UdpSock) -> +% Epn = "urn:oma:lwm2m:oma:3", +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), + +% %% device receives a command +% Request = test_recv_coap_request(UdpSock), +% ?LOGT("LwM2M client got ~p", [Request]), + +% %% device replies the commond +% test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request, true), + +% ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +% <<"msgType">> => <<"read">>, +% <<"data">> => #{ +% <<"code">> => <<"2.05">>, +% <<"codeMsg">> => <<"content">>, +% <<"content">> => [#{ +% path => <<"/3/0/0">>, +% value => <<"EMQ">> +% }] +% } +% }), +% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + +% device_update_1(UdpSock, Location) -> +% Epn = "urn:oma:lwm2m:oma:3", +% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +% ?LOGT("send UPDATE command", []), +% MsgId2 = 27, +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b~s?lt=789", [?PORT, join_path(Location, <<>>)]), +% #coap_content{payload = <<>>}, +% [], +% MsgId2), +% #coap_message{type = ack, id = MsgId2, method = Method2} = test_recv_coap_response(UdpSock), +% {ok,changed} = Method2, +% test_recv_mqtt_response(RespTopic). + +% test_recv_mqtt_response(RespTopic) -> +% receive +% {publish, #{topic := RespTopic, payload := RM}} -> +% ?LOGT("test_recv_mqtt_response Response=~p", [RM]), +% RM +% after 1000 -> timeout_test_recv_mqtt_response +% end. + +% test_send_coap_request(UdpSock, Method, Uri, Content, Options, MsgId) -> +% is_record(Content, coap_content) orelse error("Content must be a #coap_content!"), +% is_list(Options) orelse error("Options must be a list"), +% case resolve_uri(Uri) of +% {coap, {IpAddr, Port}, Path, Query} -> +% Request0 = lwm2m_coap_message:request(con, Method, Content, [{uri_path, Path}, {uri_query, Query} | Options]), +% Request = Request0#coap_message{id = MsgId}, +% ?LOGT("send_coap_request Request=~p", [Request]), +% RequestBinary = lwm2m_coap_message_parser:encode(Request), +% ?LOGT("test udp socket send to ~p:~p, data=~p", [IpAddr, Port, RequestBinary]), +% ok = gen_udp:send(UdpSock, IpAddr, Port, RequestBinary); +% {SchemeDiff, ChIdDiff, _, _} -> +% error(lists:flatten(io_lib:format("scheme ~s or ChId ~s does not match with socket", [SchemeDiff, ChIdDiff]))) +% end. + +% test_recv_coap_response(UdpSock) -> +% {ok, {Address, Port, Packet}} = gen_udp:recv(UdpSock, 0, 2000), +% Response = lwm2m_coap_message_parser:decode(Packet), +% ?LOGT("test udp receive from ~p:~p, data1=~p, Response=~p", [Address, Port, Packet, Response]), +% #coap_message{type = ack, method = Method, id=Id, token = Token, options = Options, payload = Payload} = Response, +% ?LOGT("receive coap response Method=~p, Id=~p, Token=~p, Options=~p, Payload=~p", [Method, Id, Token, Options, Payload]), +% Response. + +% test_recv_coap_request(UdpSock) -> +% case gen_udp:recv(UdpSock, 0, 2000) of +% {ok, {_Address, _Port, Packet}} -> +% Request = lwm2m_coap_message_parser:decode(Packet), +% #coap_message{type = con, method = Method, id=Id, token = Token, payload = Payload, options = Options} = Request, +% ?LOGT("receive coap request Method=~p, Id=~p, Token=~p, Options=~p, Payload=~p", [Method, Id, Token, Options, Payload]), +% Request; +% {error, Reason} -> +% ?LOGT("test_recv_coap_request failed, Reason=~p", [Reason]), +% timeout_test_recv_coap_request +% end. + +% test_send_coap_response(UdpSock, Host, Port, Code, Content, Request, Ack) -> +% is_record(Content, coap_content) orelse error("Content must be a #coap_content!"), +% is_list(Host) orelse error("Host is not a string"), + +% {ok, IpAddr} = inet:getaddr(Host, inet), +% Response = lwm2m_coap_message:response(Code, Content, Request), +% Response2 = case Ack of +% true -> Response#coap_message{type = ack}; +% false -> Response +% end, +% ?LOGT("test_send_coap_response Response=~p", [Response2]), +% ok = gen_udp:send(UdpSock, IpAddr, Port, lwm2m_coap_message_parser:encode(Response2)). + +% test_send_empty_ack(UdpSock, Host, Port, Request) -> +% is_list(Host) orelse error("Host is not a string"), +% {ok, IpAddr} = inet:getaddr(Host, inet), +% EmptyACK = lwm2m_coap_message:ack(Request), +% ?LOGT("test_send_empty_ack EmptyACK=~p", [EmptyACK]), +% ok = gen_udp:send(UdpSock, IpAddr, Port, lwm2m_coap_message_parser:encode(EmptyACK)). + +% test_send_coap_observe_ack(UdpSock, Host, Port, Code, Content, Request) -> +% is_record(Content, coap_content) orelse error("Content must be a #coap_content!"), +% is_list(Host) orelse error("Host is not a string"), + +% {ok, IpAddr} = inet:getaddr(Host, inet), +% Response = lwm2m_coap_message:response(Code, Content, Request), +% Response1 = lwm2m_coap_message:set(observe, 0, Response), +% Response2 = Response1#coap_message{type = ack}, + +% ?LOGT("test_send_coap_observe_ack Response=~p", [Response2]), +% ResponseBinary = lwm2m_coap_message_parser:encode(Response2), +% ok = gen_udp:send(UdpSock, IpAddr, Port, ResponseBinary). + +% test_send_coap_notif(UdpSock, Host, Port, Content, ObSeq, Request) -> +% is_record(Content, coap_content) orelse error("Content must be a #coap_content!"), +% is_list(Host) orelse error("Host is not a string"), + +% {ok, IpAddr} = inet:getaddr(Host, inet), +% Notif = lwm2m_coap_message:response({ok, content}, Content, Request), +% NewNotif = lwm2m_coap_message:set(observe, ObSeq, Notif), +% ?LOGT("test_send_coap_notif Response=~p", [NewNotif]), +% NotifBinary = lwm2m_coap_message_parser:encode(NewNotif), +% ?LOGT("test udp socket send to ~p:~p, data=~p", [IpAddr, Port, NotifBinary]), +% ok = gen_udp:send(UdpSock, IpAddr, Port, NotifBinary). + +% std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic) -> +% test_send_coap_request( UdpSock, +% post, +% sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), +% #coap_content{content_format = <<"text/plain">>, payload = ObjectList}, +% [], +% MsgId1), +% #coap_message{method = {ok,created}} = test_recv_coap_response(UdpSock), +% test_recv_mqtt_response(RespTopic), +% timer:sleep(100). + +% resolve_uri(Uri) -> +% {ok, #{scheme := Scheme, +% host := Host, +% port := PortNo, +% path := Path} = URIMap} = emqx_http_lib:uri_parse(Uri), +% Query = maps:get(query, URIMap, ""), +% {ok, PeerIP} = inet:getaddr(Host, inet), +% {Scheme, {PeerIP, PortNo}, split_path(Path), split_query(Query)}. + +% split_path([]) -> []; +% split_path([$/]) -> []; +% split_path([$/ | Path]) -> split_segments(Path, $/, []). + +% split_query([]) -> []; +% split_query(Path) -> split_segments(Path, $&, []). + +% split_segments(Path, Char, Acc) -> +% case string:rchr(Path, Char) of +% 0 -> +% [make_segment(Path) | Acc]; +% N when N > 0 -> +% split_segments(string:substr(Path, 1, N-1), Char, +% [make_segment(string:substr(Path, N+1)) | Acc]) +% end. + +% make_segment(Seg) -> +% list_to_binary(emqx_http_lib:uri_decode(Seg)). + + +% get_coap_path(Options) -> +% get_path(Options, <<>>). + +% get_coap_query(Options) -> +% proplists:get_value(uri_query, Options, []). + +% get_coap_observe(Options) -> +% get_observe(Options). + + +% get_path([], Acc) -> +% %?LOGT("get_path Acc=~p", [Acc]), +% Acc; +% get_path([{uri_path, Path1}|T], Acc) -> +% %?LOGT("Path=~p, Acc=~p", [Path1, Acc]), +% get_path(T, join_path(Path1, Acc)); +% get_path([{_, _}|T], Acc) -> +% get_path(T, Acc). + +% get_observe([]) -> +% undefined; +% get_observe([{observe, V}|_T]) -> +% V; +% get_observe([{_, _}|T]) -> +% get_observe(T). + +% join_path([], Acc) -> Acc; +% join_path([<<"/">>|T], Acc) -> +% join_path(T, Acc); +% join_path([H|T], Acc) -> +% join_path(T, <>). + +% sprintf(Format, Args) -> +% lists:flatten(io_lib:format(Format, Args)). diff --git a/apps/emqx_gateway/src/lwm2m/test/emqx_tlv_SUITE.erl b/apps/emqx_gateway/src/lwm2m/test/emqx_tlv_SUITE.erl new file mode 100644 index 000000000..d4d3c70cf --- /dev/null +++ b/apps/emqx_gateway/src/lwm2m/test/emqx_tlv_SUITE.erl @@ -0,0 +1,240 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_tlv_SUITE). + +% -compile(export_all). +% -compile(nowarn_export_all). + +% -define(LOGT(Format, Args), logger:debug("TEST_SUITE: " ++ Format, Args)). + +% -include("src/lwm2m/include/emqx_lwm2m.hrl"). +% -include_lib("lwm2m_coap/include/coap.hrl"). +% -include_lib("eunit/include/eunit.hrl"). + + +% all() -> [case01, case02, case03, case03_0, case04, case05, case06, case07, case08, case09]. + + + +% init_per_suite(Config) -> +% Config. + +% end_per_suite(Config) -> +% Config. + + +% case01(_Config) -> +% Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65>>, +% R = emqx_lwm2m_tlv:parse(Data), +% Exp = [ +% #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>} +% ], +% ?assertEqual(Exp, R), +% EncodedBinary = emqx_lwm2m_tlv:encode(Exp), +% ?assertEqual(EncodedBinary, Data). + +% case02(_Config) -> +% Data = <<16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05>>, +% R = emqx_lwm2m_tlv:parse(Data), +% Exp = [ +% #{tlv_multiple_resource => 16#06, value => [ +% #{tlv_resource_instance => 16#00, value => <<1>>}, +% #{tlv_resource_instance => 16#01, value => <<5>>} +% ]} +% ], +% ?assertEqual(Exp, R), +% EncodedBinary = emqx_lwm2m_tlv:encode(Exp), +% ?assertEqual(EncodedBinary, Data). + +% case03(_Config) -> +% Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33>>, +% R = emqx_lwm2m_tlv:parse(Data), +% Exp = [ +% #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>}, +% #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>}, +% #{tlv_resource_with_value => 16#02, value => <<"345000123">>} +% ], +% ?assertEqual(Exp, R), +% EncodedBinary = emqx_lwm2m_tlv:encode(Exp), +% ?assertEqual(EncodedBinary, Data). + +% case03_0(_Config) -> +% Data = <<16#87, 16#02, 16#41, 16#7F, 16#07, 16#61, 16#01, 16#36, 16#01>>, +% R = emqx_lwm2m_tlv:parse(Data), +% Exp = [ +% #{tlv_multiple_resource => 16#02, value => [ +% #{tlv_resource_instance => 16#7F, value => <<16#07>>}, +% #{tlv_resource_instance => 16#0136, value => <<16#01>>} +% ]} +% ], +% ?assertEqual(Exp, R), +% EncodedBinary = emqx_lwm2m_tlv:encode(Exp), +% ?assertEqual(EncodedBinary, Data). + +% case04(_Config) -> +% % 6.4.3.1 Single Object Instance Request Example +% Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33, 16#C3, 16#03, 16#31, 16#2E, 16#30, 16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05, 16#88, 16#07, 16#08, 16#42, 16#00, 16#0E, 16#D8, 16#42, 16#01, 16#13, 16#88, 16#87, 16#08, 16#41, 16#00, 16#7D, 16#42, 16#01, 16#03, 16#84, 16#C1, 16#09, 16#64, 16#C1, 16#0A, 16#0F, 16#83, 16#0B, 16#41, 16#00, 16#00, 16#C4, 16#0D, 16#51, 16#82, 16#42, 16#8F, 16#C6, 16#0E, 16#2B, 16#30, 16#32, 16#3A, 16#30, 16#30, 16#C1, 16#10, 16#55>>, +% R = emqx_lwm2m_tlv:parse(Data), +% Exp = [ +% #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>}, +% #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>}, +% #{tlv_resource_with_value => 16#02, value => <<"345000123">>}, +% #{tlv_resource_with_value => 16#03, value => <<"1.0">>}, +% #{tlv_multiple_resource => 16#06, value => [ +% #{tlv_resource_instance => 16#00, value => <<1>>}, +% #{tlv_resource_instance => 16#01, value => <<5>>} +% ]}, +% #{tlv_multiple_resource => 16#07, value => [ +% #{tlv_resource_instance => 16#00, value => <<16#0ED8:16>>}, +% #{tlv_resource_instance => 16#01, value => <<16#1388:16>>} +% ]}, +% #{tlv_multiple_resource => 16#08, value => [ +% #{tlv_resource_instance => 16#00, value => <<16#7d>>}, +% #{tlv_resource_instance => 16#01, value => <<16#0384:16>>} +% ]}, +% #{tlv_resource_with_value => 16#09, value => <<16#64>>}, +% #{tlv_resource_with_value => 16#0A, value => <<16#0F>>}, +% #{tlv_multiple_resource => 16#0B, value => [ +% #{tlv_resource_instance => 16#00, value => <<16#00>>} +% ]}, +% #{tlv_resource_with_value => 16#0D, value => <<16#5182428F:32>>}, +% #{tlv_resource_with_value => 16#0E, value => <<"+02:00">>}, +% #{tlv_resource_with_value => 16#10, value => <<"U">>} +% ], +% ?assertEqual(Exp, R), +% EncodedBinary = emqx_lwm2m_tlv:encode(Exp), +% ?assertEqual(EncodedBinary, Data). + +% case05(_Config) -> +% % 6.4.3.2 Multiple Object Instance Request Examples +% % A) Request on Single-Instance Object +% Data = <<16#08, 16#00, 16#79, 16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33, 16#C3, 16#03, 16#31, 16#2E, 16#30, 16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05, 16#88, 16#07, 16#08, 16#42, 16#00, 16#0E, 16#D8, 16#42, 16#01, 16#13, 16#88, 16#87, 16#08, 16#41, 16#00, 16#7D, 16#42, 16#01, 16#03, 16#84, 16#C1, 16#09, 16#64, 16#C1, 16#0A, 16#0F, 16#83, 16#0B, 16#41, 16#00, 16#00, 16#C4, 16#0D, 16#51, 16#82, 16#42, 16#8F, 16#C6, 16#0E, 16#2B, 16#30, 16#32, 16#3A, 16#30, 16#30, 16#C1, 16#10, 16#55>>, +% R = emqx_lwm2m_tlv:parse(Data), +% Exp = [ +% #{tlv_object_instance => 16#00, value => [ +% #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>}, +% #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>}, +% #{tlv_resource_with_value => 16#02, value => <<"345000123">>}, +% #{tlv_resource_with_value => 16#03, value => <<"1.0">>}, +% #{tlv_multiple_resource => 16#06, value => [ +% #{tlv_resource_instance => 16#00, value => <<1>>}, +% #{tlv_resource_instance => 16#01, value => <<5>>} +% ]}, +% #{tlv_multiple_resource => 16#07, value => [ +% #{tlv_resource_instance => 16#00, value => <<16#0ED8:16>>}, +% #{tlv_resource_instance => 16#01, value => <<16#1388:16>>} +% ]}, +% #{tlv_multiple_resource => 16#08, value => [ +% #{tlv_resource_instance => 16#00, value => <<16#7d>>}, +% #{tlv_resource_instance => 16#01, value => <<16#0384:16>>} +% ]}, +% #{tlv_resource_with_value => 16#09, value => <<16#64>>}, +% #{tlv_resource_with_value => 16#0A, value => <<16#0F>>}, +% #{tlv_multiple_resource => 16#0B, value => [ +% #{tlv_resource_instance => 16#00, value => <<16#00>>} +% ]}, +% #{tlv_resource_with_value => 16#0D, value => <<16#5182428F:32>>}, +% #{tlv_resource_with_value => 16#0E, value => <<"+02:00">>}, +% #{tlv_resource_with_value => 16#10, value => <<"U">>} +% ]} +% ], +% ?assertEqual(Exp, R), +% EncodedBinary = emqx_lwm2m_tlv:encode(Exp), +% ?assertEqual(EncodedBinary, Data). + +% case06(_Config) -> +% % 6.4.3.2 Multiple Object Instance Request Examples +% % B) Request on Multiple-Instances Object having 2 instances +% Data = <<16#08, 16#00, 16#0E, 16#C1, 16#00, 16#01, 16#C1, 16#01, 16#00, 16#83, 16#02, 16#41, 16#7F, 16#07, 16#C1, 16#03, 16#7F, 16#08, 16#02, 16#12, 16#C1, 16#00, 16#03, 16#C1, 16#01, 16#00, 16#87, 16#02, 16#41, 16#7F, 16#07, 16#61, 16#01, 16#36, 16#01, 16#C1, 16#03, 16#7F>>, +% R = emqx_lwm2m_tlv:parse(Data), +% Exp = [ +% #{tlv_object_instance => 16#00, value => [ +% #{tlv_resource_with_value => 16#00, value => <<16#01>>}, +% #{tlv_resource_with_value => 16#01, value => <<16#00>>}, +% #{tlv_multiple_resource => 16#02, value => [ +% #{tlv_resource_instance => 16#7F, value => <<16#07>>} +% ]}, +% #{tlv_resource_with_value => 16#03, value => <<16#7F>>} +% ]}, +% #{tlv_object_instance => 16#02, value => [ +% #{tlv_resource_with_value => 16#00, value => <<16#03>>}, +% #{tlv_resource_with_value => 16#01, value => <<16#00>>}, +% #{tlv_multiple_resource => 16#02, value => [ +% #{tlv_resource_instance => 16#7F, value => <<16#07>>}, +% #{tlv_resource_instance => 16#0136, value => <<16#01>>} +% ]}, +% #{tlv_resource_with_value => 16#03, value => <<16#7F>>} +% ]} +% ], +% ?assertEqual(Exp, R), +% EncodedBinary = emqx_lwm2m_tlv:encode(Exp), +% ?assertEqual(EncodedBinary, Data). + +% case07(_Config) -> +% % 6.4.3.2 Multiple Object Instance Request Examples +% % C) Request on Multiple-Instances Object having 1 instance only +% Data = <<16#08, 16#00, 16#0F, 16#C1, 16#00, 16#01, 16#C4, 16#01, 16#00, 16#01, 16#51, 16#80, 16#C1, 16#06, 16#01, 16#C1, 16#07, 16#55>>, +% R = emqx_lwm2m_tlv:parse(Data), +% Exp = [ +% #{tlv_object_instance => 16#00, value => [ +% #{tlv_resource_with_value => 16#00, value => <<16#01>>}, +% #{tlv_resource_with_value => 16#01, value => <<86400:32>>}, +% #{tlv_resource_with_value => 16#06, value => <<16#01>>}, +% #{tlv_resource_with_value => 16#07, value => <<$U>>}]} +% ], +% ?assertEqual(Exp, R), +% EncodedBinary = emqx_lwm2m_tlv:encode(Exp), +% ?assertEqual(EncodedBinary, Data). + +% case08(_Config) -> +% % 6.4.3.3 Example of Request on an Object Instance containing an Object Link Resource +% % Example 1) request to Object 65 Instance 0: Read /65/0 +% Data = <<16#88, 16#00, 16#0C, 16#44, 16#00, 16#00, 16#42, 16#00, 16#00, 16#44, 16#01, 16#00, 16#42, 16#00, 16#01, 16#C8, 16#01, 16#0D, 16#38, 16#36, 16#31, 16#33, 16#38, 16#30, 16#30, 16#37, 16#35, 16#35, 16#35, 16#30, 16#30, 16#C4, 16#02, 16#12, 16#34, 16#56, 16#78>>, +% R = emqx_lwm2m_tlv:parse(Data), +% Exp = [ +% #{tlv_multiple_resource => 16#00, value => [ +% #{tlv_resource_instance => 16#00, value => <<16#00, 16#42, 16#00, 16#00>>}, +% #{tlv_resource_instance => 16#01, value => <<16#00, 16#42, 16#00, 16#01>>} +% ]}, +% #{tlv_resource_with_value => 16#01, value => <<"8613800755500">>}, +% #{tlv_resource_with_value => 16#02, value => <<16#12345678:32>>} +% ], +% ?assertEqual(Exp, R), +% EncodedBinary = emqx_lwm2m_tlv:encode(Exp), +% ?assertEqual(EncodedBinary, Data). + +% case09(_Config) -> +% % 6.4.3.3 Example of Request on an Object Instance containing an Object Link Resource +% % Example 2) request to Object 66: Read /66: TLV payload will contain 2 Object Instances +% Data = <<16#08, 16#00, 16#26, 16#C8, 16#00, 16#0B, 16#6D, 16#79, 16#53, 16#65, 16#72, 16#76, 16#69, 16#63, 16#65, 16#20, 16#31, 16#C8, 16#01, 16#0F, 16#49, 16#6E, 16#74, 16#65, 16#72, 16#6E, 16#65, 16#74, 16#2E, 16#31, 16#35, 16#2E, 16#32, 16#33, 16#34, 16#C4, 16#02, 16#00, 16#43, 16#00, 16#00, 16#08, 16#01, 16#26, 16#C8, 16#00, 16#0B, 16#6D, 16#79, 16#53, 16#65, 16#72, 16#76, 16#69, 16#63, 16#65, 16#20, 16#32, 16#C8, 16#01, 16#0F, 16#49, 16#6E, 16#74, 16#65, 16#72, 16#6E, 16#65, 16#74, 16#2E, 16#31, 16#35, 16#2E, 16#32, 16#33, 16#35, 16#C4, 16#02, 16#FF, 16#FF, 16#FF, 16#FF>>, +% R = emqx_lwm2m_tlv:parse(Data), +% Exp = [ +% #{tlv_object_instance => 16#00, value => [ +% #{tlv_resource_with_value => 16#00, value => <<"myService 1">>}, +% #{tlv_resource_with_value => 16#01, value => <<"Internet.15.234">>}, +% #{tlv_resource_with_value => 16#02, value => <<16#00, 16#43, 16#00, 16#00>>} +% ]}, +% #{tlv_object_instance => 16#01, value => [ +% #{tlv_resource_with_value => 16#00, value => <<"myService 2">>}, +% #{tlv_resource_with_value => 16#01, value => <<"Internet.15.235">>}, +% #{tlv_resource_with_value => 16#02, value => <<16#FF, 16#FF, 16#FF, 16#FF>>} +% ]} +% ], +% ?assertEqual(Exp, R), +% EncodedBinary = emqx_lwm2m_tlv:encode(Exp), +% ?assertEqual(EncodedBinary, Data). + diff --git a/apps/emqx_gateway/src/lwm2m/test/test_mqtt_broker.erl b/apps/emqx_gateway/src/lwm2m/test/test_mqtt_broker.erl new file mode 100644 index 000000000..4a6c77444 --- /dev/null +++ b/apps/emqx_gateway/src/lwm2m/test/test_mqtt_broker.erl @@ -0,0 +1,171 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(test_mqtt_broker). + +% -compile(nowarn_export_all). +% -compile(export_all). + +% -define(LOGT(Format, Args), logger:debug("TEST_BROKER: " ++ Format, Args)). + +% -record(state, {subscriber}). + +% -include_lib("emqx/include/emqx.hrl"). + +% -include_lib("emqx/include/emqx_mqtt.hrl"). + +% -include_lib("eunit/include/eunit.hrl"). + +% start(_, <<"attacker">>, _, _, _) -> +% {stop, auth_failure}; +% start(ClientId, Username, Password, _Channel, KeepaliveInterval) -> +% true = is_binary(ClientId), +% (true = ( is_binary(Username)) orelse (Username == undefined) ), +% (true = ( is_binary(Password)) orelse (Password == undefined) ), +% self() ! {keepalive, start, KeepaliveInterval}, +% {ok, []}. + +% publish(Topic, Payload, Qos) -> +% ClientId = <<"lwm2m_test_suite">>, +% Msg = emqx_message:make(ClientId, Qos, Topic, Payload), +% emqx:publish(Msg). + +% subscribe(Topic) -> +% gen_server:call(?MODULE, {subscribe, Topic, self()}). + +% unsubscribe(Topic) -> +% gen_server:call(?MODULE, {unsubscribe, Topic}). + +% get_subscrbied_topics() -> +% [Topic || {_Client, Topic} <- ets:tab2list(emqx_subscription)]. + +% start_link() -> +% gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +% stop() -> +% gen_server:stop(?MODULE). + +% init(_Param) -> +% {ok, #state{subscriber = []}}. + +% handle_call({subscribe, Topic, Proc}, _From, State=#state{subscriber = SubList}) -> +% ?LOGT("test broker subscribe Topic=~p, Pid=~p~n", [Topic, Proc]), +% is_binary(Topic) orelse error("Topic should be a binary"), +% {reply, {ok, []}, State#state{subscriber = [{Topic, Proc}|SubList]}}; + +% handle_call(get_subscribed_topics, _From, State=#state{subscriber = SubList}) -> +% Response = subscribed_topics(SubList, []), +% ?LOGT("test broker get subscribed topics=~p~n", [Response]), +% {reply, Response, State}; + +% handle_call({unsubscribe, Topic}, _From, State=#state{subscriber = SubList}) -> +% ?LOGT("test broker unsubscribe Topic=~p~n", [Topic]), +% is_binary(Topic) orelse error("Topic should be a binary"), +% NewSubList = proplists:delete(Topic, SubList), +% {reply, {ok, []}, State#state{subscriber = NewSubList}}; + + +% handle_call({publish, {Topic, Msg, MatchedTopicFilter}}, _From, State=#state{subscriber = SubList}) -> +% (is_binary(Topic) and is_binary(Msg)) orelse error("Topic and Msg should be binary"), +% Pid = proplists:get_value(MatchedTopicFilter, SubList), +% ?LOGT("test broker publish topic=~p, Msg=~p, Pid=~p, MatchedTopicFilter=~p, SubList=~p~n", [Topic, Msg, Pid, MatchedTopicFilter, SubList]), +% (Pid == undefined) andalso ?LOGT("!!!!! this topic ~p has never been subscribed, please specify a valid topic filter", [MatchedTopicFilter]), +% ?assertNotEqual(undefined, Pid), +% Pid ! {deliver, #message{topic = Topic, payload = Msg}}, +% {reply, ok, State}; + +% handle_call(stop, _From, State) -> +% {stop, normal, stopped, State}; + +% handle_call(Req, _From, State) -> +% ?LOGT("test_broker_server: ignore call Req=~p~n", [Req]), +% {reply, {error, badreq}, State}. + + +% handle_cast(Msg, State) -> +% ?LOGT("test_broker_server: ignore cast msg=~p~n", [Msg]), +% {noreply, State}. + +% handle_info(Info, State) -> +% ?LOGT("test_broker_server: ignore info=~p~n", [Info]), +% {noreply, State}. + +% terminate(Reason, _State) -> +% ?LOGT("test_broker_server: terminate Reason=~p~n", [Reason]), +% ok. + +% code_change(_OldVsn, State, _Extra) -> +% {ok, State}. + + + + +% subscribed_topics([], Acc) -> +% Acc; +% subscribed_topics([{Topic,_Pid}|T], Acc) -> +% subscribed_topics(T, [Topic|Acc]). + + + + +% -record(keepalive, {statfun, statval, tsec, tmsg, tref, repeat = 0}). + +% -type(keepalive() :: #keepalive{}). + +% %% @doc Start a keepalive +% -spec(start(fun(), integer(), any()) -> undefined | keepalive()). +% start(_, 0, _) -> +% undefined; +% start(StatFun, TimeoutSec, TimeoutMsg) -> +% {ok, StatVal} = StatFun(), +% #keepalive{statfun = StatFun, statval = StatVal, +% tsec = TimeoutSec, tmsg = TimeoutMsg, +% tref = timer(TimeoutSec, TimeoutMsg)}. + +% %% @doc Check keepalive, called when timeout. +% -spec(check(keepalive()) -> {ok, keepalive()} | {error, any()}). +% check(KeepAlive = #keepalive{statfun = StatFun, statval = LastVal, repeat = Repeat}) -> +% case StatFun() of +% {ok, NewVal} -> +% if NewVal =/= LastVal -> +% {ok, resume(KeepAlive#keepalive{statval = NewVal, repeat = 0})}; +% Repeat < 1 -> +% {ok, resume(KeepAlive#keepalive{statval = NewVal, repeat = Repeat + 1})}; +% true -> +% {error, timeout} +% end; +% {error, Error} -> +% {error, Error} +% end. + +% resume(KeepAlive = #keepalive{tsec = TimeoutSec, tmsg = TimeoutMsg}) -> +% KeepAlive#keepalive{tref = timer(TimeoutSec, TimeoutMsg)}. + +% %% @doc Cancel Keepalive +% -spec(cancel(keepalive()) -> ok). +% cancel(#keepalive{tref = TRef}) -> +% cancel(TRef); +% cancel(undefined) -> +% ok; +% cancel(TRef) -> +% catch erlang:cancel_timer(TRef). + +% timer(Sec, Msg) -> +% erlang:send_after(timer:seconds(Sec), self(), Msg). + + +% log(Format, Args) -> +% logger:debug(Format, Args). diff --git a/apps/emqx_sn/README.md b/apps/emqx_gateway/src/mqttsn/README.md similarity index 94% rename from apps/emqx_sn/README.md rename to apps/emqx_gateway/src/mqttsn/README.md index d7251c49c..8179dde62 100644 --- a/apps/emqx_sn/README.md +++ b/apps/emqx_gateway/src/mqttsn/README.md @@ -1,10 +1,9 @@ -emqx-sn -======= +# MQTT-SN Gateway EMQ X MQTT-SN Gateway. -Configure Plugin ----------------- +## Configure Plugin + File: etc/emqx_sn.conf @@ -72,8 +71,7 @@ mqtt.sn.password = abc - mqtt.sn.password * This parameter is optional. Pair with username above. -Load Plugin ------------ +## Load Plugin ``` ./bin/emqx_ctl plugins load emqx_sn @@ -95,23 +93,18 @@ Load Plugin - https://github.com/njh/mqtt-sn-tools - https://github.com/arobenko/mqtt-sn -sleeping device ------------ +### sleeping device PINGREQ must have a ClientId which is identical to the one in CONNECT message. Without ClientId, emqx-sn will ignore such PINGREQ. -pre-defined topics ------------ +### pre-defined topics The mapping of a pre-defined topic id and topic name should be known inadvance by both client's application and gateway. We define this mapping info in emqx_sn.conf file, and which shall be kept equivalent in all client's side. -License -------- +## License Apache License Version 2.0 -Author ------- - -EMQ X-Men Team. +## Author +EMQ X Team. diff --git a/apps/emqx_sn/src/emqx_sn_broadcast.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl similarity index 96% rename from apps/emqx_sn/src/emqx_sn_broadcast.erl rename to apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl index a1630b844..f890cc774 100644 --- a/apps/emqx_sn/src/emqx_sn_broadcast.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl @@ -18,7 +18,7 @@ -behaviour(gen_server). --include("emqx_sn.hrl"). +-include("src/mqttsn/include/emqx_sn.hrl"). -export([ start_link/2 , stop/0 @@ -89,7 +89,7 @@ ensure_advertise(State = #state{duration = Duration}) -> send_advertise(#state{gwid = GwId, sock = Sock, port = Port, addrs = Addrs, duration = Duration}) -> - Data = emqx_sn_frame:serialize(?SN_ADVERTISE_MSG(GwId, Duration)), + Data = emqx_sn_frame:serialize_pkt(?SN_ADVERTISE_MSG(GwId, Duration), #{}), lists:foreach(fun(Addr) -> ?LOG(debug, "SEND SN_ADVERTISE to ~p~n", [Addr]), gen_udp:send(Sock, Addr, Port, Data) diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl new file mode 100644 index 000000000..d7717fdc4 --- /dev/null +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl @@ -0,0 +1,1457 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_sn_channel). + +-behavior(emqx_gateway_channel). + +-include("src/mqttsn/include/emqx_sn.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-logger_header("[SN-Proto]"). + +%% API +-export([ info/1 + , info/2 + , stats/1 + ]). + +-export([ init/2 + , handle_in/2 + , handle_out/3 + , handle_deliver/2 + , handle_timeout/3 + , terminate/2 + , set_conn_state/2 + ]). + +-export([ handle_call/2 + , handle_cast/2 + , handle_info/2 + ]). + +-record(channel, { + %% Context + ctx :: emqx_gateway_ctx:context(), + %% Registry + registry :: emqx_sn_registry:registry(), + %% Gateway Id + gateway_id :: integer(), + %% Enable QoS3 + enable_qos3 :: boolean(), %% XXX: Get confs from ctx ? + %% MQTT-SN Connection Info + conninfo :: emqx_types:conninfo(), + %% MQTT-SN Client Info + clientinfo :: emqx_types:clientinfo(), + %% Session + session :: emqx_session:session() | undefined, + %% Keepalive + keepalive :: emqx_keepalive:keepalive() | undefined, + %% Will Msg + will_msg :: emqx_types:message() | undefined, + %% ClientInfo override specs + clientinfo_override :: map(), + %% Connection State + conn_state :: conn_state(), + %% Timer + timers :: #{atom() => disable | undefined | reference()}, + %%% Takeover + takeover :: boolean(), + %% Resume + resuming :: boolean(), + %% Pending delivers when takeovering + pendings :: list() + }). + +-type(channel() :: #channel{}). + +-type(conn_state() :: idle | connecting | connected | asleep | disconnected). + +-type(reply() :: {outgoing, mqtt_sn_message()} + | {outgoing, [mqtt_sn_message()]} + | {event, conn_state()|updated} + | {close, Reason :: atom()}). + +-type(replies() :: reply() | [reply()]). + +-define(TIMER_TABLE, #{ + alive_timer => keepalive, + retry_timer => retry_delivery, + await_timer => expire_awaiting_rel, + asleep_timer => expire_asleep + }). + +-define(DEFAULT_OVERRIDE, + #{ clientid => <<"">> %% Generate clientid by default + , username => <<"${Packet.headers.login}">> + , password => <<"${Packet.headers.passcode}">> + }). + +-define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]). + +-define(NEG_QOS_CLIENT_ID, <<"NegQoS-Client">>). + +%%-------------------------------------------------------------------- +%% Init the channel +%%-------------------------------------------------------------------- + +%% @doc Init protocol +init(ConnInfo = #{peername := {PeerHost, _}, + sockname := {_, SockPort}}, Option) -> + Peercert = maps:get(peercert, ConnInfo, undefined), + Mountpoint = maps:get(mountpoint, Option, undefined), + Registry = maps:get(registry, Option), + GwId = maps:get(gateway_id, Option), + EnableQoS3 = maps:get(enable_qos3, Option, true), + ClientInfo = set_peercert_infos( + Peercert, + #{ zone => undefined %% XXX: + , protocol => 'mqtt-sn' + , peerhost => PeerHost + , sockport => SockPort + , clientid => undefined + , username => undefined + , is_bridge => false + , is_superuser => false + , mountpoint => Mountpoint + } + ), + + Ctx = maps:get(ctx, Option), + Override = maps:merge(?DEFAULT_OVERRIDE, + maps:get(clientinfo_override, Option, #{}) + ), + #channel{ ctx = Ctx + , registry = Registry + , gateway_id = GwId + , enable_qos3 = EnableQoS3 + , conninfo = ConnInfo + , clientinfo = ClientInfo + , clientinfo_override = Override + , conn_state = idle + , timers = #{} + , takeover = false + , resuming = false + , pendings = [] + }. + +set_peercert_infos(NoSSL, ClientInfo) + when NoSSL =:= nossl; + NoSSL =:= undefined -> + ClientInfo; +set_peercert_infos(Peercert, ClientInfo) -> + {DN, CN} = {esockd_peercert:subject(Peercert), + esockd_peercert:common_name(Peercert)}, + ClientInfo#{dn => DN, cn => CN}. + +-spec info(channel()) -> emqx_types:infos(). +info(Channel) -> + maps:from_list(info(?INFO_KEYS, Channel)). + +-spec info(list(atom())|atom(), channel()) -> term(). +info(Keys, Channel) when is_list(Keys) -> + [{Key, info(Key, Channel)} || Key <- Keys]; + +info(conninfo, #channel{conninfo = ConnInfo}) -> + ConnInfo; +info(conn_state, #channel{conn_state = ConnState}) -> + ConnState; +info(clientinfo, #channel{clientinfo = ClientInfo}) -> + ClientInfo; +info(session, #channel{session = Session}) -> + emqx_misc:maybe_apply(fun emqx_session:info/1, Session); +info(will_msg, #channel{will_msg = WillMsg}) -> + WillMsg; +info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> + ClientId; +info(ctx, #channel{ctx = Ctx}) -> + Ctx. + +-spec(stats(channel()) -> emqx_types:stats()). +stats(#channel{session = undefined})-> + []; +stats(#channel{session = Session})-> + emqx_session:stats(Session). + +set_conn_state(ConnState, Channel) -> + Channel#channel{conn_state = ConnState}. + +enrich_conninfo(?SN_CONNECT_MSG(_Flags, _ProtoId, Duration, _ClientId), + Channel = #channel{conninfo = ConnInfo}) -> + NConnInfo = ConnInfo#{ proto_name => <<"MQTT-SN">> + , proto_ver => <<"1.2">> + , clean_start => true + , keepalive => Duration + , expiry_interval => 0 + }, + {ok, Channel#channel{conninfo = NConnInfo}}. + +run_conn_hooks(Packet, Channel = #channel{ctx = Ctx, + conninfo = ConnInfo}) -> + %% XXX: Assign headers of Packet to ConnProps + ConnProps = #{}, + case run_hooks(Ctx, 'client.connect', [ConnInfo], ConnProps) of + Error = {error, _Reason} -> Error; + _NConnProps -> + {ok, Packet, Channel} + end. + +enrich_clientinfo(Packet, + Channel = #channel{ + conninfo = ConnInfo, + clientinfo = ClientInfo0, + clientinfo_override = Override}) -> + ClientInfo = write_clientinfo( + feedvar(Override, Packet, ConnInfo, ClientInfo0), + ClientInfo0 + ), + {ok, NPacket, NClientInfo} = emqx_misc:pipeline( + [ fun maybe_assign_clientid/2 + %% FIXME: CALL After authentication successfully + , fun fix_mountpoint/2 + ], Packet, ClientInfo + ), + {ok, NPacket, Channel#channel{clientinfo = NClientInfo}}. + +feedvar(Override, Packet, ConnInfo, ClientInfo) -> + Envs = #{ 'ConnInfo' => ConnInfo + , 'ClientInfo' => ClientInfo + , 'Packet' => connect_packet_to_map(Packet) + }, + maps:map(fun(_K, V) -> + Tokens = emqx_rule_utils:preproc_tmpl(V), + emqx_rule_utils:proc_tmpl(Tokens, Envs) + end, Override). + +connect_packet_to_map(#mqtt_sn_message{}) -> + %% XXX: Empty now + #{}. + +write_clientinfo(Override, ClientInfo) -> + Override1 = maps:with([username, password, clientid], Override), + maps:merge(ClientInfo, Override1). + +maybe_assign_clientid(_Packet, ClientInfo = #{clientid := ClientId}) + when ClientId == undefined; + ClientId == <<>> -> + {ok, ClientInfo#{clientid => emqx_guid:to_base62(emqx_guid:gen())}}; + +maybe_assign_clientid(_Packet, ClientInfo) -> + {ok, ClientInfo}. + +fix_mountpoint(_Packet, #{mountpoint := undefined}) -> ok; +fix_mountpoint(_Packet, ClientInfo = #{mountpoint := Mountpoint}) -> + %% TODO: Enrich the varibale replacement???? + %% i.e: ${ClientInfo.auth_result.productKey} + Mountpoint1 = emqx_mountpoint:replvar(Mountpoint, ClientInfo), + {ok, ClientInfo#{mountpoint := Mountpoint1}}. + +set_log_meta(_Packet, #channel{clientinfo = #{clientid := ClientId}}) -> + emqx_logger:set_metadata_clientid(ClientId), + ok. + +maybe_require_will_msg(?SN_CONNECT_MSG(Flags, _, _, _), Channel) -> + #mqtt_sn_flags{will = Will} = Flags, + case Will of + true -> + {error, need_will_msg, Channel}; + _ -> + ok + end. + +auth_connect(_Packet, Channel = #channel{ctx = Ctx, + clientinfo = ClientInfo}) -> + #{clientid := ClientId, + username := Username} = ClientInfo, + case emqx_gateway_ctx:authenticate(Ctx, ClientInfo) of + {ok, NClientInfo} -> + {ok, Channel#channel{clientinfo = NClientInfo}}; + {error, Reason} -> + ?LOG(warning, "Client ~s (Username: '~s') login failed for ~0p", + [ClientId, Username, Reason]), + %% FIXME: ReasonCode? + {error, Reason} + end. + +ensure_connected(Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo}) -> + NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, + ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]), + Channel#channel{conninfo = NConnInfo, + conn_state = connected + }. + +process_connect(Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo + }) -> + SessFun = fun(_,_) -> emqx_session:init(#{max_inflight => 1}) end, + case emqx_gateway_ctx:open_session( + Ctx, + true, + ClientInfo, + ConnInfo, + SessFun + ) of + {ok, #{session := Session, + present := _Present}} -> + handle_out(connack, ?SN_RC_ACCEPTED, + Channel#channel{session = Session}); + {error, Reason} -> + ?LOG(error, "Failed to open session due to ~p", [Reason]), + handle_out(connack, ?SN_RC_FAILED_SESSION, Channel) + end. + +%%-------------------------------------------------------------------- +%% Enrich Keepalive + +ensure_keepalive(Channel = #channel{conninfo = ConnInfo}) -> + ensure_keepalive_timer(maps:get(keepalive, ConnInfo), Channel). + +ensure_keepalive_timer(0, Channel) -> Channel; +ensure_keepalive_timer(Interval, Channel) -> + Keepalive = emqx_keepalive:init(round(timer:seconds(Interval))), + ensure_timer(alive_timer, Channel#channel{keepalive = Keepalive}). + +%%-------------------------------------------------------------------- +%% Handle incoming packet +%%-------------------------------------------------------------------- + +-spec handle_in(emqx_types:packet() | {frame_error, any()}, channel()) + -> {ok, channel()} + | {ok, replies(), channel()} + | {shutdown, Reason :: term(), channel()} + | {shutdown, Reason :: term(), replies(), channel()}. + +%% SEARCHGW, GWINFO +handle_in(?SN_SEARCHGW_MSG(_Radius), + Channel = #channel{gateway_id = GwId}) -> + {ok, {outgoing, ?SN_GWINFO_MSG(GwId, <<>>)}, Channel}; + +handle_in(?SN_ADVERTISE_MSG(_GwId, _Radius), Channel) -> + % ingore + shutdown(normal, Channel); + +handle_in(?SN_PUBLISH_MSG(#mqtt_sn_flags{qos = ?QOS_NEG1, + topic_id_type = TopicIdType + }, + TopicId, _MsgId, Data), + Channel = #channel{conn_state = idle, registry = Registry}) -> + %% FIXME: check enable_qos3 ?? + TopicName = case (TopicIdType =:= ?SN_SHORT_TOPIC) of + true -> + <>; + false -> + emqx_sn_registry:lookup_topic( + Registry, + ?NEG_QOS_CLIENT_ID, + TopicId + ) + end, + _ = case TopicName =/= undefined of + true -> + Msg = emqx_message:make( + ?NEG_QOS_CLIENT_ID, + ?QOS_0, + TopicName, + Data + ), + emqx_broker:publish(Msg); + false -> + ok + end, + ?LOG(debug, "Client id=~p receives a publish with QoS=-1 in idle mode!", + [?NEG_QOS_CLIENT_ID]), + {ok, Channel}; + +handle_in(Pkt = #mqtt_sn_message{type = Type}, + Channel = #channel{conn_state = idle}) + when Type /= ?SN_CONNECT -> + ?LOG(warning, "Receive unknown packet ~0p in idle state", [Pkt]), + shutdown(normal, Channel); + +handle_in(?SN_CONNECT_MSG(_Flags, _ProtoId, _Duration, _ClientId), + Channel = #channel{conn_state = connecting}) -> + ?LOG(warning, "Receive connect packet in connecting state"), + {ok, Channel}; + +handle_in(?SN_CONNECT_MSG(_Flags, _ProtoId, _Duration, _ClientId), + Channel = #channel{conn_state = connected}) -> + {error, unexpected_connect, Channel}; + +handle_in(?SN_WILLTOPIC_EMPTY_MSG, + Channel = #channel{conn_state = connecting}) -> + %% 6.3: + %% Note that if a client wants to delete only its Will data at + %% connection setup, it could send a CONNECT message with + %% 'CleanSession=false' and 'Will=true', + %% and sends an empty WILLTOPIC message to the GW when prompted to do so + case auth_connect(fake_packet, Channel#channel{will_msg = undefined}) of + {ok, NChannel} -> + process_connect(ensure_connected(NChannel)); + {error, ReasonCode} -> + handle_out(connack, ReasonCode, Channel) + end; + +handle_in(?SN_WILLTOPIC_MSG(Flags, Topic), + Channel = #channel{conn_state = connecting, + clientinfo = #{clientid := ClientId}}) -> + #mqtt_sn_flags{qos = QoS, retain = Retain} = Flags, + WillMsg0 = emqx_message:make(ClientId, QoS, Topic, <<>>), + WillMsg = emqx_message:set_flag(retain, Retain, WillMsg0), + NChannel = Channel#channel{will_msg = WillMsg}, + {ok, {outgoing, ?SN_WILLMSGREQ_MSG()}, NChannel}; + +handle_in(?SN_WILLMSG_MSG(Payload), + Channel = #channel{conn_state = connecting, + will_msg = WillMsg}) -> + NWillMsg = WillMsg#message{payload = Payload}, + case auth_connect(fake_packet, Channel#channel{will_msg = NWillMsg}) of + {ok, NChannel} -> + process_connect(ensure_connected(NChannel)); + {error, ReasonCode} -> + handle_out(connack, ReasonCode, Channel) + end; + +handle_in(Packet = ?SN_CONNECT_MSG(_Flags, _ProtoId, _Duration, _ClientId), + Channel) -> + case emqx_misc:pipeline( + [ fun enrich_conninfo/2 + , fun run_conn_hooks/2 + , fun enrich_clientinfo/2 + , fun set_log_meta/2 + %% TODO: How to implement the banned in the gateway instance? + %, fun check_banned/2 + , fun maybe_require_will_msg/2 + , fun auth_connect/2 + ], Packet, Channel#channel{conn_state = connecting}) of + {ok, _NPacket, NChannel} -> + process_connect(ensure_connected(NChannel)); + {error, need_will_msg, NChannel} -> + {ok, {outgoing, ?SN_WILLTOPICREQ_MSG()}, NChannel}; + {error, ReasonCode, NChannel} -> + handle_out(connack, ReasonCode, NChannel) + end; + +handle_in(?SN_REGISTER_MSG(_TopicId, MsgId, TopicName), + Channel = #channel{registry = Registry, + clientinfo = #{clientid := ClientId}}) -> + case emqx_sn_registry:register_topic(Registry, ClientId, TopicName) of + TopicId when is_integer(TopicId) -> + ?LOG(debug, "register TopicName=~p, TopicId=~p", + [TopicName, TopicId]), + AckPacket = ?SN_REGACK_MSG(TopicId, MsgId, ?SN_RC_ACCEPTED), + {ok, {outgoing, AckPacket}, Channel}; + {error, too_large} -> + ?LOG(error, "TopicId is full! TopicName=~p", [TopicName]), + AckPacket = ?SN_REGACK_MSG( + ?SN_INVALID_TOPIC_ID, + MsgId, + ?SN_RC_NOT_SUPPORTED + ), + {ok, {outgoing, AckPacket}, Channel}; + {error, wildcard_topic} -> + ?LOG(error, "wildcard topic can not be registered! TopicName=~p", + [TopicName]), + AckPacket = ?SN_REGACK_MSG( + ?SN_INVALID_TOPIC_ID, + MsgId, + ?SN_RC_NOT_SUPPORTED + ), + {ok, {outgoing, AckPacket}, Channel} + end; + +handle_in(PubPkt = ?SN_PUBLISH_MSG(_Flags, TopicId0, MsgId, _Data), Channel) -> + TopicId = case is_integer(TopicId0) of + true -> TopicId0; + _ -> <> = TopicId0, Id + end, + case emqx_misc:pipeline( + [ fun check_qos3_enable/2 + , fun preproc_pub_pkt/2 + , fun convert_topic_id_to_name/2 + , fun check_pub_authz/2 + , fun convert_pub_to_msg/2 + ], PubPkt, Channel) of + {ok, Msg, NChannel} -> + do_publish(TopicId, MsgId, Msg, NChannel); + {error, ReturnCode, NChannel} -> + handle_out(puback, {TopicId, MsgId, ReturnCode}, NChannel) + end; + +handle_in(?SN_PUBACK_MSG(TopicId, MsgId, ReturnCode), + Channel = #channel{ + ctx = Ctx, + registry = Registry, + session = Session, + clientinfo = ClientInfo = #{clientid := ClientId}}) -> + case ReturnCode of + ?SN_RC_ACCEPTED -> + case emqx_session:puback(MsgId, Session) of + {ok, Msg, NSession} -> + ok = after_message_acked(ClientInfo, Msg, Channel), + {ok, Channel#channel{session = NSession}}; + {ok, Msg, Publishes, NSession} -> + ok = after_message_acked(ClientInfo, Msg, Channel), + handle_out(publish, + Publishes, + Channel#channel{session = NSession}); + {error, ?RC_PACKET_IDENTIFIER_IN_USE} -> + ?LOG(warning, "The PUBACK MsgId ~w is inuse.", + [MsgId]), + ok = metrics_inc(Ctx, 'packets.puback.inuse'), + {ok, Channel}; + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} -> + ?LOG(warning, "The PUBACK MsgId ~w is not found.", + [MsgId]), + ok = metrics_inc(Ctx, 'packets.puback.missed'), + {ok, Channel} + end; + ?SN_RC_INVALID_TOPIC_ID -> + case emqx_sn_registry:lookup_topic(Registry, ClientId, TopicId) of + undefined -> + {ok, Channel}; + TopicName -> + %% notice that this TopicName maybe normal or predefined, + %% involving the predefined topic name in register to + %% enhance the gateway's robustness even inconsistent + %% with MQTT-SN channels + RegPkt = ?SN_REGISTER_MSG(TopicId, MsgId, TopicName), + {ok, {outgoing, RegPkt}, Channel} + end; + _ -> + ?LOG(error, "CAN NOT handle PUBACK ReturnCode=~p", [ReturnCode]), + {ok, Channel} + end; + +handle_in(?SN_PUBREC_MSG(?SN_PUBREC, MsgId), + Channel = #channel{ctx = Ctx, + session = Session, + clientinfo = ClientInfo}) -> + case emqx_session:pubrec(MsgId, Session) of + {ok, Msg, NSession} -> + ok = after_message_acked(ClientInfo, Msg, Channel), + NChannel = Channel#channel{session = NSession}, + handle_out(pubrel, MsgId, NChannel); + {error, ?RC_PACKET_IDENTIFIER_IN_USE} -> + ?LOG(warning, "The PUBREC MsgId ~w is inuse.", [MsgId]), + ok = metrics_inc(Ctx, 'packets.pubrec.inuse'), + handle_out(pubrel, MsgId, Channel); + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} -> + ?LOG(warning, "The PUBREC ~w is not found.", [MsgId]), + ok = metrics_inc(Ctx, 'packets.pubrec.missed'), + handle_out(pubrel, MsgId, Channel) + end; + +handle_in(?SN_PUBREC_MSG(?SN_PUBREL, MsgId), + Channel = #channel{ctx = Ctx, session = Session}) -> + case emqx_session:pubrel(MsgId, Session) of + {ok, NSession} -> + NChannel = Channel#channel{session = NSession}, + handle_out(pubcomp, MsgId, NChannel); + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} -> + ?LOG(warning, "The PUBREL MsgId ~w is not found.", [MsgId]), + ok = metrics_inc(Ctx, 'packets.pubrel.missed'), + handle_out(pubcomp, MsgId, Channel) + end; + +handle_in(?SN_PUBREC_MSG(?SN_PUBCOMP, MsgId), + Channel = #channel{ctx = Ctx, session = Session}) -> + case emqx_session:pubcomp(MsgId, Session) of + {ok, NSession} -> + {ok, Channel#channel{session = NSession}}; + {ok, Publishes, NSession} -> + handle_out(publish, Publishes, + Channel#channel{session = NSession}); + {error, ?RC_PACKET_IDENTIFIER_IN_USE} -> + ok = metrics_inc(Ctx, 'packets.pubcomp.inuse'), + {ok, Channel}; + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} -> + ?LOG(warning, "The PUBCOMP MsgId ~w is not found", [MsgId]), + ok = metrics_inc(Ctx, 'packets.pubcomp.missed'), + {ok, Channel} + end; + +handle_in(SubPkt = ?SN_SUBSCRIBE_MSG(_, MsgId, _), Channel) -> + case emqx_misc:pipeline( + [ fun preproc_subs_type/2 + , fun check_subscribe_authz/2 + , fun do_subscribe/2 + ], SubPkt, Channel) of + {ok, {TopicId, GrantedQoS}, NChannel} -> + SubAck = ?SN_SUBACK_MSG(#mqtt_sn_flags{qos = GrantedQoS}, + TopicId, MsgId, ?SN_RC_ACCEPTED), + {ok, outgoing_and_update(SubAck), NChannel}; + {error, ReturnCode, NChannel} -> + SubAck = ?SN_SUBACK_MSG(#mqtt_sn_flags{}, + ?SN_INVALID_TOPIC_ID, + MsgId, + ReturnCode), + {ok, {outgoing, SubAck}, NChannel} + end; + +handle_in(UnsubPkt = ?SN_UNSUBSCRIBE_MSG(_, MsgId, TopicIdOrName), + Channel) -> + case emqx_misc:pipeline( + [ fun preproc_unsub_type/2 + , fun do_unsubscribe/2 + ], UnsubPkt, Channel) of + {ok, _TopicName, NChannel} -> + UnsubAck = ?SN_UNSUBACK_MSG(MsgId), + {ok, outgoing_and_update(UnsubAck), NChannel}; + {error, Reason, NChannel} -> + ?LOG(warning, "Unsubscribe ~p failed: ~0p", + [TopicIdOrName, Reason]), + %% XXX: Even if it fails, the reply is successful. + UnsubAck = ?SN_UNSUBACK_MSG(MsgId), + {ok, {outgoing, UnsubAck}, NChannel} + end; + +handle_in(?SN_PINGREQ_MSG(_ClientId), + Channel = #channel{conn_state = asleep}) -> + {ok, Outgoing, NChannel} = awake(Channel), + NOutgoings = Outgoing ++ [{outgoing, ?SN_PINGRESP_MSG()}], + {ok, NOutgoings, NChannel}; + +handle_in(?SN_PINGREQ_MSG(_ClientId), Channel) -> + {ok, {outgoing, ?SN_PINGRESP_MSG()}, Channel}; + +handle_in(?SN_PINGRESP_MSG(), Channel) -> + {ok, Channel}; + +handle_in(?SN_DISCONNECT_MSG(Duration), Channel) -> + AckPkt = ?SN_DISCONNECT_MSG(undefined), + case Duration of + undefined -> + shutdown(normal, AckPkt, Channel); + _ -> + %% TODO: asleep mechnisa + {ok, {outgoing, AckPkt}, asleep(Duration, Channel)} + end; + +handle_in(?SN_WILLTOPICUPD_MSG(Flags, Topic), + Channel = #channel{will_msg = WillMsg, + clientinfo = #{clientid := ClientId}}) -> + NWillMsg = case Topic of + undefined -> undefined; + _ -> + update_will_topic(WillMsg, Flags, Topic, ClientId) + end, + AckPkt = ?SN_WILLTOPICRESP_MSG(?SN_RC_ACCEPTED), + {ok, {outgoing, AckPkt}, Channel#channel{will_msg = NWillMsg}}; + +handle_in(?SN_WILLMSGUPD_MSG(Payload), + Channel = #channel{will_msg = WillMsg}) -> + AckPkt = ?SN_WILLMSGRESP_MSG(?SN_RC_ACCEPTED), + NWillMsg = update_will_msg(WillMsg, Payload), + {ok, {outgoing, AckPkt}, Channel#channel{will_msg = NWillMsg}}; + +handle_in({frame_error, Reason}, + Channel = #channel{conn_state = _ConnState}) -> + ?LOG(error, "Unexpected frame error: ~p", [Reason]), + shutdown(Reason, Channel). + +after_message_acked(ClientInfo, Msg, #channel{ctx = Ctx}) -> + ok = metrics_inc(Ctx, 'messages.acked'), + run_hooks_without_metrics(Ctx, + 'message.acked', + [ClientInfo, emqx_message:set_header(puback_props, #{}, Msg)]). + +outgoing_and_update(Pkt) -> + [{outgoing, Pkt}, {event, update}]. + +%%-------------------------------------------------------------------- +%% Handle Publish + +check_qos3_enable(?SN_PUBLISH_MSG(Flags, _, _, _), + #channel{enable_qos3 = EnableQoS3}) -> + #mqtt_sn_flags{qos = QoS} = Flags, + case EnableQoS3 =:= false andalso QoS =:= ?QOS_NEG1 of + true -> + ?LOG(debug, "The enable_qos3 is false, ignore the received " + "publish with QoS=-1 in connected mode!"), + {error, ?SN_RC_NOT_SUPPORTED}; + false -> + ok + end. + +preproc_pub_pkt(?SN_PUBLISH_MSG(Flags, Topic0, _MsgId, Data), + Channel) -> + #mqtt_sn_flags{topic_id_type = TopicIdType} = Flags, + case TopicIdType of + ?SN_NORMAL_TOPIC -> + <> = Topic0, + TopicIndicator = {id, TopicId}, + {ok, {TopicIndicator, Flags, Data}, Channel}; + ?SN_PREDEFINED_TOPIC -> + TopicIndicator = {id, Topic0}, + {ok, {TopicIndicator, Flags, Data}, Channel}; + ?SN_SHORT_TOPIC -> + case emqx_topic:wildcard(Topic0) of + true -> + {error, ?SN_RC_NOT_SUPPORTED}; + false -> + TopicIndicator = {name, Topic0}, + {ok, {TopicIndicator, Flags, Data}, Channel} + end + end. + +convert_topic_id_to_name({{name, TopicName}, Flags, Data}, Channel) -> + {ok, {TopicName, Flags, Data}, Channel}; + +convert_topic_id_to_name({{id, TopicId}, Flags, Data}, + Channel = #channel{ + registry = Registry, + clientinfo = #{clientid := ClientId}} + ) -> + case emqx_sn_registry:lookup_topic(Registry, ClientId, TopicId) of + undefined -> + {error, ?SN_RC_INVALID_TOPIC_ID}; + TopicName -> + {ok, {TopicName, Flags, Data}, Channel} + end. + +check_pub_authz({TopicName, _Flags, _Data}, + #channel{ctx = Ctx, clientinfo = ClientInfo}) -> + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, publish, TopicName) of + allow -> ok; + deny -> {error, ?SN_RC_NOT_AUTHORIZE} + end. + +convert_pub_to_msg({TopicName, Flags, Data}, + Channel = #channel{ + clientinfo = #{clientid := ClientId}}) -> + #mqtt_sn_flags{qos = QoS, dup = Dup, retain = Retain} = Flags, + NewQoS = get_corrected_qos(QoS), + Message = emqx_message:make(ClientId, NewQoS, TopicName, Data), + NMessage = emqx_message:set_flags( + #{dup => Dup, retain => Retain}, + Message + ), + {ok, NMessage, Channel}. + +get_corrected_qos(?QOS_NEG1) -> ?QOS_0; +get_corrected_qos(QoS) -> QoS. + +do_publish(_TopicId, _MsgId, Msg = #message{qos = ?QOS_0}, Channel) -> + _ = emqx_broker:publish(Msg), + {ok, Channel}; + +do_publish(TopicId, MsgId, Msg = #message{qos = ?QOS_1}, Channel) -> + _ = emqx_broker:publish(Msg), + handle_out(puback, {TopicId, MsgId, ?SN_RC_ACCEPTED}, Channel); + +do_publish(TopicId, MsgId, Msg = #message{qos = ?QOS_2}, + Channel = #channel{ctx = Ctx, session = Session}) -> + case emqx_session:publish(MsgId, Msg, Session) of + {ok, _PubRes, NSession} -> + NChannel1 = ensure_timer(await_timer, + Channel#channel{session = NSession} + ), + handle_out(pubrec, MsgId, NChannel1); + {error, ?RC_PACKET_IDENTIFIER_IN_USE} -> + ok = metrics_inc(Ctx, 'packets.publish.inuse'), + %% XXX: Use PUBACK to reply a PUBLISH Error Code + handle_out(puback , {TopicId, MsgId, ?SN_RC_NOT_SUPPORTED}, + Channel); + {error, ?RC_RECEIVE_MAXIMUM_EXCEEDED} -> + ?LOG(warning, "Dropped the qos2 packet ~w " + "due to awaiting_rel is full.", [MsgId]), + ok = metrics_inc(Ctx, 'packets.publish.dropped'), + handle_out(puback, {TopicId, MsgId, ?SN_RC_CONGESTION}, Channel) + end. + +%%-------------------------------------------------------------------- +%% Handle Susbscribe + +preproc_subs_type(?SN_SUBSCRIBE_MSG_TYPE(?SN_NORMAL_TOPIC, + TopicName, QoS), + Channel = #channel{ + registry = Registry, + clientinfo = #{clientid := ClientId} + }) -> + %% If the gateway is able accept the subscription, + %% it assigns a topic id to the received topic name + %% and returns it within a SUBACK message + case emqx_sn_registry:register_topic(Registry, ClientId, TopicName) of + {error, too_large} -> + {error, ?SN_EXCEED_LIMITATION}; + {error, wildcard_topic} -> + %% If the client subscribes to a topic name which contains a + %% wildcard character, the returning SUBACK message will contain + %% the topic id value 0x0000. The GW will the use the registration + %% procedure to inform the client about the to-be-used topic id + %% value when it has the first PUBLISH message with a matching + %% topic name to be sent to the client, see also Section 6.10. + {ok, {?SN_INVALID_TOPIC_ID, TopicName, QoS}, Channel}; + TopicId when is_integer(TopicId) -> + {ok, {TopicId, TopicName, QoS}, Channel} + end; + +preproc_subs_type(?SN_SUBSCRIBE_MSG_TYPE(?SN_PREDEFINED_TOPIC, + TopicId, QoS), + Channel = #channel{ + registry = Registry, + clientinfo = #{clientid := ClientId} + }) -> + case emqx_sn_registry:lookup_topic(Registry, + ClientId, TopicId) of + undefined -> + {error, ?SN_RC_INVALID_TOPIC_ID}; + TopicName -> + {ok, {TopicId, TopicName, QoS}, Channel} + end; + +preproc_subs_type(?SN_SUBSCRIBE_MSG_TYPE(?SN_SHORT_TOPIC, + TopicId, QoS), + Channel) -> + TopicName = case is_binary(TopicId) of + true -> TopicId; + false -> <> + end, + %% XXX: ?SN_INVALID_TOPIC_ID ??? + {ok, {?SN_INVALID_TOPIC_ID, TopicName, QoS}, Channel}; + +preproc_subs_type(?SN_SUBSCRIBE_MSG_TYPE(_Reserved, _TopicId, _QoS), + _Channel) -> + {error, ?SN_RC_NOT_SUPPORTED}. + +check_subscribe_authz({_TopicId, TopicName, _QoS}, + Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) -> + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, TopicName) of + allow -> + {ok, Channel}; + _ -> + {error, ?SN_RC_NOT_AUTHORIZE} + end. + +do_subscribe({TopicId, TopicName, QoS}, + Channel = #channel{ + ctx = Ctx, + session = Session, + clientinfo = ClientInfo + = #{mountpoint := Mountpoint}}) -> + + {TopicName1, SubOpts0} = emqx_topic:parse(TopicName), + TopicFilters = [{TopicName1, SubOpts0#{qos => QoS}}], + case run_hooks(Ctx, 'client.subscribe', + [ClientInfo, #{}], TopicFilters) of + [] -> + ?LOG(warning, "Skip to subscribe ~s, " + "due to 'client.subscribe' denied!", [TopicName]), + {ok, Channel}; + [{NTopicName, NSubOpts}|_] -> + NTopicName1 = emqx_mountpoint:mount(Mountpoint, NTopicName), + NSubOpts1 = maps:merge(?DEFAULT_SUBOPTS, NSubOpts), + case emqx_session:subscribe(ClientInfo, NTopicName1, NSubOpts1, Session) of + {ok, NSession} -> + {ok, {TopicId, QoS}, + Channel#channel{session = NSession}}; + {error, ?RC_QUOTA_EXCEEDED} -> + ?LOG(warning, "Cannot subscribe ~s due to ~s.", + [TopicName, emqx_reason_codes:text(?RC_QUOTA_EXCEEDED)]), + {error, ?SN_EXCEED_LIMITATION} + end + end. + +%%-------------------------------------------------------------------- +%% Handle Unsubscribe + +preproc_unsub_type(?SN_UNSUBSCRIBE_MSG_TYPE(?SN_NORMAL_TOPIC, + TopicName), + Channel) -> + {ok, TopicName, Channel}; +preproc_unsub_type(?SN_UNSUBSCRIBE_MSG_TYPE(?SN_PREDEFINED_TOPIC, + TopicId), + Channel = #channel{ + registry = Registry, + clientinfo = #{clientid := ClientId} + }) -> + case emqx_sn_registry:lookup_topic(Registry, ClientId, + TopicId) of + undefined -> + {error, not_found}; + TopicName -> + {ok, TopicName, Channel} + end; +preproc_unsub_type(?SN_UNSUBSCRIBE_MSG_TYPE(?SN_SHORT_TOPIC, + TopicId), + Channel) -> + TopicName = case is_binary(TopicId) of + true -> TopicId; + false -> <> + end, + {ok, TopicName, Channel}. + +do_unsubscribe(TopicName, + Channel = #channel{ + ctx = Ctx, + session = Session, + clientinfo = ClientInfo + = #{mountpoint := Mountpoint}}) -> + TopicFilters = [emqx_topic:parse(TopicName)], + case run_hooks(Ctx, 'client.unsubscribe', + [ClientInfo, #{}], TopicFilters) of + [] -> + %% Skip to unsubscribe + {ok, Channel}; + [{NTopicName, NSubOpts}|_] -> + NTopicName1 = emqx_mountpoint:mount(Mountpoint, NTopicName), + NSubOpts1 = maps:merge( + emqx_gateway_utils:default_subopts(), + NSubOpts + ), + case emqx_session:unsubscribe(ClientInfo, NTopicName1, + NSubOpts1, Session) of + {ok, NSession} -> + {ok, Channel#channel{session = NSession}}; + {error, ?RC_NO_SUBSCRIPTION_EXISTED} -> + {ok, Channel} + end + end. + +%%-------------------------------------------------------------------- +%% Awake & Asleep + +awake(Channel = #channel{session = Session}) -> + {ok, Publishes, Session1} = emqx_session:replay(Session), + {NPublishes, NSession} = case emqx_session:deliver([], Session1) of + {ok, Session2} -> + {Publishes, Session2}; + {ok, More, Session2} -> + {lists:append(Publishes, More), Session2} + end, + {Packets, NChannel} = do_deliver(NPublishes, + Channel#channel{session = NSession}), + Outgoing = [{outgoing, Packets} || length(Packets) > 0], + {ok, Outgoing, NChannel}. + +asleep(Duration, Channel = #channel{conn_state = asleep}) -> + %% 6.14: The client can also modify its sleep duration + %% by sending a DISCONNECT message with a new value of + %% the sleep duration + ensure_timer(asleep_timer, Duration, + cancel_timer(asleep_timer, Channel) + ); + +asleep(Duration, Channel = #channel{conn_state = connected}) -> + ensure_timer(asleep_timer, Duration, + Channel#channel{conn_state = asleep} + ). + +%%-------------------------------------------------------------------- +%% Handle outgoing packet +%%-------------------------------------------------------------------- + +-spec handle_out(atom(), term(), channel()) + -> {ok, channel()} + | {ok, replies(), channel()} + | {shutdown, Reason :: term(), channel()} + | {shutdown, Reason :: term(), replies(), channel()}. + +handle_out(connack, ?SN_RC_ACCEPTED, + Channel = #channel{ctx = Ctx, conninfo = ConnInfo}) -> + _ = run_hooks(Ctx, 'client.connack', + [ConnInfo, returncode_name(?SN_RC_ACCEPTED)], + #{} + ), + return_connack(?SN_CONNACK_MSG(?SN_RC_ACCEPTED), + ensure_keepalive(Channel)); + +handle_out(connack, ReasonCode, + Channel = #channel{ctx = Ctx, conninfo = ConnInfo}) -> + Reason = returncode_name(ReasonCode), + _ = run_hooks(Ctx, 'client.connack', [ConnInfo, Reason], #{}), + AckPacket = ?SN_CONNACK_MSG(ReasonCode), + shutdown(Reason, AckPacket, Channel); + +handle_out(publish, Publishes, Channel) -> + {Packets, NChannel} = do_deliver(Publishes, Channel), + {ok, {outgoing, Packets}, NChannel}; + +handle_out(puback, {TopicId, MsgId, Rc}, Channel) -> + {ok, {outgoing, ?SN_PUBACK_MSG(TopicId, MsgId, Rc)}, Channel}; + +handle_out(pubrec, MsgId, Channel) -> + {ok, {outgoing, ?SN_PUBREC_MSG(?SN_PUBREC, MsgId)}, Channel}; + +handle_out(pubrel, MsgId, Channel) -> + {ok, {outgoing, ?SN_PUBREC_MSG(?SN_PUBREL, MsgId)}, Channel}; + +handle_out(pubcomp, MsgId, Channel) -> + {ok, {outgoing, ?SN_PUBREC_MSG(?SN_PUBCOMP, MsgId)}, Channel}; + +handle_out(disconnect, RC, Channel) -> + DisPkt = ?SN_DISCONNECT_MSG(undefined), + {ok, [{outgoing, DisPkt}, {close, RC}], Channel}. + +%%-------------------------------------------------------------------- +%% Return ConnAck +%%-------------------------------------------------------------------- + +return_connack(AckPacket, Channel) -> + Replies = [{event, connected}, {outgoing, AckPacket}], + case maybe_resume_session(Channel) of + ignore -> {ok, Replies, Channel}; + {ok, Publishes, NSession} -> + NChannel = Channel#channel{session = NSession, + resuming = false, + pendings = [] + }, + {Packets, NChannel1} = do_deliver(Publishes, NChannel), + Outgoing = [{outgoing, Packets} || length(Packets) > 0], + {ok, Replies ++ Outgoing, NChannel1} + end. + +%%-------------------------------------------------------------------- +%% Maybe Resume Session + +maybe_resume_session(#channel{resuming = false}) -> + ignore; +maybe_resume_session(#channel{session = Session, + resuming = true, + pendings = Pendings}) -> + {ok, Publishes, Session1} = emqx_session:replay(Session), + case emqx_session:deliver(Pendings, Session1) of + {ok, Session2} -> + {ok, Publishes, Session2}; + {ok, More, Session2} -> + {ok, lists:append(Publishes, More), Session2} + end. + +%%-------------------------------------------------------------------- +%% Deliver publish: broker -> client +%%-------------------------------------------------------------------- + +%% return list(emqx_types:packet()) +do_deliver({pubrel, MsgId}, Channel) -> + {[?SN_PUBREC_MSG(?SN_PUBREL, MsgId)], Channel}; + +do_deliver({MsgId, Msg}, + Channel = #channel{ + ctx = Ctx, + clientinfo = ClientInfo + = #{mountpoint := Mountpoint}}) -> + metrics_inc(Ctx, 'messages.delivered'), + Msg1 = run_hooks_without_metrics( + Ctx, + 'message.delivered', + [ClientInfo], + emqx_message:update_expiry(Msg) + ), + Msg2 = emqx_mountpoint:unmount(Mountpoint, Msg1), + Packet = message_to_packet(MsgId, Msg2, Channel), + {[Packet], Channel}; + +do_deliver([Publish], Channel) -> + do_deliver(Publish, Channel); + +do_deliver(Publishes, Channel) when is_list(Publishes) -> + {Packets, NChannel} = + lists:foldl(fun(Publish, {Acc, Chann}) -> + {Packets, NChann} = do_deliver(Publish, Chann), + {Packets ++ Acc, NChann} + end, {[], Channel}, Publishes), + {lists:reverse(Packets), NChannel}. + +message_to_packet(MsgId, Message, + #channel{registry = Registry, + clientinfo = #{clientid := ClientId}}) -> + QoS = emqx_message:qos(Message), + Topic = emqx_message:topic(Message), + Payload = emqx_message:payload(Message), + NMsgId = case QoS of + ?QOS_0 -> 0; + _ -> MsgId + end, + {TopicIdType, NTopicId} = + case emqx_sn_registry:lookup_topic_id(Registry, ClientId, Topic) of + {predef, PredefTopicId} -> + {?SN_PREDEFINED_TOPIC, PredefTopicId}; + TopicId when is_integer(TopicId) -> + {?SN_NORMAL_TOPIC, TopicId}; + undefined -> + {?SN_SHORT_TOPIC, Topic} + end, + Flags = #mqtt_sn_flags{qos = QoS, topic_id_type = TopicIdType}, + ?SN_PUBLISH_MSG(Flags, NTopicId, NMsgId, Payload). + +%%-------------------------------------------------------------------- +%% Handle call +%%-------------------------------------------------------------------- + +-spec handle_call(Req :: term(), channel()) + -> {reply, Reply :: term(), channel()} + | {shutdown, Reason :: term(), Reply :: term(), channel()} + | {shutdown, Reason :: term(), Reply :: term(), + emqx_types:packet(), channel()}. +handle_call(kick, Channel) -> + NChannel = ensure_disconnected(kicked, Channel), + shutdown_and_reply(kicked, ok, NChannel); + +handle_call(discard, Channel) -> + shutdown_and_reply(discarded, ok, Channel); + +%% XXX: No Session Takeover +%handle_call({takeover, 'begin'}, Channel = #channel{session = Session}) -> +% reply(Session, Channel#channel{takeover = true}); +% +%handle_call({takeover, 'end'}, Channel = #channel{session = Session, +% pendings = Pendings}) -> +% ok = emqx_session:takeover(Session), +% %% TODO: Should not drain deliver here (side effect) +% Delivers = emqx_misc:drain_deliver(), +% AllPendings = lists:append(Delivers, Pendings), +% shutdown_and_reply(takeovered, AllPendings, Channel); + +%handle_call(list_authz_cache, Channel) -> +% {reply, emqx_authz_cache:list_authz_cache(), Channel}; + +%% XXX: No Quota Now +% handle_call({quota, Policy}, Channel) -> +% Zone = info(zone, Channel), +% Quota = emqx_limiter:init(Zone, Policy), +% reply(ok, Channel#channel{quota = Quota}); + +handle_call(Req, Channel) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + reply(ignored, Channel). + +%%-------------------------------------------------------------------- +%% Handle Cast +%%-------------------------------------------------------------------- + +-spec handle_cast(Req :: term(), channel()) + -> ok | {ok, channel()} | {shutdown, Reason :: term(), channel()}. +handle_cast(_Req, Channel) -> + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Handle Info +%%-------------------------------------------------------------------- + +-spec handle_info(Info :: term(), channel()) + -> ok | {ok, channel()} | {shutdown, Reason :: term(), channel()}. + +%% XXX: Received from the emqx-management ??? +%handle_info({subscribe, TopicFilters}, Channel ) -> +% {_, NChannel} = lists:foldl( +% fun({TopicFilter, SubOpts}, {_, ChannelAcc}) -> +% do_subscribe(TopicFilter, SubOpts, ChannelAcc) +% end, {[], Channel}, parse_topic_filters(TopicFilters)), +% {ok, NChannel}; +% +%handle_info({unsubscribe, TopicFilters}, Channel) -> +% {_RC, NChannel} = process_unsubscribe(TopicFilters, #{}, Channel), +% {ok, NChannel}; + +handle_info({sock_closed, Reason}, + Channel = #channel{conn_state = idle}) -> + shutdown(Reason, Channel); + +handle_info({sock_closed, Reason}, + Channel = #channel{conn_state = connecting}) -> + shutdown(Reason, Channel); + +handle_info({sock_closed, Reason}, + Channel = #channel{conn_state = connected, + clientinfo = _ClientInfo}) -> + %% XXX: Flapping detect ??? + %% How to get the flapping detect policy ??? + %emqx_zone:enable_flapping_detect(Zone) + % andalso emqx_flapping:detect(ClientInfo), + NChannel = ensure_disconnected(Reason, mabye_publish_will_msg(Channel)), + %% XXX: Session keepper detect here + shutdown(Reason, NChannel); + +handle_info({sock_closed, Reason}, + Channel = #channel{conn_state = disconnected}) -> + ?LOG(error, "Unexpected sock_closed: ~p", [Reason]), + {ok, Channel}; + +handle_info(clean_authz_cache, Channel) -> + ok = emqx_authz_cache:empty_authz_cache(), + {ok, Channel}; + +handle_info(Info, Channel) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Ensure disconnected + +ensure_disconnected(Reason, Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo}) -> + NConnInfo = ConnInfo#{disconnected_at => erlang:system_time(millisecond)}, + ok = run_hooks(Ctx, 'client.disconnected', + [ClientInfo, Reason, NConnInfo]), + Channel#channel{conninfo = NConnInfo, conn_state = disconnected}. + +mabye_publish_will_msg(Channel = #channel{will_msg = undefined}) -> + Channel; +mabye_publish_will_msg(Channel = #channel{will_msg = WillMsg}) -> + ok = publish_will_msg(WillMsg), + Channel#channel{will_msg = undefined}. + +publish_will_msg(Msg) -> + _ = emqx_broker:publish(Msg), + ok. + +%%-------------------------------------------------------------------- +%% Handle Delivers from broker to client +%%-------------------------------------------------------------------- + +-spec handle_deliver(list(emqx_types:deliver()), channel()) + -> {ok, channel()} + | {ok, replies(), channel()}. +handle_deliver(Delivers, Channel = #channel{ + ctx = Ctx, + conn_state = ConnState, + session = Session, + clientinfo = #{clientid := ClientId}}) + when ConnState =:= disconnected; + ConnState =:= asleep -> + NSession = emqx_session:enqueue( + ignore_local(maybe_nack(Delivers), ClientId, Session, Ctx), + Session + ), + {ok, Channel#channel{session = NSession}}; + +handle_deliver(Delivers, Channel = #channel{ + ctx = Ctx, + takeover = true, + pendings = Pendings, + session = Session, + clientinfo = #{clientid := ClientId}}) -> + NPendings = lists:append( + Pendings, + ignore_local(maybe_nack(Delivers), ClientId, Session, Ctx) + ), + {ok, Channel#channel{pendings = NPendings}}; + +handle_deliver(Delivers, Channel = #channel{ + ctx = Ctx, + session = Session, + clientinfo = #{clientid := ClientId}}) -> + case emqx_session:deliver( + ignore_local(Delivers, ClientId, Session, Ctx), + Session + ) of + {ok, Publishes, NSession} -> + NChannel = Channel#channel{session = NSession}, + handle_out(publish, Publishes, + ensure_timer(retry_timer, NChannel)); + {ok, NSession} -> + {ok, Channel#channel{session = NSession}} + end. + +ignore_local(Delivers, Subscriber, Session, Ctx) -> + Subs = emqx_session:info(subscriptions, Session), + lists:dropwhile(fun({deliver, Topic, #message{from = Publisher}}) -> + case maps:find(Topic, Subs) of + {ok, #{nl := 1}} when Subscriber =:= Publisher -> + ok = metrics_inc(Ctx, 'delivery.dropped'), + ok = metrics_inc(Ctx, 'delivery.dropped.no_local'), + true; + _ -> + false + end + end, Delivers). + +%% Nack delivers from shared subscription +maybe_nack(Delivers) -> + lists:filter(fun not_nacked/1, Delivers). + +not_nacked({deliver, _Topic, Msg}) -> + not (emqx_shared_sub:is_ack_required(Msg) + andalso (ok == emqx_shared_sub:nack_no_connection(Msg))). + +%%-------------------------------------------------------------------- +%% Handle timeout +%%-------------------------------------------------------------------- + +-spec handle_timeout(reference(), Msg :: term(), channel()) + -> {ok, channel()} + | {ok, replies(), channel()} + | {shutdown, Reason :: term(), channel()}. + +handle_timeout(_TRef, {keepalive, _StatVal}, + Channel = #channel{keepalive = undefined}) -> + {ok, Channel}; +handle_timeout(_TRef, {keepalive, _StatVal}, + Channel = #channel{conn_state = ConnState}) + when ConnState =:= disconnected; + ConnState =:= asleep -> + {ok, Channel}; +handle_timeout(_TRef, {keepalive, StatVal}, + Channel = #channel{keepalive = Keepalive}) -> + case emqx_keepalive:check(StatVal, Keepalive) of + {ok, NKeepalive} -> + NChannel = Channel#channel{keepalive = NKeepalive}, + {ok, reset_timer(alive_timer, NChannel)}; + {error, timeout} -> + handle_out(disconnect, ?RC_KEEP_ALIVE_TIMEOUT, Channel) + end; + +handle_timeout(_TRef, retry_delivery, + Channel = #channel{conn_state = disconnected}) -> + {ok, Channel}; +handle_timeout(_TRef, retry_delivery, + Channel = #channel{conn_state = asleep}) -> + {ok, reset_timer(retry_timer, Channel)}; +handle_timeout(_TRef, retry_delivery, + Channel = #channel{session = Session}) -> + case emqx_session:retry(Session) of + {ok, NSession} -> + {ok, clean_timer(retry_timer, Channel#channel{session = NSession})}; + {ok, Publishes, Timeout, NSession} -> + NChannel = Channel#channel{session = NSession}, + handle_out(publish, Publishes, reset_timer(retry_timer, Timeout, NChannel)) + end; + +handle_timeout(_TRef, expire_awaiting_rel, + Channel = #channel{conn_state = disconnected}) -> + {ok, Channel}; +handle_timeout(_TRef, expire_awaiting_rel, + Channel = #channel{conn_state = asleep}) -> + {ok, reset_timer(await_timer, Channel)}; +handle_timeout(_TRef, expire_awaiting_rel, + Channel = #channel{session = Session}) -> + case emqx_session:expire(awaiting_rel, Session) of + {ok, NSession} -> + {ok, clean_timer(await_timer, Channel#channel{session = NSession})}; + {ok, Timeout, NSession} -> + {ok, reset_timer(await_timer, Timeout, Channel#channel{session = NSession})} + end; + +handle_timeout(_TRef, expire_asleep, Channel) -> + shutdown(asleep_timeout, Channel); + +handle_timeout(_TRef, Msg, Channel) -> + ?LOG(error, "Unexpected timeout: ~p~n", [Msg]), + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Terminate +%%-------------------------------------------------------------------- + +terminate(_Reason, _Channel) -> + ok. + +reply(Reply, Channel) -> + {reply, Reply, Channel}. + +shutdown(Reason, Channel) -> + {shutdown, Reason, Channel}. + +shutdown(Reason, AckFrame, Channel) -> + {shutdown, Reason, AckFrame, Channel}. + +shutdown_and_reply(Reason, Reply, Channel) -> + {shutdown, Reason, Reply, Channel}. + +%%-------------------------------------------------------------------- +%% Will + +update_will_topic(undefined, #mqtt_sn_flags{qos = QoS, retain = Retain}, + Topic, ClientId) -> + WillMsg0 = emqx_message:make(ClientId, QoS, Topic, <<>>), + emqx_message:set_flag(retain, Retain, WillMsg0); +update_will_topic(Will, #mqtt_sn_flags{qos = QoS, retain = Retain}, + Topic, _ClientId) -> + emqx_message:set_flag(retain, Retain, + Will#message{qos = QoS, topic = Topic}). + +update_will_msg(Will, Payload) -> + Will#message{payload = Payload}. + +%%-------------------------------------------------------------------- +%% Timer + +cancel_timer(Name, Channel = #channel{timers = Timers}) -> + case maps:get(Name, Timers, undefined) of + undefined -> + Channel; + TRef -> + emqx_misc:cancel_timer(TRef), + Channel#channel{timers = maps:without([Name], Timers)} + end. + +ensure_timer(Name, Channel = #channel{timers = Timers}) -> + TRef = maps:get(Name, Timers, undefined), + Time = interval(Name, Channel), + case TRef == undefined andalso is_integer(Time) andalso Time > 0 of + true -> ensure_timer(Name, Time, Channel); + false -> Channel %% Timer disabled or exists + end. + +ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> + Msg = maps:get(Name, ?TIMER_TABLE), + TRef = emqx_misc:start_timer(Time, Msg), + Channel#channel{timers = Timers#{Name => TRef}}. + +reset_timer(Name, Channel) -> + ensure_timer(Name, clean_timer(Name, Channel)). + +reset_timer(Name, Time, Channel) -> + ensure_timer(Name, Time, clean_timer(Name, Channel)). + +clean_timer(Name, Channel = #channel{timers = Timers}) -> + Channel#channel{timers = maps:remove(Name, Timers)}. + +interval(alive_timer, #channel{keepalive = KeepAlive}) -> + emqx_keepalive:info(interval, KeepAlive); +interval(retry_timer, #channel{session = Session}) -> + emqx_session:info(retry_interval, Session); +interval(await_timer, #channel{session = Session}) -> + emqx_session:info(await_rel_timeout, Session). + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- + +run_hooks(Ctx, Name, Args) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name), + emqx_hooks:run(Name, Args). + +run_hooks(Ctx, Name, Args, Acc) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name), + emqx_hooks:run_fold(Name, Args, Acc). + +run_hooks_without_metrics(_Ctx, Name, Args) -> + emqx_hooks:run(Name, Args). + +run_hooks_without_metrics(_Ctx, Name, Args, Acc) -> + emqx_hooks:run_fold(Name, Args, Acc). + +metrics_inc(Ctx, Name) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name). + +returncode_name(?SN_RC_ACCEPTED) -> accepted; +returncode_name(?SN_RC_CONGESTION) -> rejected_congestion; +returncode_name(?SN_RC_INVALID_TOPIC_ID) -> rejected_invaild_topic_id; +returncode_name(?SN_RC_NOT_SUPPORTED) -> rejected_not_supported; +returncode_name(?SN_RC_NOT_AUTHORIZE) -> rejected_not_authorize; +returncode_name(?SN_RC_FAILED_SESSION) -> rejected_failed_open_session; +returncode_name(?SN_EXCEED_LIMITATION) -> rejected_exceed_limitation; +returncode_name(_) -> accepted. diff --git a/apps/emqx_sn/src/emqx_sn_frame.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl similarity index 74% rename from apps/emqx_sn/src/emqx_sn_frame.erl rename to apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl index eed32803d..32d1a21a2 100644 --- a/apps/emqx_sn/src/emqx_sn_frame.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl @@ -15,31 +15,54 @@ %% limitations under the License. %%-------------------------------------------------------------------- +%% @doc The frame parser for MQTT-SN protocol -module(emqx_sn_frame). --include("emqx_sn.hrl"). +-behavior(emqx_gateway_frame). --export([ parse/1 - , serialize/1 +-include("src/mqttsn/include/emqx_sn.hrl"). + +-export([ initial_parse_state/1 + , serialize_opts/0 + , parse/2 + , serialize_pkt/2 , message_type/1 , format/1 + , type/1 + , is_message/1 ]). -define(flag, 1/binary). -define(byte, 8/big-integer). -define(short, 16/big-integer). +-type parse_state() :: #{}. +-type serialize_opts() :: #{}. + +-export_type([ parse_state/0 + , serialize_opts/0 + ]). + +%%-------------------------------------------------------------------- +%% Initial + +initial_parse_state(_) -> + #{}. + +serialize_opts() -> + #{}. + %%-------------------------------------------------------------------- %% Parse MQTT-SN Message %%-------------------------------------------------------------------- -parse(<<16#01:?byte, Len:?short, Type:?byte, Var/binary>>) -> - parse(Type, Len - 4, Var); -parse(<>) -> - parse(Type, Len - 2, Var). +parse(<<16#01:?byte, Len:?short, Type:?byte, Var/binary>>, _State) -> + {ok, parse(Type, Len - 4, Var), <<>>, _State}; +parse(<>, _State) -> + {ok, parse(Type, Len - 2, Var), <<>>, _State}. parse(Type, Len, Var) when Len =:= size(Var) -> - {ok, #mqtt_sn_message{type = Type, variable = parse_var(Type, Var)}}; + #mqtt_sn_message{type = Type, variable = parse_var(Type, Var)}; parse(_Type, _Len, _Var) -> error(malformed_message_len). @@ -127,70 +150,70 @@ parse_topic(2#11, Topic) -> Topic. %% Serialize MQTT-SN Message %%-------------------------------------------------------------------- -serialize(#mqtt_sn_message{type = Type, variable = Var}) -> - VarBin = serialize(Type, Var), VarLen = size(VarBin), +serialize_pkt(#mqtt_sn_message{type = Type, variable = Var}, Opts) -> + VarBin = serialize(Type, Var, Opts), VarLen = size(VarBin), if VarLen < 254 -> <<(VarLen + 2), Type, VarBin/binary>>; true -> <<16#01, (VarLen + 4):?short, Type, VarBin/binary>> end. -serialize(?SN_ADVERTISE, {GwId, Duration}) -> +serialize(?SN_ADVERTISE, {GwId, Duration}, _Opts) -> <>; -serialize(?SN_SEARCHGW, Radius) -> +serialize(?SN_SEARCHGW, Radius, _Opts) -> <>; -serialize(?SN_GWINFO, {GwId, GwAdd}) -> +serialize(?SN_GWINFO, {GwId, GwAdd}, _Opts) -> <>; -serialize(?SN_CONNECT, {Flags, ProtocolId, Duration, ClientId}) -> +serialize(?SN_CONNECT, {Flags, ProtocolId, Duration, ClientId}, _Opts) -> <<(serialize_flags(Flags))/binary, ProtocolId, Duration:?short, ClientId/binary>>; -serialize(?SN_CONNACK, ReturnCode) -> +serialize(?SN_CONNACK, ReturnCode, _Opts) -> <>; -serialize(?SN_WILLTOPICREQ, _) -> +serialize(?SN_WILLTOPICREQ, _, _Opts) -> <<>>; -serialize(?SN_WILLTOPIC, undefined) -> +serialize(?SN_WILLTOPIC, undefined, _Opts) -> <<>>; -serialize(?SN_WILLTOPIC, {Flags, Topic}) -> +serialize(?SN_WILLTOPIC, {Flags, Topic}, _Opts) -> %% The WillTopic must a short topic name <<(serialize_flags(Flags))/binary, Topic/binary>>; -serialize(?SN_WILLMSGREQ, _) -> +serialize(?SN_WILLMSGREQ, _, _Opts) -> <<>>; -serialize(?SN_WILLMSG, WillMsg) -> +serialize(?SN_WILLMSG, WillMsg, _Opts) -> WillMsg; -serialize(?SN_REGISTER, {TopicId, MsgId, TopicName}) -> +serialize(?SN_REGISTER, {TopicId, MsgId, TopicName}, _Opts) -> <>; -serialize(?SN_REGACK, {TopicId, MsgId, ReturnCode}) -> +serialize(?SN_REGACK, {TopicId, MsgId, ReturnCode}, _Opts) -> <>; -serialize(?SN_PUBLISH, {Flags=#mqtt_sn_flags{topic_id_type = ?SN_NORMAL_TOPIC}, TopicId, MsgId, Data}) -> +serialize(?SN_PUBLISH, {Flags=#mqtt_sn_flags{topic_id_type = ?SN_NORMAL_TOPIC}, TopicId, MsgId, Data}, _Opts) -> <<(serialize_flags(Flags))/binary, TopicId:?short, MsgId:?short, Data/binary>>; -serialize(?SN_PUBLISH, {Flags=#mqtt_sn_flags{topic_id_type = ?SN_PREDEFINED_TOPIC}, TopicId, MsgId, Data}) -> +serialize(?SN_PUBLISH, {Flags=#mqtt_sn_flags{topic_id_type = ?SN_PREDEFINED_TOPIC}, TopicId, MsgId, Data}, _Opts) -> <<(serialize_flags(Flags))/binary, TopicId:?short, MsgId:?short, Data/binary>>; -serialize(?SN_PUBLISH, {Flags=#mqtt_sn_flags{topic_id_type = ?SN_SHORT_TOPIC}, STopicName, MsgId, Data}) -> +serialize(?SN_PUBLISH, {Flags=#mqtt_sn_flags{topic_id_type = ?SN_SHORT_TOPIC}, STopicName, MsgId, Data}, _Opts) -> <<(serialize_flags(Flags))/binary, STopicName:2/binary, MsgId:?short, Data/binary>>; -serialize(?SN_PUBACK, {TopicId, MsgId, ReturnCode}) -> +serialize(?SN_PUBACK, {TopicId, MsgId, ReturnCode}, _Opts) -> <>; -serialize(PubRec, MsgId) when PubRec == ?SN_PUBREC; PubRec == ?SN_PUBREL; PubRec == ?SN_PUBCOMP -> +serialize(PubRec, MsgId, _Opts) when PubRec == ?SN_PUBREC; PubRec == ?SN_PUBREL; PubRec == ?SN_PUBCOMP -> <>; -serialize(Sub, {Flags = #mqtt_sn_flags{topic_id_type = IdType}, MsgId, Topic}) +serialize(Sub, {Flags = #mqtt_sn_flags{topic_id_type = IdType}, MsgId, Topic}, _Opts) when Sub == ?SN_SUBSCRIBE; Sub == ?SN_UNSUBSCRIBE -> <<(serialize_flags(Flags))/binary, MsgId:16, (serialize_topic(IdType, Topic))/binary>>; -serialize(?SN_SUBACK, {Flags, TopicId, MsgId, ReturnCode}) -> +serialize(?SN_SUBACK, {Flags, TopicId, MsgId, ReturnCode}, _Opts) -> <<(serialize_flags(Flags))/binary, TopicId:?short, MsgId:?short, ReturnCode>>; -serialize(?SN_UNSUBACK, MsgId) -> +serialize(?SN_UNSUBACK, MsgId, _Opts) -> <>; -serialize(?SN_PINGREQ, ClientId) -> +serialize(?SN_PINGREQ, ClientId, _Opts) -> ClientId; -serialize(?SN_PINGRESP, _) -> +serialize(?SN_PINGRESP, _, _Opts) -> <<>>; -serialize(?SN_WILLTOPICUPD, {Flags, WillTopic}) -> +serialize(?SN_WILLTOPICUPD, {Flags, WillTopic}, _Opts) -> <<(serialize_flags(Flags))/binary, WillTopic/binary>>; -serialize(?SN_WILLMSGUPD, WillMsg) -> +serialize(?SN_WILLMSGUPD, WillMsg, _Opts) -> WillMsg; -serialize(?SN_WILLTOPICRESP, ReturnCode) -> +serialize(?SN_WILLTOPICRESP, ReturnCode, _Opts) -> <>; -serialize(?SN_WILLMSGRESP, ReturnCode) -> +serialize(?SN_WILLMSGRESP, ReturnCode, _Opts) -> <>; -serialize(?SN_DISCONNECT, undefined) -> +serialize(?SN_DISCONNECT, undefined, _Opts) -> <<>>; -serialize(?SN_DISCONNECT, Duration) -> +serialize(?SN_DISCONNECT, Duration, _Opts) -> <>. serialize_flags(#mqtt_sn_flags{dup = Dup, qos = QoS, retain = Retain, will = Will, @@ -305,3 +328,38 @@ format_flag(#mqtt_sn_flags{dup = Dup, qos = QoS, retain = Retain, will = Will, c [Dup, QoS, Retain, Will, CleanStart, TopicType]); format_flag(_Flag) -> "invalid flag". +is_message(#mqtt_sn_message{type = Type}) + when Type == ?SN_PUBLISH -> + true; +is_message(_) -> + false. + +type(#mqtt_sn_message{type = Type}) -> + type(Type); +type(?SN_ADVERTISE) -> advertise; +type(?SN_SEARCHGW) -> serachgw; +type(?SN_GWINFO) -> gwinfo; +type(?SN_CONNECT) -> connect; +type(?SN_CONNACK) -> connack; +type(?SN_WILLTOPICREQ) -> willtopicreq; +type(?SN_WILLTOPIC) -> willtopic; +type(?SN_WILLMSGREQ) -> willmsgreq; +type(?SN_WILLMSG) -> willmsg; +type(?SN_REGISTER) -> register; +type(?SN_REGACK) -> regack; +type(?SN_PUBLISH) -> publish; +type(?SN_PUBACK) -> puback; +type(?SN_PUBCOMP) -> pubcomp; +type(?SN_PUBREC) -> pubrec; +type(?SN_PUBREL) -> pubrel; +type(?SN_SUBSCRIBE) -> subscribe; +type(?SN_SUBACK) -> suback; +type(?SN_UNSUBSCRIBE) -> unsubscribe; +type(?SN_UNSUBACK) -> unsuback; +type(?SN_PINGREQ) -> pingreq; +type(?SN_PINGRESP) -> pingresp; +type(?SN_DISCONNECT) -> disconnect; +type(?SN_WILLTOPICUPD) -> willtopicupd; +type(?SN_WILLTOPICRESP) -> willtopicresp; +type(?SN_WILLMSGUPD) -> willmsgupd; +type(?SN_WILLMSGRESP) -> willmsgresp. diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl new file mode 100644 index 000000000..bd27c9fb5 --- /dev/null +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl @@ -0,0 +1,165 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The MQTT-SN Gateway Implement interface +-module(emqx_sn_impl). + +-behavior(emqx_gateway_impl). + +%% APIs +-export([ load/0 + , unload/0 + ]). + +-export([]). + +-export([ init/1 + , on_insta_create/3 + , on_insta_update/4 + , on_insta_destroy/3 + ]). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +load() -> + RegistryOptions = [ {cbkmod, ?MODULE} + ], + emqx_gateway_registry:load(mqttsn, RegistryOptions, []). + +unload() -> + emqx_gateway_registry:unload(mqttsn). + +init(_) -> + GwState = #{}, + {ok, GwState}. + +%%-------------------------------------------------------------------- +%% emqx_gateway_registry callbacks +%%-------------------------------------------------------------------- + +on_insta_create(_Insta = #{ id := InstaId, + rawconf := RawConf + }, Ctx, _GwState) -> + + %% We Also need to start `emqx_sn_broadcast` & + %% `emqx_sn_registry` process + SnGwId = maps:get(gateway_id, RawConf), + case maps:get(broadcast, RawConf) of + false -> + ok; + true -> + %% FIXME: + Port = 1884, + _ = emqx_sn_broadcast:start_link(SnGwId, Port), ok + end, + + PredefTopics = maps:get(predefined, RawConf), + {ok, RegistrySvr} = emqx_sn_registry:start_link(InstaId, PredefTopics), + + NRawConf = maps:without( + [broadcast, predefined], + RawConf#{registry => emqx_sn_registry:lookup_name(RegistrySvr)} + ), + Listeners = emqx_gateway_utils:normalize_rawconf(NRawConf), + + ListenerPids = lists:map(fun(Lis) -> + start_listener(InstaId, Ctx, Lis) + end, Listeners), + {ok, ListenerPids, _InstaState = #{ctx => Ctx}}. + +on_insta_update(NewInsta, OldInsta, GwInstaState = #{ctx := Ctx}, GwState) -> + InstaId = maps:get(id, NewInsta), + try + %% XXX: 1. How hot-upgrade the changes ??? + %% XXX: 2. Check the New confs first before destroy old instance ??? + on_insta_destroy(OldInsta, GwInstaState, GwState), + on_insta_create(NewInsta, Ctx, GwState) + catch + Class : Reason : Stk -> + logger:error("Failed to update stomp instance ~s; " + "reason: {~0p, ~0p} stacktrace: ~0p", + [InstaId, Class, Reason, Stk]), + {error, {Class, Reason}} + end. + +on_insta_destroy(_Insta = #{ id := InstaId, + rawconf := RawConf + }, _GwInstaState, _GwState) -> + Listeners = emqx_gateway_utils:normalize_rawconf(RawConf), + lists:foreach(fun(Lis) -> + stop_listener(InstaId, Lis) + end, Listeners). + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +start_listener(InstaId, Ctx, {Type, ListenOn, SocketOpts, Cfg}) -> + ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), + case start_listener(InstaId, Ctx, Type, ListenOn, SocketOpts, Cfg) of + {ok, Pid} -> + io:format("Start mqttsn ~s:~s listener on ~s successfully.~n", + [InstaId, Type, ListenOnStr]), + Pid; + {error, Reason} -> + io:format(standard_error, + "Failed to start mqttsn ~s:~s listener on ~s: ~0p~n", + [InstaId, Type, ListenOnStr, Reason]), + throw({badconf, Reason}) + end. + +start_listener(InstaId, Ctx, Type, ListenOn, SocketOpts, Cfg) -> + Name = name(InstaId, Type), + NCfg = Cfg#{ + ctx => Ctx, + frame_mod => emqx_sn_frame, + chann_mod => emqx_sn_channel + }, + esockd:open_udp(Name, ListenOn, merge_default(SocketOpts), + {emqx_gateway_conn, start_link, [NCfg]}). + +name(InstaId, Type) -> + list_to_atom(lists:concat([InstaId, ":", Type])). + +merge_default(Options) -> + Default = emqx_gateway_utils:default_udp_options(), + case lists:keytake(udp_options, 1, Options) of + {value, {udp_options, TcpOpts}, Options1} -> + [{udp_options, emqx_misc:merge_opts(Default, TcpOpts)} + | Options1]; + false -> + [{udp_options, Default} | Options] + end. + +stop_listener(InstaId, {Type, ListenOn, SocketOpts, Cfg}) -> + StopRet = stop_listener(InstaId, Type, ListenOn, SocketOpts, Cfg), + ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), + case StopRet of + ok -> io:format("Stop mqttsn ~s:~s listener on ~s successfully.~n", + [InstaId, Type, ListenOnStr]); + {error, Reason} -> + io:format(standard_error, + "Failed to stop mqttsn ~s:~s listener on ~s: ~0p~n", + [InstaId, Type, ListenOnStr, Reason] + ) + end, + StopRet. + +stop_listener(InstaId, Type, ListenOn, _SocketOpts, _Cfg) -> + Name = name(InstaId, Type), + esockd:close(Name, ListenOn). diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl new file mode 100644 index 000000000..1249831cc --- /dev/null +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl @@ -0,0 +1,241 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The MQTT-SN Topic Registry +%% +%% XXX: +-module(emqx_sn_registry). + +-behaviour(gen_server). + +-include("src/mqttsn/include/emqx_sn.hrl"). + +-define(LOG(Level, Format, Args), + emqx_logger:Level("MQTT-SN(registry): " ++ Format, Args)). + +-export([ start_link/2 + ]). + +-export([ register_topic/3 + , unregister_topic/2 + ]). + +-export([ lookup_topic/3 + , lookup_topic_id/3 + ]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-export([lookup_name/1]). + +-define(SN_SHARD, emqx_sn_shard). + +-record(state, {tabname, max_predef_topic_id = 0}). + +-record(emqx_sn_registry, {key, value}). + +%% Mnesia bootstrap +%-export([mnesia/1]). + +%-boot_mnesia({mnesia, [boot]}). +%-copy_mnesia({mnesia, [copy]}). + +%-rlog_shard({?SN_SHARD, ?TAB}). + +%%% @doc Create or replicate tables. +%-spec(mnesia(boot | copy) -> ok). +%mnesia(boot) -> +% %% Optimize storage +% StoreProps = [{ets, [{read_concurrency, true}]}], +% ok = ekka_mnesia:create_table(?MODULE, [ +% {attributes, record_info(fields, emqx_sn_registry)}, +% {ram_copies, [node()]}, +% {storage_properties, StoreProps}]); +% +%mnesia(copy) -> +% ok = ekka_mnesia:copy_table(?MODULE, ram_copies). + +-type registry() :: {Tab :: atom(), + RegistryPid :: pid()}. + +%%----------------------------------------------------------------------------- + +-spec start_link(atom(), list()) + -> ignore + | {ok, pid()} + | {error, Reason :: term()}. +start_link(InstaId, PredefTopics) -> + gen_server:start_link(?MODULE, [InstaId, PredefTopics], []). + +-spec register_topic(registry(), emqx_types:clientid(), emqx_types:topic()) + -> integer() + | {error, term()}. +register_topic({_, Pid}, ClientId, TopicName) when is_binary(TopicName) -> + case emqx_topic:wildcard(TopicName) of + false -> + gen_server:call(Pid, {register, ClientId, TopicName}); + %% TopicId: in case of “accepted” the value that will be used as topic + %% id by the gateway when sending PUBLISH messages to the client (not + %% relevant in case of subscriptions to a short topic name or to a topic + %% name which contains wildcard characters) + true -> {error, wildcard_topic} + end. + +-spec lookup_topic(registry(), emqx_types:clientid(), pos_integer()) + -> undefined + | binary(). +lookup_topic({Tab, _}, ClientId, TopicId) when is_integer(TopicId) -> + case lookup_element(Tab, {predef, TopicId}, 3) of + undefined -> + lookup_element(Tab, {ClientId, TopicId}, 3); + Topic -> Topic + end. + +-spec lookup_topic_id(registry(), emqx_types:clientid(), emqx_types:topic()) + -> undefined + | pos_integer() + | {predef, integer()}. +lookup_topic_id({Tab, _}, ClientId, TopicName) when is_binary(TopicName) -> + case lookup_element(Tab, {predef, TopicName}, 3) of + undefined -> + lookup_element(Tab, {ClientId, TopicName}, 3); + TopicId -> + {predef, TopicId} + end. + +%% @private +lookup_element(Tab, Key, Pos) -> + try ets:lookup_element(Tab, Key, Pos) catch error:badarg -> undefined end. + +-spec unregister_topic(registry(), emqx_types:clientid()) -> ok. +unregister_topic({_, Pid}, ClientId) -> + gen_server:call(Pid, {unregister, ClientId}). + +lookup_name(Pid) -> + gen_server:call(Pid, name). + +%%----------------------------------------------------------------------------- + +name(InstaId) -> + list_to_atom(lists:concat([emqx_sn_, InstaId, '_registry'])). + +init([InstaId, PredefTopics]) -> + %% {predef, TopicId} -> TopicName + %% {predef, TopicName} -> TopicId + %% {ClientId, TopicId} -> TopicName + %% {ClientId, TopicName} -> TopicId + Tab = name(InstaId), + ok = ekka_mnesia:create_table(Tab, [ + {ram_copies, [node()]}, + {record_name, emqx_sn_registry}, + {attributes, record_info(fields, emqx_sn_registry)}, + {storage_properties, [{ets, [{read_concurrency, true}]}]} + ]), + ok = ekka_mnesia:copy_table(Tab, ram_copies), + % FIXME: + %ok = ekka_rlog:wait_for_shards([?CM_SHARD], infinity), + MaxPredefId = lists:foldl( + fun(#{id := TopicId, topic := TopicName0}, AccId) -> + TopicName = iolist_to_binary(TopicName0), + ekka_mnesia:dirty_write(Tab, #emqx_sn_registry{ + key = {predef, TopicId}, + value = TopicName} + ), + ekka_mnesia:dirty_write(Tab, #emqx_sn_registry{ + key = {predef, TopicName}, + value = TopicId} + ), + if TopicId > AccId -> TopicId; true -> AccId end + end, 0, PredefTopics), + {ok, #state{tabname = Tab, max_predef_topic_id = MaxPredefId}}. + +handle_call({register, ClientId, TopicName}, _From, + State = #state{tabname = Tab, max_predef_topic_id = PredefId}) -> + case lookup_topic_id({Tab, self()}, ClientId, TopicName) of + {predef, PredefTopicId} when is_integer(PredefTopicId) -> + {reply, PredefTopicId, State}; + TopicId when is_integer(TopicId) -> + {reply, TopicId, State}; + undefined -> + case next_topic_id(Tab, PredefId, ClientId) of + TopicId when TopicId >= 16#FFFF -> + {reply, {error, too_large}, State}; + TopicId -> + Fun = fun() -> + mnesia:write(Tab, #emqx_sn_registry{ + key = {ClientId, next_topic_id}, + value = TopicId + 1}, write), + mnesia:write(Tab, #emqx_sn_registry{ + key = {ClientId, TopicName}, + value = TopicId}, write), + mnesia:write(Tab, #emqx_sn_registry{ + key = {ClientId, TopicId}, + value = TopicName}, write) + end, + case ekka_mnesia:transaction(?SN_SHARD, Fun) of + {atomic, ok} -> + {reply, TopicId, State}; + {aborted, Error} -> + {reply, {error, Error}, State} + end + end + end; + +handle_call({unregister, ClientId}, _From, State = #state{tabname = Tab}) -> + Registry = mnesia:dirty_match_object( + Tab, + {emqx_sn_registry, {ClientId, '_'}, '_'} + ), + lists:foreach(fun(R) -> + ekka_mnesia:dirty_delete_object(Tab, R) + end, Registry), + {reply, ok, State}; + +handle_call(name, _From, State = #state{tabname = Tab}) -> + {reply, {Tab, self()}, State}; + +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected request: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + ?LOG(error, "Unexpected msg: ~p", [Msg]), + {noreply, State}. + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%----------------------------------------------------------------------------- + +next_topic_id(Tab, PredefId, ClientId) -> + case mnesia:dirty_read(Tab, {ClientId, next_topic_id}) of + [#emqx_sn_registry{value = Id}] -> Id; + [] -> PredefId + 1 + end. diff --git a/apps/emqx_sn/include/emqx_sn.hrl b/apps/emqx_gateway/src/mqttsn/include/emqx_sn.hrl similarity index 89% rename from apps/emqx_sn/include/emqx_sn.hrl rename to apps/emqx_gateway/src/mqttsn/include/emqx_sn.hrl index 29c5b2c86..ca0433011 100644 --- a/apps/emqx_sn/include/emqx_sn.hrl +++ b/apps/emqx_gateway/src/mqttsn/include/emqx_sn.hrl @@ -49,14 +49,18 @@ -type(mqtt_sn_type() :: ?SN_ADVERTISE..?SN_WILLMSGRESP). --define(SN_RC_ACCEPTED, 16#00). +-define(SN_RC_ACCEPTED, 16#00). -define(SN_RC_CONGESTION, 16#01). -define(SN_RC_INVALID_TOPIC_ID, 16#02). -define(SN_RC_NOT_SUPPORTED, 16#03). +%% Custome Reason code by emqx +-define(SN_RC_NOT_AUTHORIZE, 16#04). +-define(SN_RC_FAILED_SESSION, 16#05). +-define(SN_EXCEED_LIMITATION, 16#06). -define(QOS_NEG1, 3). --type(mqtt_sn_return_code() :: ?SN_RC_ACCEPTED .. ?SN_RC_NOT_SUPPORTED). +-type(mqtt_sn_return_code() :: ?SN_RC_ACCEPTED .. ?SN_EXCEED_LIMITATION). %%-------------------------------------------------------------------- %% MQTT-SN Message @@ -139,6 +143,12 @@ #mqtt_sn_message{type = ?SN_SUBSCRIBE, variable = {Flags, MsgId, Topic}}). +-define(SN_SUBSCRIBE_MSG_TYPE(Type, Topic, QoS), + #mqtt_sn_message{type = ?SN_SUBSCRIBE, + variable = { + #mqtt_sn_flags{qos = QoS, topic_id_type = Type}, + _, Topic}}). + -define(SN_SUBACK_MSG(Flags, TopicId, MsgId, ReturnCode), #mqtt_sn_message{type = ?SN_SUBACK, variable = {Flags, TopicId, MsgId, ReturnCode}}). @@ -147,6 +157,12 @@ #mqtt_sn_message{type = ?SN_UNSUBSCRIBE, variable = {Flags, MsgId, Topic}}). +-define(SN_UNSUBSCRIBE_MSG_TYPE(Type, Topic), + #mqtt_sn_message{type = ?SN_UNSUBSCRIBE, + variable = { + #mqtt_sn_flags{topic_id_type = Type}, + _, Topic}}). + -define(SN_UNSUBACK_MSG(MsgId), #mqtt_sn_message{type = ?SN_UNSUBACK, variable = MsgId}). @@ -181,5 +197,4 @@ -define(SN_SHORT_TOPIC, 2). -define(SN_RESERVED_TOPIC, 3). - -define(SN_INVALID_TOPIC_ID, 0). diff --git a/apps/emqx_stomp/README.md b/apps/emqx_gateway/src/stomp/README.md similarity index 90% rename from apps/emqx_stomp/README.md rename to apps/emqx_gateway/src/stomp/README.md index ec841b1e6..c5736a755 100644 --- a/apps/emqx_stomp/README.md +++ b/apps/emqx_gateway/src/stomp/README.md @@ -1,13 +1,12 @@ -emqx-stomp -========== +# emqx-stomp + The plugin adds STOMP 1.0/1.1/1.2 protocol supports to the EMQ X broker. The STOMP clients could PubSub to the MQTT clients. -Configuration -------------- +## Configuration etc/emqx_stomp.conf @@ -58,20 +57,17 @@ stomp.frame.max_header_length = 1024 stomp.frame.max_body_length = 8192 ``` -Load the Plugin ---------------- +## Load the Plugin ``` ./bin/emqx_ctl plugins load emqx_stomp ``` -License -------- +## License Apache License Version 2.0 -Author ------- +## Author EMQ X Team. diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl new file mode 100644 index 000000000..ef6e21e66 --- /dev/null +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl @@ -0,0 +1,1063 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_stomp_channel). + +-behavior(emqx_gateway_channel). + +-include("src/stomp/include/emqx_stomp.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-logger_header("[Stomp-Proto]"). + +-import(proplists, [get_value/2, get_value/3]). + +%% API +-export([ info/1 + , info/2 + , stats/1 + ]). + +-export([ init/2 + , handle_in/2 + , handle_out/3 + , handle_deliver/2 + , handle_timeout/3 + , terminate/2 + , set_conn_state/2 + ]). + +-export([ handle_call/2 + , handle_cast/2 + , handle_info/2 + ]). + +%% for trans callback +-export([ handle_recv_send_frame/2 + , handle_recv_ack_frame/2 + , handle_recv_nack_frame/2 + ]). + +-record(channel, { + %% Context + ctx :: emqx_gateway_ctx:context(), + %% Stomp Connection Info + conninfo :: emqx_types:conninfo(), + %% Stomp Client Info + clientinfo :: emqx_types:clientinfo(), + %% Session + session :: undefined | map(), + %% ClientInfo override specs + clientinfo_override :: map(), + %% Connection Channel + conn_state :: conn_state(), + %% Heartbeat + heartbeat :: emqx_stomp_heartbeat:heartbeat(), + %% Subscriptions + subscriptions = [], + %% Timer + timers :: #{atom() => disable | undefined | reference()}, + %% Transaction + transaction :: #{binary() => list()} + }). + +-type(channel() :: #channel{}). + +-type(conn_state() :: idle | connecting | connected | disconnected). + +-type(reply() :: {outgoing, stomp_frame()} + | {outgoing, [stomp_frame()]} + | {event, conn_state()|updated} + | {close, Reason :: atom()}). + +-type(replies() :: reply() | [reply()]). + +-define(TIMER_TABLE, #{ + incoming_timer => keepalive, + outgoing_timer => keepalive_send, + clean_trans_timer => clean_trans + }). + +-define(TRANS_TIMEOUT, 60000). + +-define(DEFAULT_OVERRIDE, + #{ clientid => <<"">> %% Generate clientid by default + , username => <<"${Packet.headers.login}">> + , password => <<"${Packet.headers.passcode}">> + }). + +-define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]). + +%%-------------------------------------------------------------------- +%% Init the channel +%%-------------------------------------------------------------------- + +%% @doc Init protocol +init(ConnInfo = #{peername := {PeerHost, _}, + sockname := {_, SockPort}}, Option) -> + Peercert = maps:get(peercert, ConnInfo, undefined), + Mountpoint = maps:get(mountpoint, Option, undefined), + ClientInfo = setting_peercert_infos( + Peercert, + #{ zone => default + , listener => mqtt_tcp + , protocol => stomp + , peerhost => PeerHost + , sockport => SockPort + , clientid => undefined + , username => undefined + , is_bridge => false + , is_superuser => false + , mountpoint => Mountpoint + } + ), + + Ctx = maps:get(ctx, Option), + Override = maps:merge(?DEFAULT_OVERRIDE, + maps:get(clientinfo_override, Option, #{}) + ), + #channel{ ctx = Ctx + , conninfo = ConnInfo + , clientinfo = ClientInfo + , clientinfo_override = Override + , timers = #{} + , transaction = #{} + , conn_state = idle + }. + +setting_peercert_infos(NoSSL, ClientInfo) + when NoSSL =:= nossl; + NoSSL =:= undefined -> + ClientInfo; +setting_peercert_infos(Peercert, ClientInfo) -> + {DN, CN} = {esockd_peercert:subject(Peercert), + esockd_peercert:common_name(Peercert)}, + ClientInfo#{dn => DN, cn => CN}. + +-spec info(channel()) -> emqx_types:infos(). +info(Channel) -> + maps:from_list(info(?INFO_KEYS, Channel)). + +-spec(info(list(atom())|atom(), channel()) -> term()). +info(Keys, Channel) when is_list(Keys) -> + [{Key, info(Key, Channel)} || Key <- Keys]; + +info(conninfo, #channel{conninfo = ConnInfo}) -> + ConnInfo; +info(conn_state, #channel{conn_state = ConnState}) -> + ConnState; +info(clientinfo, #channel{clientinfo = ClientInfo}) -> + ClientInfo; +info(session, _) -> + #{}; +info(will_msg, _) -> + undefined; +info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> + ClientId; +info(ctx, #channel{ctx = Ctx}) -> + Ctx. + +-spec(stats(channel()) -> emqx_types:stats()). +stats(#channel{subscriptions = Subs}) -> + [{subscriptions_cnt, length(Subs)}]. + +set_conn_state(ConnState, Channel) -> + Channel#channel{conn_state = ConnState}. + +enrich_conninfo(_Packet, + Channel = #channel{conninfo = ConnInfo}) -> + %% XXX: How enrich more infos? + NConnInfo = ConnInfo#{ proto_name => <<"STOMP">> + , proto_ver => <<"1.2">> + , clean_start => true + , keepalive => 0 + , expiry_interval => 0 + }, + {ok, Channel#channel{conninfo = NConnInfo}}. + +run_conn_hooks(Packet, Channel = #channel{ctx = Ctx, + conninfo = ConnInfo}) -> + %% XXX: Assign headers of Packet to ConnProps + ConnProps = #{}, + case run_hooks(Ctx, 'client.connect', [ConnInfo], ConnProps) of + Error = {error, _Reason} -> Error; + _NConnProps -> + {ok, Packet, Channel} + end. + +negotiate_version(#stomp_frame{headers = Headers}, + Channel = #channel{conninfo = ConnInfo}) -> + %% XXX: + case do_negotiate_version(header(<<"accept-version">>, Headers)) of + {ok, Version} -> + {ok, Channel#channel{conninfo = ConnInfo#{proto_ver => Version}}}; + {error, Reason}-> + {error, Reason} + end. + +enrich_clientinfo(Packet, + Channel = #channel{ + conninfo = ConnInfo, + clientinfo = ClientInfo0, + clientinfo_override = Override}) -> + ClientInfo = write_clientinfo( + feedvar(Override, Packet, ConnInfo, ClientInfo0), + ClientInfo0 + ), + {ok, NPacket, NClientInfo} = emqx_misc:pipeline( + [ fun maybe_assign_clientid/2 + , fun parse_heartbeat/2 + %% FIXME: CALL After authentication successfully + , fun fix_mountpoint/2 + ], Packet, ClientInfo + ), + {ok, NPacket, Channel#channel{clientinfo = NClientInfo}}. + +feedvar(Override, Packet, ConnInfo, ClientInfo) -> + Envs = #{ 'ConnInfo' => ConnInfo + , 'ClientInfo' => ClientInfo + , 'Packet' => connect_packet_to_map(Packet) + }, + maps:map(fun(_K, V) -> + Tokens = emqx_rule_utils:preproc_tmpl(V), + emqx_rule_utils:proc_tmpl(Tokens, Envs) + end, Override). + +connect_packet_to_map(#stomp_frame{headers = Headers}) -> + #{headers => maps:from_list(Headers)}. + +write_clientinfo(Override, ClientInfo) -> + Override1 = maps:with([username, password, clientid], Override), + maps:merge(ClientInfo, Override1). + +maybe_assign_clientid(_Packet, ClientInfo = #{clientid := ClientId}) + when ClientId == undefined; + ClientId == <<>> -> + {ok, ClientInfo#{clientid => emqx_guid:to_base62(emqx_guid:gen())}}; + +maybe_assign_clientid(_Packet, ClientInfo) -> + {ok, ClientInfo}. + +parse_heartbeat(#stomp_frame{headers = Headers}, ClientInfo) -> + Heartbeat0 = header(<<"heart-beat">>, Headers, <<"0,0">>), + CxCy = re:split(Heartbeat0, <<",">>, [{return, list}]), + Heartbeat = list_to_tuple([list_to_integer(S) || S <- CxCy]), + {ok, ClientInfo#{heartbeat => Heartbeat}}. + +fix_mountpoint(_Packet, #{mountpoint := undefined}) -> ok; +fix_mountpoint(_Packet, ClientInfo = #{mountpoint := Mountpoint}) -> + %% TODO: Enrich the varibale replacement???? + %% i.e: ${ClientInfo.auth_result.productKey} + Mountpoint1 = emqx_mountpoint:replvar(Mountpoint, ClientInfo), + {ok, ClientInfo#{mountpoint := Mountpoint1}}. + +set_log_meta(_Packet, #channel{clientinfo = #{clientid := ClientId}}) -> + emqx_logger:set_metadata_clientid(ClientId), + ok. + +auth_connect(_Packet, Channel = #channel{ctx = Ctx, + clientinfo = ClientInfo}) -> + #{clientid := ClientId, + username := Username} = ClientInfo, + case emqx_gateway_ctx:authenticate(Ctx, ClientInfo) of + {ok, NClientInfo} -> + {ok, Channel#channel{clientinfo = NClientInfo}}; + {error, Reason} -> + ?LOG(warning, "Client ~s (Username: '~s') login failed for ~0p", + [ClientId, Username, Reason]), + {error, Reason} + end. + +ensure_connected(Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo}) -> + NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, + ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]), + Channel#channel{conninfo = NConnInfo, + conn_state = connected + }. + +process_connect(Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo + }) -> + SessFun = fun(_,_) -> #{} end, + case emqx_gateway_ctx:open_session( + Ctx, + true, + ClientInfo, + ConnInfo, + SessFun + ) of + {ok, #{session := Session}} -> + #{proto_ver := Version} = ConnInfo, + #{heartbeat := Heartbeat} = ClientInfo, + Headers = [{<<"version">>, Version}, + {<<"heart-beat">>, reverse_heartbeats(Heartbeat)}], + handle_out(connected, Headers, Channel#channel{session = Session}); + {error, Reason} -> + ?LOG(error, "Failed to open session du to ~p", [Reason]), + Headers = [{<<"version">>, <<"1.0,1.1,1.2">>}, + {<<"content-type">>, <<"text/plain">>}], + handle_out(connerr, {Headers, undefined, <<"Not Authenticated">>}, Channel) + end. + +%%-------------------------------------------------------------------- +%% Handle incoming packet +%%-------------------------------------------------------------------- + +-spec handle_in(stomp_frame() | {frame_error, any()}, channel()) + -> {ok, channel()} + | {ok, replies(), channel()} + | {shutdown, Reason :: term(), channel()} + | {shutdown, Reason :: term(), replies(), channel()}. + +handle_in(Frame = ?PACKET(?CMD_STOMP), Channel) -> + handle_in(Frame#stomp_frame{command = <<"CONNECT">>}, Channel); + +handle_in(?PACKET(?CMD_CONNECT), + Channel = #channel{conn_state = connected}) -> + {error, unexpected_connect, Channel}; + +handle_in(Packet = ?PACKET(?CMD_CONNECT), Channel) -> + case emqx_misc:pipeline( + [ fun enrich_conninfo/2 + , fun run_conn_hooks/2 + , fun negotiate_version/2 + , fun enrich_clientinfo/2 + , fun set_log_meta/2 + %% TODO: How to implement the banned in the gateway instance? + %, fun check_banned/2 + , fun auth_connect/2 + ], Packet, Channel#channel{conn_state = connecting}) of + {ok, _NPacket, NChannel} -> + process_connect(ensure_connected(NChannel)); + {error, ReasonCode, NChannel} -> + ErrMsg = io_lib:format("Login Failed: ~s", [ReasonCode]), + handle_out(connerr, {[], undefined, ErrMsg}, NChannel) + end; + +handle_in(Frame = ?PACKET(?CMD_SEND, Headers), + Channel = #channel{ + ctx = Ctx, + clientinfo = ClientInfo + }) -> + Topic = header(<<"destination">>, Headers), + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, publish, Topic) of + deny -> + handle_out(error, {receipt_id(Headers), "Authorization Deny"}, Channel); + allow -> + case header(<<"transaction">>, Headers) of + undefined -> + handle_recv_send_frame(Frame, Channel); + TxId -> + add_action(TxId, {fun ?MODULE:handle_recv_send_frame/2, [Frame]}, receipt_id(Headers), Channel) + end + end; + +handle_in(?PACKET(?CMD_SUBSCRIBE, Headers), + Channel = #channel{ + ctx = Ctx, + subscriptions = Subs, + clientinfo = ClientInfo + }) -> + + SubId = header(<<"id">>, Headers), + Topic = header(<<"destination">>, Headers), + Ack = header(<<"ack">>, Headers, <<"auto">>), + case emqx_misc:pipeline( + [ fun parse_topic_filter/2 + , fun check_subscribed_status/2 + , fun check_sub_acl/2 + ], {SubId, Topic}, Channel) of + {ok, {_, TopicFilter}, NChannel} -> + TopicFilters = [TopicFilter], + NTopicFilters = run_hooks(Ctx, 'client.subscribe', + [ClientInfo, #{}], TopicFilters), + case do_subscribe(NTopicFilters, NChannel) of + [] -> + ErrMsg = "Permission denied", + handle_out(error, {receipt_id(Headers), ErrMsg}, Channel); + [MountedTopic|_] -> + NChannel1 = NChannel#channel{ + subscriptions = [{SubId, MountedTopic, Ack} + | Subs] + }, + handle_out(receipt, receipt_id(Headers), NChannel1) + end; + {error, ErrMsg, NChannel} -> + ?LOG(error, "Failed to subscribe topic ~s, reason: ~s", + [Topic, ErrMsg]), + handle_out(error, {receipt_id(Headers), ErrMsg}, NChannel) + end; + +handle_in(?PACKET(?CMD_UNSUBSCRIBE, Headers), + Channel = #channel{ + ctx = Ctx, + clientinfo = ClientInfo + = #{mountpoint := Mountpoint}, + subscriptions = Subs}) -> + SubId = header(<<"id">>, Headers), + {ok, NChannel} = + case lists:keyfind(SubId, 1, Subs) of + {SubId, MountedTopic, _Ack} -> + Topic = emqx_mountpoint:unmount(Mountpoint, MountedTopic), + %% XXX: eval the return topics? + _ = run_hooks(Ctx, 'client.unsubscribe', + [ClientInfo, #{}], [{Topic, #{}}]), + ok = emqx_broker:unsubscribe(MountedTopic), + _ = run_hooks(Ctx, 'session.unsubscribe', + [ClientInfo, MountedTopic, #{}]), + {ok, Channel#channel{subscriptions = lists:keydelete(SubId, 1, Subs)}}; + false -> + {ok, Channel} + end, + handle_out(receipt, receipt_id(Headers), NChannel); + +%% XXX: How to ack a frame ??? +handle_in(Frame = ?PACKET(?CMD_ACK, Headers), Channel) -> + case header(<<"transaction">>, Headers) of + undefined -> handle_recv_ack_frame(Frame, Channel); + TxId -> add_action(TxId, {fun ?MODULE:handle_recv_ack_frame/2, [Frame]}, receipt_id(Headers), Channel) + end; + +%% NACK +%% id:12345 +%% transaction:tx1 +%% +%% ^@ +handle_in(Frame = ?PACKET(?CMD_NACK, Headers), Channel) -> + case header(<<"transaction">>, Headers) of + undefined -> handle_recv_nack_frame(Frame, Channel); + TxId -> add_action(TxId, {fun ?MODULE:handle_recv_nack_frame/2, [Frame]}, receipt_id(Headers), Channel) + end; + +%% The transaction header is REQUIRED, and the transaction identifier +%% will be used for SEND, COMMIT, ABORT, ACK, and NACK frames to bind +%% them to the named transaction. +%% +%% BEGIN +%% transaction:tx1 +%% +%% ^@ +handle_in(?PACKET(?CMD_BEGIN, Headers), + Channel = #channel{transaction = Trans}) -> + TxId = header(<<"transaction">>, Headers), + case maps:get(TxId, Trans, undefined) of + undefined -> + StartedAt = erlang:system_time(millisecond), + NChannel = ensure_clean_trans_timer( + Channel#channel{ + transaction = Trans#{TxId => {StartedAt, []}}} + ), + handle_out(receipt, receipt_id(Headers), NChannel); + _ -> + ErrMsg = ["Transaction ", TxId, " already started"], + handle_out(error, {receipt_id(Headers), ErrMsg}, Channel) + end; + +%% COMMIT +%% transaction:tx1 +%% +%% ^@ +handle_in(?PACKET(?CMD_COMMIT, Headers), Channel) -> + with_transaction(Headers, Channel, fun(TxId, Actions) -> + Chann0 = remove_trans(TxId, Channel), + case trans_pipeline(lists:reverse(Actions), [], Chann0) of + {ok, Outgoings, Chann1} -> + maybe_outgoing_receipt(receipt_id(Headers), Outgoings, Chann1); + {error, Reason, Chann1} -> + %% FIXME: atomic for transaction ?? + ErrMsg = io_lib:format("Execute transaction ~s falied: ~0p", + [TxId, Reason] + ), + handle_out(error, {receipt_id(Headers), ErrMsg}, Chann1) + end + end); + +%% ABORT +%% transaction:tx1 +%% +%% ^@ +handle_in(?PACKET(?CMD_ABORT, Headers), + Channel = #channel{transaction = Trans}) -> + with_transaction(Headers, Channel, fun(Id, _Actions) -> + NChannel = Channel#channel{transaction = maps:remove(Id, Trans)}, + handle_out(receipt, receipt_id(Headers), NChannel) + end); + +handle_in(?PACKET(?CMD_DISCONNECT, Headers), Channel) -> + shutdown_with_recepit(normal, receipt_id(Headers), Channel); + +handle_in({frame_error, Reason}, Channel = #channel{conn_state = _ConnState}) -> + ?LOG(error, "Unexpected frame error: ~p", [Reason]), + shutdown(Reason, Channel). + +with_transaction(Headers, Channel = #channel{transaction = Trans}, Fun) -> + Id = header(<<"transaction">>, Headers), + ReceiptId = receipt_id(Headers), + case maps:get(Id, Trans, undefined) of + {_, Actions} -> + Fun(Id, Actions); + _ -> + ErrMsg = ["Transaction ", Id, " not found"], + handle_out(error, {ReceiptId, ErrMsg}, Channel) + end. + +remove_trans(Id, Channel = #channel{transaction = Trans}) -> + Channel#channel{transaction = maps:remove(Id, Trans)}. + +trans_pipeline([], Outgoings, Channel) -> + {ok, Outgoings, Channel}; + +trans_pipeline([{Func, Args}|More], Outgoings, Channel) -> + case erlang:apply(Func, Args ++ [Channel]) of + {ok, NChannel} -> + trans_pipeline(More, Outgoings, NChannel); + {ok, Outgoings1, NChannel} -> + trans_pipeline(More, Outgoings ++ Outgoings1, NChannel); + {error, Reason} -> + {error, Reason, Channel} + end. + +%%-------------------------------------------------------------------- +%% Subs + +parse_topic_filter({SubId, Topic}, Channel) -> + TopicFilter = emqx_topic:parse(Topic), + {ok, {SubId, TopicFilter}, Channel}. + +check_subscribed_status({SubId, TopicFilter}, + #channel{ + subscriptions = Subs, + clientinfo = #{mountpoint := Mountpoint} + }) -> + MountedTopic = emqx_mountpoint:mount(Mountpoint, TopicFilter), + case lists:keyfind(SubId, 1, Subs) of + {SubId, MountedTopic, _Ack} -> + ok; + {SubId, _OtherTopic, _Ack} -> + {error, "Conflict subscribe id"}; + false -> + ok + end. + +check_sub_acl({_SubId, TopicFilter}, + #channel{ + ctx = Ctx, + clientinfo = ClientInfo}) -> + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, TopicFilter) of + deny -> {error, "ACL Deny"}; + allow -> ok + end. + +do_subscribe(TopicFilters, Channel) -> + do_subscribe(TopicFilters, Channel, []). + +do_subscribe([], _Channel, Acc) -> + lists:reverse(Acc); +do_subscribe([{TopicFilter, Option}|More], + Channel = #channel{ + ctx = Ctx, + clientinfo = ClientInfo + = #{clientid := ClientId, + mountpoint := Mountpoint}}, Acc) -> + SubOpts = maps:merge(emqx_gateway_utils:default_subopts(), Option), + MountedTopic = emqx_mountpoint:mount(Mountpoint, TopicFilter), + _ = emqx_broker:subscribe(MountedTopic, ClientId, SubOpts), + run_hooks(Ctx, 'session.subscribed', [ClientInfo, MountedTopic, SubOpts]), + do_subscribe(More, Channel, [MountedTopic|Acc]). + +%%-------------------------------------------------------------------- +%% Handle outgoing packet +%%-------------------------------------------------------------------- + +-spec(handle_out(atom(), term(), channel()) + -> {ok, channel()} + | {ok, replies(), channel()} + | {shutdown, Reason :: term(), channel()} + | {shutdown, Reason :: term(), replies(), channel()}). + +handle_out(connerr, {Headers, ReceiptId, ErrMsg}, Channel) -> + Frame = error_frame(Headers, ReceiptId, ErrMsg), + shutdown(ErrMsg, Frame, Channel); + +handle_out(error, {ReceiptId, ErrMsg}, Channel) -> + Frame = error_frame(ReceiptId, ErrMsg), + {ok, {outgoing, Frame}, Channel}; + +handle_out(connected, Headers, Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo + }) -> + %% XXX: connection_accepted is not defined by stomp protocol + _ = run_hooks(Ctx, 'client.connack', [ConnInfo, connection_accepted, []]), + Replies = [{outgoing, connected_frame(Headers)}, + {event, connected} + ], + {ok, Replies, ensure_heartbeart_timer(Channel)}; + +handle_out(receipt, undefined, Channel) -> + {ok, Channel}; +handle_out(receipt, ReceiptId, Channel) -> + Frame = receipt_frame(ReceiptId), + {ok, {outgoing, Frame}, Channel}. + +%%-------------------------------------------------------------------- +%% Handle call +%%-------------------------------------------------------------------- + +-spec(handle_call(Req :: term(), channel()) + -> {reply, Reply :: term(), channel()} + | {shutdown, Reason :: term(), Reply :: term(), channel()} + | {shutdown, Reason :: term(), Reply :: term(), stomp_frame(), channel()}). +handle_call(kick, Channel) -> + NChannel = ensure_disconnected(kicked, Channel), + Frame = error_frame(undefined, <<"Kicked out">>), + shutdown_and_reply(kicked, ok, Frame, NChannel); + +handle_call(discard, Channel) -> + Frame = error_frame(undefined, <<"Discarded">>), + shutdown_and_reply(discarded, ok, Frame, Channel); + +%% XXX: No Session Takeover +%handle_call({takeover, 'begin'}, Channel = #channel{session = Session}) -> +% reply(Session, Channel#channel{takeover = true}); +% +%handle_call({takeover, 'end'}, Channel = #channel{session = Session, +% pendings = Pendings}) -> +% ok = emqx_session:takeover(Session), +% %% TODO: Should not drain deliver here (side effect) +% Delivers = emqx_misc:drain_deliver(), +% AllPendings = lists:append(Delivers, Pendings), +% shutdown_and_reply(takeovered, AllPendings, Channel); + +handle_call(list_authz_cache, Channel) -> + %% This won't work + {reply, emqx_authz_cache:list_authz_cache(default), Channel}; + +%% XXX: No Quota Now +% handle_call({quota, Policy}, Channel) -> +% Zone = info(zone, Channel), +% Quota = emqx_limiter:init(Zone, Policy), +% reply(ok, Channel#channel{quota = Quota}); + +handle_call(Req, Channel) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + reply(ignored, Channel). + + +%%-------------------------------------------------------------------- +%% Handle cast +%%-------------------------------------------------------------------- + +-spec handle_cast(Req :: term(), channel()) + -> ok | {ok, channel()} | {shutdown, Reason :: term(), channel()}. +handle_cast(_Req, Channel) -> + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Handle Info +%%-------------------------------------------------------------------- + +-spec(handle_info(Info :: term(), channel()) + -> ok | {ok, channel()} | {shutdown, Reason :: term(), channel()}). + +%% XXX: Received from the emqx-management ??? +%handle_info({subscribe, TopicFilters}, Channel ) -> +% {_, NChannel} = lists:foldl( +% fun({TopicFilter, SubOpts}, {_, ChannelAcc}) -> +% do_subscribe(TopicFilter, SubOpts, ChannelAcc) +% end, {[], Channel}, parse_topic_filters(TopicFilters)), +% {ok, NChannel}; +% +%handle_info({unsubscribe, TopicFilters}, Channel) -> +% {_RC, NChannel} = process_unsubscribe(TopicFilters, #{}, Channel), +% {ok, NChannel}; + +handle_info({sock_closed, Reason}, + Channel = #channel{conn_state = idle}) -> + shutdown(Reason, Channel); + +handle_info({sock_closed, Reason}, + Channel = #channel{conn_state = connecting}) -> + shutdown(Reason, Channel); + +handle_info({sock_closed, Reason}, + Channel = #channel{conn_state = connected, + clientinfo = _ClientInfo}) -> + %% XXX: Flapping detect ??? + %% How to get the flapping detect policy ??? + %emqx_zone:enable_flapping_detect(Zone) + % andalso emqx_flapping:detect(ClientInfo), + NChannel = ensure_disconnected(Reason, Channel), + %% XXX: Session keepper detect here + shutdown(Reason, NChannel); + +handle_info({sock_closed, Reason}, + Channel = #channel{conn_state = disconnected}) -> + ?LOG(error, "Unexpected sock_closed: ~p", [Reason]), + {ok, Channel}; + +handle_info(clean_authz_cache, Channel) -> + ok = emqx_authz_cache:empty_authz_cache(), + {ok, Channel}; + +handle_info(Info, Channel) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Ensure disconnected + +ensure_disconnected(Reason, Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo}) -> + NConnInfo = ConnInfo#{disconnected_at => erlang:system_time(millisecond)}, + ok = run_hooks(Ctx, 'client.disconnected', + [ClientInfo, Reason, NConnInfo]), + Channel#channel{conninfo = NConnInfo, conn_state = disconnected}. + +%%-------------------------------------------------------------------- +%% Handle Delivers from broker to client +%%-------------------------------------------------------------------- + +-spec(handle_deliver(list(emqx_types:deliver()), channel()) + -> {ok, channel()} + | {ok, replies(), channel()}). + +handle_deliver(Delivers, + Channel = #channel{ + ctx = Ctx, + clientinfo = ClientInfo, + subscriptions = Subs + }) -> + + %% TODO: Re-deliver ??? + %% Shared-subscription support ??? + + Frames0 = lists:foldl(fun({_, _, Message}, Acc) -> + Topic0 = emqx_message:topic(Message), + case lists:keyfind(Topic0, 2, Subs) of + {Id, Topic, Ack} -> + %% XXX: refactor later + metrics_inc('messages.delivered', Channel), + NMessage = run_hooks_without_metrics( + Ctx, + 'message.delivered', + [ClientInfo], + Message + ), + Topic = emqx_message:topic(NMessage), + Headers = emqx_message:get_headers(NMessage), + Payload = emqx_message:payload(NMessage), + Headers0 = [{<<"subscription">>, Id}, + {<<"message-id">>, next_msgid()}, + {<<"destination">>, Topic}, + {<<"content-type">>, <<"text/plain">>}], + Headers1 = case Ack of + _ when Ack =:= <<"client">>; + Ack =:= <<"client-individual">> -> + Headers0 ++ [{<<"ack">>, next_ackid()}]; + _ -> + Headers0 + end, + Frame = #stomp_frame{command = <<"MESSAGE">>, + headers = Headers1 ++ maps:get(stomp_headers, Headers, []), + body = Payload + }, + [Frame|Acc]; + false -> + ?LOG(error, "Dropped message ~0p due to not found " + "subscription id for ~s", + [Message, emqx_message:topic(Message)]), + metrics_inc('delivery.dropped', Channel), + metrics_inc('delivery.dropped.no_subid', Channel), + Acc + end + end, [], Delivers), + {ok, [{outgoing, lists:reverse(Frames0)}], Channel}. + +%%-------------------------------------------------------------------- +%% Handle timeout +%%-------------------------------------------------------------------- + +-spec(handle_timeout(reference(), Msg :: term(), channel()) + -> {ok, channel()} + | {ok, replies(), channel()} + | {shutdown, Reason :: term(), channel()}). + +handle_timeout(_TRef, {keepalive, NewVal}, + Channel = #channel{heartbeat = HrtBt}) -> + case emqx_stomp_heartbeat:check(incoming, NewVal, HrtBt) of + {error, timeout} -> + shutdown(heartbeat_timeout, Channel); + {ok, NHrtBt} -> + {ok, reset_timer(incoming_timer, + Channel#channel{heartbeat = NHrtBt} + )} + end; + +handle_timeout(_TRef, {keepalive_send, NewVal}, + Channel = #channel{heartbeat = HrtBt}) -> + case emqx_stomp_heartbeat:check(outgoing, NewVal, HrtBt) of + {error, timeout} -> + NHrtBt = emqx_stomp_heartbeat:reset(outgoing, NewVal, HrtBt), + NChannel = Channel#channel{heartbeat = NHrtBt}, + {ok, {outgoing, emqx_stomp_frame:make(?CMD_HEARTBEAT)}, + reset_timer(outgoing_timer, NChannel)}; + {ok, NHrtBt} -> + {ok, reset_timer(outgoing_timer, + Channel#channel{heartbeat = NHrtBt} + )} + end; + +handle_timeout(_TRef, clean_trans, Channel = #channel{transaction = Trans}) -> + Now = erlang:system_time(millisecond), + NTrans = maps:filter(fun(_, {StartedAt, _}) -> + StartedAt + ?TRANS_TIMEOUT < Now + end, Trans), + {ok, ensure_clean_trans_timer(Channel#channel{transaction = NTrans})}. + +%%-------------------------------------------------------------------- +%% Terminate +%%-------------------------------------------------------------------- + +terminate(Reason, #channel{ + ctx = Ctx, + session = Session, + clientinfo = ClientInfo}) -> + run_hooks(Ctx, 'session.terminated', [ClientInfo, Reason, Session]). + +reply(Reply, Channel) -> + {reply, Reply, Channel}. + +shutdown(Reason, Channel) -> + {shutdown, Reason, Channel}. + +shutdown_with_recepit(Reason, ReceiptId, Channel) -> + case ReceiptId of + undefined -> + {shutdown, Reason, Channel}; + _ -> + {shutdown, Reason, receipt_frame(ReceiptId), Channel} + end. + +shutdown(Reason, AckFrame, Channel) -> + {shutdown, Reason, AckFrame, Channel}. + +%shutdown_and_reply(Reason, Reply, Channel) -> +% {shutdown, Reason, Reply, Channel}. + +shutdown_and_reply(Reason, Reply, OutPkt, Channel) -> + {shutdown, Reason, Reply, OutPkt, Channel}. + +do_negotiate_version(undefined) -> + {ok, <<"1.0">>}; + +do_negotiate_version(Accepts) -> + do_negotiate_version( + ?STOMP_VER, + lists:reverse(lists:sort(binary:split(Accepts, <<",">>, [global]))) + ). + +do_negotiate_version(Ver, []) -> + {error, <<"Supported protocol versions < ", Ver/binary>>}; +do_negotiate_version(Ver, [AcceptVer|_]) when Ver >= AcceptVer -> + {ok, AcceptVer}; +do_negotiate_version(Ver, [_|T]) -> + do_negotiate_version(Ver, T). + +header(Name, Headers) -> + get_value(Name, Headers). +header(Name, Headers, Val) -> + get_value(Name, Headers, Val). + +connected_frame(Headers) -> + emqx_stomp_frame:make(<<"CONNECTED">>, Headers). + +receipt_frame(ReceiptId) -> + emqx_stomp_frame:make(<<"RECEIPT">>, [{<<"receipt-id">>, ReceiptId}]). + +error_frame(ReceiptId, Msg) -> + error_frame([{<<"content-type">>, <<"text/plain">>}], ReceiptId, Msg). + +error_frame(Headers, undefined, Msg) -> + emqx_stomp_frame:make(<<"ERROR">>, Headers, Msg); +error_frame(Headers, ReceiptId, Msg) -> + emqx_stomp_frame:make(<<"ERROR">>, [{<<"receipt-id">>, ReceiptId} | Headers], Msg). + +next_msgid() -> + MsgId = case get(msgid) of + undefined -> 1; + I -> I + end, + put(msgid, MsgId + 1), + MsgId. + +next_ackid() -> + AckId = case get(ackid) of + undefined -> 1; + I -> I + end, + put(ackid, AckId + 1), + AckId. + +frame2message(?PACKET(?CMD_SEND, Headers, Body), + #channel{ + conninfo = #{proto_ver := ProtoVer}, + clientinfo = #{ + protocol := Protocol, + clientid := ClientId, + username := Username, + peerhost := PeerHost, + mountpoint := Mountpoint + }}) -> + Topic = header(<<"destination">>, Headers), + Msg = emqx_message:make(ClientId, Topic, Body), + StompHeaders = lists:foldl( + fun(Key, Headers0) -> + proplists:delete(Key, Headers0) + end, Headers, + [<<"destination">>, + <<"content-length">>, + <<"content-type">>, + <<"transaction">>, + <<"receipt">> + ]), + %% Pass-through of custom headers on the sending side + NMsg = emqx_message:set_headers(#{proto_ver => ProtoVer, + protocol => Protocol, + username => Username, + peerhost => PeerHost, + stomp_headers => StompHeaders + }, Msg), + emqx_mountpoint:mount(Mountpoint, NMsg). + +receipt_id(Headers) -> + header(<<"receipt">>, Headers). + +%%-------------------------------------------------------------------- +%% Trans + +add_action(TxId, Action, ReceiptId, Channel = #channel{transaction = Trans}) -> + case maps:get(TxId, Trans, undefined) of + {_StartedAt, Actions} -> + NTrans = Trans#{TxId => {_StartedAt, [Action|Actions]}}, + {ok, Channel#channel{transaction = NTrans}}; + _ -> + ErrFrame = error_frame(ReceiptId, + ["Transaction ", TxId, " not found"]), + {ok, {outgoing, ErrFrame}, Channel} + end. + +%%-------------------------------------------------------------------- +%% Transaction Handle + +handle_recv_send_frame(Frame = ?PACKET(?CMD_SEND, Headers), Channel) -> + Msg = frame2message(Frame, Channel), + _ = emqx_broker:publish(Msg), + maybe_outgoing_receipt(receipt_id(Headers), Channel). + +handle_recv_ack_frame(?PACKET(?CMD_ACK, Headers), Channel) -> + maybe_outgoing_receipt(receipt_id(Headers), Channel). + +handle_recv_nack_frame(?PACKET(?CMD_NACK, Headers), Channel) -> + maybe_outgoing_receipt(receipt_id(Headers), Channel). + +maybe_outgoing_receipt(undefined, Channel) -> + {ok, [], Channel}; +maybe_outgoing_receipt(ReceiptId, Channel) -> + {ok, [{outgoing, receipt_frame(ReceiptId)}], Channel}. + +maybe_outgoing_receipt(undefined, Outgoings, Channel) -> + {ok, Outgoings, Channel}; +maybe_outgoing_receipt(ReceiptId, Outgoings, Channel) -> + {ok, lists:reverse([receipt_frame(ReceiptId)|Outgoings]), Channel}. + +ensure_clean_trans_timer(Channel = #channel{transaction = Trans}) -> + case maps:size(Trans) of + 0 -> Channel; + _ -> ensure_timer(clean_trans_timer, Channel) + end. + +%%-------------------------------------------------------------------- +%% Heartbeat + +reverse_heartbeats({Cx, Cy}) -> + iolist_to_binary(io_lib:format("~w,~w", [Cy, Cx])). + +ensure_heartbeart_timer(Channel = #channel{clientinfo = ClientInfo}) -> + Heartbeat = maps:get(heartbeat, ClientInfo), + ensure_timer( + [incoming_timer, outgoing_timer], + Channel#channel{heartbeat = emqx_stomp_heartbeat:init(Heartbeat)}). + +%%-------------------------------------------------------------------- +%% Timer + +ensure_timer([Name], Channel) -> + ensure_timer(Name, Channel); +ensure_timer([Name | Rest], Channel) -> + ensure_timer(Rest, ensure_timer(Name, Channel)); + +ensure_timer(Name, Channel = #channel{timers = Timers}) -> + TRef = maps:get(Name, Timers, undefined), + Time = interval(Name, Channel), + case TRef == undefined andalso is_integer(Time) andalso Time > 0 of + true -> ensure_timer(Name, Time, Channel); + false -> Channel %% Timer disabled or exists + end. + +ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> + Msg = maps:get(Name, ?TIMER_TABLE), + TRef = emqx_misc:start_timer(Time, Msg), + Channel#channel{timers = Timers#{Name => TRef}}. + +reset_timer(Name, Channel) -> + ensure_timer(Name, clean_timer(Name, Channel)). + +clean_timer(Name, Channel = #channel{timers = Timers}) -> + Channel#channel{timers = maps:remove(Name, Timers)}. + +interval(incoming_timer, #channel{heartbeat = HrtBt}) -> + emqx_stomp_heartbeat:interval(incoming, HrtBt); +interval(outgoing_timer, #channel{heartbeat = HrtBt}) -> + emqx_stomp_heartbeat:interval(outgoing, HrtBt); +interval(clean_trans_timer, _) -> + ?TRANS_TIMEOUT. + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- + +run_hooks(Ctx, Name, Args) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name), + emqx_hooks:run(Name, Args). + +run_hooks(Ctx, Name, Args, Acc) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name), + emqx_hooks:run_fold(Name, Args, Acc). + +run_hooks_without_metrics(_Ctx, Name, Args, Acc) -> + emqx_hooks:run_fold(Name, Args, Acc). + +metrics_inc(Name, #channel{ctx = Ctx}) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name). diff --git a/apps/emqx_stomp/src/emqx_stomp_frame.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl similarity index 64% rename from apps/emqx_stomp/src/emqx_stomp_frame.erl rename to apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl index fa9cb63a8..2b511b57a 100644 --- a/apps/emqx_stomp/src/emqx_stomp_frame.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl @@ -68,18 +68,26 @@ -module(emqx_stomp_frame). --include("emqx_stomp.hrl"). +-behavior(emqx_gateway_frame). --export([ init_parer_state/1 +-include("src/stomp/include/emqx_stomp.hrl"). + +-export([ initial_parse_state/1 , parse/2 - , serialize/1 + , serialize_opts/0 + , serialize_pkt/2 ]). --export([ make/2 +-export([ make/1 + , make/2 , make/3 , format/1 ]). +-export([ type/1 + , is_message/1 + ]). + -define(NULL, 0). -define(CR, $\r). -define(LF, $\n). @@ -96,28 +104,33 @@ -record(frame_limit, {max_header_num, max_header_length, max_body_length}). --type(result() :: {ok, stomp_frame(), binary()} - | {more, parser()} - | {error, any()}). +-type(parse_result() :: {ok, stomp_frame(), binary(), parse_state()} + | {more, parse_state()}). --type(parser() :: #{phase := none | command | headers | hdname | hdvalue | body, - pre => binary(), - state := #parser_state{}}). +-type(parse_state() :: + #{phase := none | command | headers | hdname | hdvalue | body, + pre => binary(), + state := #parser_state{} + }). + +%-dialyzer({nowarn_function, [serialize_pkt/2,make/1]}). %% @doc Initialize a parser --spec init_parer_state([proplists:property()]) -> parser(). -init_parer_state(Opts) -> +-spec initial_parse_state(map()) -> parse_state(). +initial_parse_state(Opts) -> #{phase => none, state => #parser_state{limit = limit(Opts)}}. limit(Opts) -> - #frame_limit{max_header_num = g(max_header_num, Opts, ?MAX_HEADER_NUM), - max_header_length = g(max_header_length, Opts, ?MAX_BODY_LENGTH), - max_body_length = g(max_body_length, Opts, ?MAX_BODY_LENGTH)}. + #frame_limit{ + max_header_num = g(max_header_num, Opts, ?MAX_HEADER_NUM), + max_header_length = g(max_header_length, Opts, ?MAX_BODY_LENGTH), + max_body_length = g(max_body_length, Opts, ?MAX_BODY_LENGTH) + }. g(Key, Opts, Val) -> - proplists:get_value(Key, Opts, Val). + maps:get(Key, Opts, Val). --spec parse(binary(), parser()) -> result(). +-spec parse(binary(), parse_state()) -> parse_result(). parse(<<>>, Parser) -> {more, Parser}; @@ -131,11 +144,14 @@ parse(<>, #{phase := Phase, state := State}) -> parse(<>, Parser) -> {more, Parser#{pre => <>}}; parse(<>, _Parser) -> - {error, linefeed_expected}; + error(linefeed_expected); -parse(<>, Parser = #{phase := Phase}) when Phase =:= hdname; Phase =:= hdvalue -> +parse(<>, Parser = #{phase := Phase}) when Phase =:= hdname; + Phase =:= hdvalue -> {more, Parser#{pre => <>}}; -parse(<>, #{phase := Phase, state := State}) when Phase =:= hdname; Phase =:= hdvalue -> +parse(<>, + #{phase := Phase, state := State}) when Phase =:= hdname; + Phase =:= hdvalue -> parse(Phase, Rest, acc(unescape(Ch), State)); parse(Bytes, #{phase := none, state := State}) -> @@ -153,14 +169,19 @@ parse(headers, Bin, State) -> parse(hdname, Bin, State); parse(hdname, <>, _State) -> - {error, unexpected_linefeed}; + error(unexpected_linefeed); parse(hdname, <>, State = #parser_state{acc = Acc}) -> parse(hdvalue, Rest, State#parser_state{hdname = Acc, acc = <<>>}); parse(hdname, <>, State) -> parse(hdname, Rest, acc(Ch, State)); -parse(hdvalue, <>, State = #parser_state{headers = Headers, hdname = Name, acc = Acc}) -> - parse(headers, Rest, State#parser_state{headers = add_header(Name, Acc, Headers), hdname = undefined, acc = <<>>}); +parse(hdvalue, <>, + State = #parser_state{headers = Headers, hdname = Name, acc = Acc}) -> + NState = State#parser_state{headers = add_header(Name, Acc, Headers), + hdname = undefined, + acc = <<>> + }, + parse(headers, Rest, NState); parse(hdvalue, <>, State) -> parse(hdvalue, Rest, acc(Ch, State)). @@ -170,15 +191,19 @@ parse(body, <<>>, State, Length) -> parse(body, Bin, State, none) -> case binary:split(Bin, <>) of [Chunk, Rest] -> - {ok, new_frame(acc(Chunk, State)), Rest}; + {ok, new_frame(acc(Chunk, State)), Rest, new_state(State)}; [Chunk] -> - {more, #{phase => body, length => none, state => acc(Chunk, State)}} + {more, #{phase => body, + length => none, + state => acc(Chunk, State)}} end; parse(body, Bin, State, Len) when byte_size(Bin) >= (Len+1) -> <> = Bin, - {ok, new_frame(acc(Chunk, State)), Rest}; + {ok, new_frame(acc(Chunk, State)), Rest, new_state(State)}; parse(body, Bin, State, Len) -> - {more, #{phase => body, length => Len - byte_size(Bin), state => acc(Bin, State)}}. + {more, #{phase => body, + length => Len - byte_size(Bin), + state => acc(Bin, State)}}. add_header(Name, Value, Headers) -> case lists:keyfind(Name, 1, Headers) of @@ -208,20 +233,33 @@ unescape($r) -> ?CR; unescape($n) -> ?LF; unescape($c) -> ?COLON; unescape($\\) -> ?BSL; -unescape(_Ch) -> {error, cannnot_unescape}. +unescape(_Ch) -> error(cannnot_unescape). -serialize(#stomp_frame{command = Cmd, headers = Headers, body = Body}) -> +%%-------------------------------------------------------------------- +%% Serialize funcs +%%-------------------------------------------------------------------- + +serialize_opts() -> + #{}. + +serialize_pkt(#stomp_frame{command = ?CMD_HEARTBEAT}, _SerializeOpts) -> + <<$\n>>; + +serialize_pkt(#stomp_frame{command = Cmd, headers = Headers, body = Body}, + _SerializeOpts) -> Headers1 = lists:keydelete(<<"content-length">>, 1, Headers), Headers2 = case iolist_size(Body) of 0 -> Headers1; Len -> Headers1 ++ [{<<"content-length">>, Len}] end, - [Cmd, ?LF, [serialize(header, Header) || Header <- Headers2], ?LF, Body, 0]. + [Cmd, + ?LF, [serialize_pkt(header, Header) || Header <- Headers2], + ?LF, Body, 0]; -serialize(header, {Name, Val}) when is_integer(Val) -> +serialize_pkt(header, {Name, Val}) when is_integer(Val) -> [escape(Name), ?COLON, integer_to_list(Val), ?LF]; -serialize(header, {Name, Val}) -> +serialize_pkt(header, {Name, Val}) -> [escape(Name), ?COLON, escape(Val), ?LF]. escape(Bin) when is_binary(Bin) -> @@ -232,8 +270,18 @@ escape(?BSL) -> <>; escape(?COLON) -> <>; escape(Ch) -> <>. +new_state(#parser_state{limit = Limit}) -> + #{phase => none, state => #parser_state{limit = Limit}}. + +%%-------------------------------------------------------------------- +%% ??? +%%-------------------------------------------------------------------- %% @doc Make a frame + +make(?CMD_HEARTBEAT) -> + #stomp_frame{command = ?CMD_HEARTBEAT}. + make(<<"CONNECTED">>, Headers) -> #stomp_frame{command = <<"CONNECTED">>, headers = [{<<"server">>, ?STOMP_SERVER} | Headers]}; @@ -245,5 +293,29 @@ make(Command, Headers, Body) -> #stomp_frame{command = Command, headers = Headers, body = Body}. %% @doc Format a frame -format(Frame) -> serialize(Frame). +format(Frame) -> serialize_pkt(Frame, #{}). +is_message(#stomp_frame{command = CMD}) + when CMD == ?CMD_SEND; + CMD == ?CMD_MESSAGE -> + true; +is_message(_) -> false. + +type(#stomp_frame{command = CMD}) -> + type(CMD); +type(?CMD_STOMP) -> connect; +type(?CMD_CONNECT) -> connect; +type(?CMD_SEND) -> send; +type(?CMD_SUBSCRIBE) -> subscribe; +type(?CMD_UNSUBSCRIBE) -> unsubscribe; +type(?CMD_BEGIN) -> 'begin'; +type(?CMD_COMMIT) -> commit; +type(?CMD_ABORT) -> abort; +type(?CMD_ACK) -> ack; +type(?CMD_NACK) -> nack; +type(?CMD_DISCONNECT) -> disconnect; +type(?CMD_CONNECTED) -> connected; +type(?CMD_MESSAGE) -> message; +type(?CMD_RECEIPT) -> receipt; +type(?CMD_ERROR) -> error; +type(?CMD_HEARTBEAT) -> heartbeat. diff --git a/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl similarity index 89% rename from apps/emqx_stomp/src/emqx_stomp_heartbeat.erl rename to apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl index 145359e53..99a1508e1 100644 --- a/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl @@ -17,10 +17,11 @@ %% @doc Stomp heartbeat. -module(emqx_stomp_heartbeat). --include("emqx_stomp.hrl"). +-include("src/stomp/include/emqx_stomp.hrl"). -export([ init/1 , check/3 + , reset/3 , info/1 , interval/2 ]). @@ -33,7 +34,6 @@ outgoing => #heartbeater{} }. - %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- @@ -77,6 +77,15 @@ check(NewVal, HrtBter = #heartbeater{statval = OldVal, true -> {error, timeout} end. +-spec reset(name(), pos_integer(), heartbeat()) + -> heartbeat(). +reset(Name, NewVal, HrtBt) -> + HrtBter = maps:get(Name, HrtBt), + HrtBt#{Name => reset(NewVal, HrtBter)}. + +reset(NewVal, HrtBter) -> + HrtBter#heartbeater{statval = NewVal, repeat = 1}. + -spec info(heartbeat()) -> map(). info(HrtBt) -> maps:map(fun(_, #heartbeater{interval = Intv, diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl new file mode 100644 index 000000000..64d0b32e5 --- /dev/null +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl @@ -0,0 +1,149 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_stomp_impl). + +-include_lib("emqx_gateway/include/emqx_gateway.hrl"). + +-behavior(emqx_gateway_impl). + +%% APIs +-export([ load/0 + , unload/0 + ]). + +-export([ init/1 + , on_insta_create/3 + , on_insta_update/4 + , on_insta_destroy/3 + ]). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +-spec load() -> ok | {error, any()}. +load() -> + RegistryOptions = [ {cbkmod, ?MODULE} + ], + emqx_gateway_registry:load(stomp, RegistryOptions, []). + +-spec unload() -> ok | {error, any()}. +unload() -> + emqx_gateway_registry:unload(stomp). + +init(_) -> + GwState = #{}, + {ok, GwState}. + +%%-------------------------------------------------------------------- +%% emqx_gateway_registry callbacks +%%-------------------------------------------------------------------- + +on_insta_create(_Insta = #{ id := InstaId, + rawconf := RawConf + }, Ctx, _GwState) -> + %% Step1. Fold the rawconfs to listeners + Listeners = emqx_gateway_utils:normalize_rawconf(RawConf), + %% Step2. Start listeners or escokd:specs + ListenerPids = lists:map(fun(Lis) -> + start_listener(InstaId, Ctx, Lis) + end, Listeners), + %% FIXME: How to throw an exception to interrupt the restart logic ? + %% FIXME: Assign ctx to InstaState + {ok, ListenerPids, _InstaState = #{ctx => Ctx}}. + +on_insta_update(NewInsta, OldInsta, GwInstaState = #{ctx := Ctx}, GwState) -> + InstaId = maps:get(id, NewInsta), + try + %% XXX: 1. How hot-upgrade the changes ??? + %% XXX: 2. Check the New confs first before destroy old instance ??? + on_insta_destroy(OldInsta, GwInstaState, GwState), + on_insta_create(NewInsta, Ctx, GwState) + catch + Class : Reason : Stk -> + logger:error("Failed to update stomp instance ~s; " + "reason: {~0p, ~0p} stacktrace: ~0p", + [InstaId, Class, Reason, Stk]), + {error, {Class, Reason}} + end. + +on_insta_destroy(_Insta = #{ id := InstaId, + rawconf := RawConf + }, _GwInstaState, _GwState) -> + Listeners = emqx_gateway_utils:normalize_rawconf(RawConf), + lists:foreach(fun(Lis) -> + stop_listener(InstaId, Lis) + end, Listeners). + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +start_listener(InstaId, Ctx, {Type, ListenOn, SocketOpts, Cfg}) -> + ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), + case start_listener(InstaId, Ctx, Type, ListenOn, SocketOpts, Cfg) of + {ok, Pid} -> + io:format("Start stomp ~s:~s listener on ~s successfully.~n", + [InstaId, Type, ListenOnStr]), + Pid; + {error, Reason} -> + io:format(standard_error, + "Failed to start stomp ~s:~s listener on ~s: ~0p~n", + [InstaId, Type, ListenOnStr, Reason]), + throw({badconf, Reason}) + end. + +start_listener(InstaId, Ctx, Type, ListenOn, SocketOpts, Cfg) -> + Name = name(InstaId, Type), + NCfg = Cfg#{ + ctx => Ctx, + frame_mod => emqx_stomp_frame, + chann_mod => emqx_stomp_channel + }, + esockd:open(Name, ListenOn, merge_default(SocketOpts), + {emqx_gateway_conn, start_link, [NCfg]}). + +name(InstaId, Type) -> + list_to_atom(lists:concat([InstaId, ":", Type])). + +merge_default(Options) -> + Default = emqx_gateway_utils:default_tcp_options(), + case lists:keytake(tcp_options, 1, Options) of + {value, {tcp_options, TcpOpts}, Options1} -> + [{tcp_options, emqx_misc:merge_opts(Default, TcpOpts)} + | Options1]; + false -> + [{tcp_options, Default} | Options] + end. + +stop_listener(InstaId, {Type, ListenOn, SocketOpts, Cfg}) -> + StopRet = stop_listener(InstaId, Type, ListenOn, SocketOpts, Cfg), + ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), + case StopRet of + ok -> io:format("Stop stomp ~s:~s listener on ~s successfully.~n", + [InstaId, Type, ListenOnStr]); + {error, Reason} -> + io:format(standard_error, + "Failed to stop stomp ~s:~s listener on ~s: ~0p~n", + [InstaId, Type, ListenOnStr, Reason] + ) + end, + StopRet. + +stop_listener(InstaId, Type, ListenOn, _SocketOpts, _Cfg) -> + Name = name(InstaId, Type), + esockd:close(Name, ListenOn). diff --git a/apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl b/apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl new file mode 100644 index 000000000..280c3e347 --- /dev/null +++ b/apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl @@ -0,0 +1,94 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_STOMP_HRL). +-define(EMQX_STOMP_HRL, true). + +-define(STOMP_VER, <<"1.2">>). + +-define(STOMP_SERVER, <<"emqx-stomp/1.2">>). + +%%-------------------------------------------------------------------- +%% STOMP Frame +%%-------------------------------------------------------------------- + +%% client command +-define(CMD_STOMP, <<"STOMP">>). +-define(CMD_CONNECT, <<"CONNECT">>). +-define(CMD_SEND, <<"SEND">>). +-define(CMD_SUBSCRIBE, <<"SUBSCRIBE">>). +-define(CMD_UNSUBSCRIBE, <<"UNSUBSCRIBE">>). +-define(CMD_BEGIN, <<"BEGIN">>). +-define(CMD_COMMIT, <<"COMMIT">>). +-define(CMD_ABORT, <<"ABORT">>). +-define(CMD_ACK, <<"ACK">>). +-define(CMD_NACK, <<"NACK">>). +-define(CMD_DISCONNECT, <<"DISCONNECT">>). + +%% server command +-define(CMD_CONNECTED, <<"CONNECTED">>). +-define(CMD_MESSAGE, <<"MESSAGE">>). +-define(CMD_RECEIPT, <<"RECEIPT">>). +-define(CMD_ERROR, <<"ERROR">>). +-define(CMD_HEARTBEAT, <<"HEARTBEAT">>). + +%-type client_command() :: ?CMD_SEND | ?CMD_SUBSCRIBE | ?CMD_UNSUBSCRIBE +% | ?CMD_BEGIN | ?CMD_COMMIT | ?CMD_ABORT | ?CMD_ACK +% | ?CMD_NACK | ?CMD_DISCONNECT | ?CMD_CONNECT +% | ?CMD_STOMP. +% +-type client_command() :: binary(). + +%-type server_command() :: ?CMD_CONNECTED | ?CMD_MESSAGE | ?CMD_RECEIPT +% | ?CMD_ERROR. +-type server_command() :: binary(). + +-record(stomp_frame, { + command :: client_command() | server_command(), + headers = [], + body = <<>> :: iodata()} + ). + +-type stomp_frame() :: #stomp_frame{}. + +-define(PACKET(CMD), #stomp_frame{command = CMD}). + +-define(PACKET(CMD, Headers), #stomp_frame{command = CMD, headers = Headers}). + +-define(PACKET(CMD, Headers, Body), #stomp_frame{command = CMD, + headers = Headers, + body = Body + }). + +%%-------------------------------------------------------------------- +%% Frame Size Limits +%% +%% To prevent malicious clients from exploiting memory allocation in a server, +%% servers MAY place maximum limits on: +%% +%% the number of frame headers allowed in a single frame +%% the maximum length of header lines +%% the maximum size of a frame body +%% +%% If these limits are exceeded the server SHOULD send the client an ERROR frame +%% and then close the connection. +%%-------------------------------------------------------------------- + +-define(MAX_HEADER_NUM, 10). +-define(MAX_HEADER_LENGTH, 1024). +-define(MAX_BODY_LENGTH, 65536). + +-endif. diff --git a/apps/emqx_sn/test/broadcast_test.py b/apps/emqx_gateway/test/broadcast_test.py similarity index 100% rename from apps/emqx_sn/test/broadcast_test.py rename to apps/emqx_gateway/test/broadcast_test.py diff --git a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl b/apps/emqx_gateway/test/emqx_exproto_SUITE.erl similarity index 92% rename from apps/emqx_exproto/test/emqx_exproto_SUITE.erl rename to apps/emqx_gateway/test/emqx_exproto_SUITE.erl index db64c7438..2207ca9d6 100644 --- a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_exproto_SUITE.erl @@ -55,33 +55,31 @@ metrics() -> init_per_group(GrpName, Cfg) -> put(grpname, GrpName), Svrs = emqx_exproto_echo_svr:start(), - emqx_ct_helpers:start_apps([emqx_exproto], fun set_special_cfg/1), + emqx_ct_helpers:start_apps([emqx_gateway], fun set_special_cfg/1), emqx_logger:set_log_level(debug), [{servers, Svrs}, {listener_type, GrpName} | Cfg]. end_per_group(_, Cfg) -> - emqx_ct_helpers:stop_apps([emqx_exproto]), + emqx_ct_helpers:stop_apps([emqx_gateway]), emqx_exproto_echo_svr:stop(proplists:get_value(servers, Cfg)). -set_special_cfg(emqx_exproto) -> +set_special_cfg(emqx_gateway) -> LisType = get(grpname), - Listeners = application:get_env(emqx_exproto, listeners, []), - SockOpts = socketopts(LisType), - UpgradeOpts = fun(Opts) -> - Opts2 = lists:keydelete(tcp_options, 1, Opts), - Opts3 = lists:keydelete(ssl_options, 1, Opts2), - Opts4 = lists:keydelete(udp_options, 1, Opts3), - Opts5 = lists:keydelete(dtls_options, 1, Opts4), - SockOpts ++ Opts5 - end, - NListeners = [{Proto, LisType, LisOn, UpgradeOpts(Opts)} - || {Proto, _Type, LisOn, Opts} <- Listeners], - application:set_env(emqx_exproto, listeners, NListeners); -set_special_cfg(emqx) -> - application:set_env(emqx, allow_anonymous, true), - application:set_env(emqx, enable_acl_cache, false), + emqx_config:put( + [gateway, exproto], + #{'1' => + #{authenticator => allow_anonymous, + server => #{bind => 9100}, + handler => #{address => "http://127.0.0.1:9001"}, + listener => listener_confs(LisType) + }}); +set_special_cfg(_App) -> ok. +listener_confs(Type) -> + Default = #{bind => 7993, acceptors => 8}, + #{Type => #{'1' => maps:merge(Default, maps:from_list(socketopts(Type)))}}. + %%-------------------------------------------------------------------- %% Tests cases %%-------------------------------------------------------------------- @@ -167,7 +165,7 @@ t_acl_deny(Cfg) -> Password = <<"123456">>, ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), - ok = meck:expect(emqx_access_control, check_acl, fun(_, _, _) -> deny end), + ok = meck:expect(emqx_access_control, authorize, fun(_, _, _) -> deny end), ConnBin = frame_connect(Client, Password), ConnAckBin = frame_connack(0), diff --git a/apps/emqx_exproto/test/emqx_exproto_echo_svr.erl b/apps/emqx_gateway/test/emqx_exproto_echo_svr.erl similarity index 100% rename from apps/emqx_exproto/test/emqx_exproto_echo_svr.erl rename to apps/emqx_gateway/test/emqx_exproto_echo_svr.erl diff --git a/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl new file mode 100644 index 000000000..33d577a46 --- /dev/null +++ b/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl @@ -0,0 +1,69 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway_registry_SUITE). + +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). +-compile(nowarn_export_all). + +-define(CONF_DEFAULT, <<""" +gateway: { + stomp.1 {} +} +""">>). + +all() -> emqx_ct:all(?MODULE). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +init_per_suite(Cfg) -> + ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT), + emqx_ct_helpers:start_apps([emqx_gateway]), + Cfg. + +end_per_suite(_Cfg) -> + emqx_ct_helpers:stop_apps([emqx_gateway]), + ok. + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_load_unload(_) -> + OldCnt = length(emqx_gateway_registry:list()), + RgOpts = [{cbkmod, ?MODULE}], + GwOpts = [paramsin], + ok = emqx_gateway_registry:load(test, RgOpts, GwOpts), + ?assertEqual(OldCnt+1, length(emqx_gateway_registry:list())), + + #{cbkmod := ?MODULE, + rgopts := RgOpts, + gwopts := GwOpts, + state := #{gwstate := 1}} = emqx_gateway_registry:lookup(test), + + {error, already_existed} = emqx_gateway_registry:load(test, [{cbkmod, ?MODULE}], GwOpts), + + ok = emqx_gateway_registry:unload(test), + undefined = emqx_gateway_registry:lookup(test), + OldCnt = length(emqx_gateway_registry:list()), + ok. + +init([paramsin]) -> + {ok, _GwState = #{gwstate => 1}}. diff --git a/apps/emqx_sn/test/emqx_sn_frame_SUITE.erl b/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl similarity index 73% rename from apps/emqx_sn/test/emqx_sn_frame_SUITE.erl rename to apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl index 85042a4be..7437c10b8 100644 --- a/apps/emqx_sn/test/emqx_sn_frame_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl @@ -19,13 +19,9 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("emqx_sn/include/emqx_sn.hrl"). +-include_lib("emqx_gateway/src/mqttsn/include/emqx_sn.hrl"). -include_lib("eunit/include/eunit.hrl"). --import(emqx_sn_frame, [ parse/1 - , serialize/1 - ]). - %%-------------------------------------------------------------------- %% Setups %%-------------------------------------------------------------------- @@ -33,122 +29,129 @@ all() -> emqx_ct:all(?MODULE). +parse(D) -> + {ok, P, _Rest, _State} = emqx_sn_frame:parse(D, #{}), + P. + +serialize_pkt(P) -> + emqx_sn_frame:serialize_pkt(P, #{}). + %%-------------------------------------------------------------------- %% Test cases %%-------------------------------------------------------------------- t_advertise(_) -> Adv = ?SN_ADVERTISE_MSG(1, 100), - ?assertEqual({ok, Adv}, parse(serialize(Adv))). + ?assertEqual(Adv, parse(serialize_pkt(Adv))). t_searchgw(_) -> Sgw = #mqtt_sn_message{type = ?SN_SEARCHGW, variable = 1}, - ?assertEqual({ok, Sgw}, parse(serialize(Sgw))). + ?assertEqual(Sgw, parse(serialize_pkt(Sgw))). t_gwinfo(_) -> GwInfo = #mqtt_sn_message{type = ?SN_GWINFO, variable = {2, <<"EMQGW">>}}, - ?assertEqual({ok, GwInfo}, parse(serialize(GwInfo))). + ?assertEqual(GwInfo, parse(serialize_pkt(GwInfo))). t_connect(_) -> Flags = #mqtt_sn_flags{will = true, clean_start = true}, Conn = #mqtt_sn_message{type = ?SN_CONNECT, variable = {Flags, 4, 300, <<"ClientId">>}}, - ?assertEqual({ok, Conn}, parse(serialize(Conn))). + ?assertEqual(Conn, parse(serialize_pkt(Conn))). t_connack(_) -> ConnAck = #mqtt_sn_message{type = ?SN_CONNACK, variable = 2}, - ?assertEqual({ok, ConnAck}, parse(serialize(ConnAck))). + ?assertEqual(ConnAck, parse(serialize_pkt(ConnAck))). t_willtopicreq(_) -> WtReq = #mqtt_sn_message{type = ?SN_WILLTOPICREQ}, - ?assertEqual({ok, WtReq}, parse(serialize(WtReq))). + ?assertEqual(WtReq, parse(serialize_pkt(WtReq))). t_willtopic(_) -> Flags = #mqtt_sn_flags{qos = 1, retain = false}, Wt = #mqtt_sn_message{type = ?SN_WILLTOPIC, variable = {Flags, <<"WillTopic">>}}, - ?assertEqual({ok, Wt}, parse(serialize(Wt))). + ?assertEqual(Wt, parse(serialize_pkt(Wt))). t_willmsgreq(_) -> WmReq = #mqtt_sn_message{type = ?SN_WILLMSGREQ}, - ?assertEqual({ok, WmReq}, parse(serialize(WmReq))). + ?assertEqual(WmReq, parse(serialize_pkt(WmReq))). t_willmsg(_) -> WlMsg = #mqtt_sn_message{type = ?SN_WILLMSG, variable = <<"WillMsg">>}, - ?assertEqual({ok, WlMsg}, parse(serialize(WlMsg))). + ?assertEqual(WlMsg, parse(serialize_pkt(WlMsg))). t_register(_) -> RegMsg = ?SN_REGISTER_MSG(1, 2, <<"Topic">>), - ?assertEqual({ok, RegMsg}, parse(serialize(RegMsg))). + ?assertEqual(RegMsg, parse(serialize_pkt(RegMsg))). t_regack(_) -> RegAck = ?SN_REGACK_MSG(1, 2, 0), - ?assertEqual({ok, RegAck}, parse(serialize(RegAck))). + ?assertEqual(RegAck, parse(serialize_pkt(RegAck))). t_publish(_) -> Flags = #mqtt_sn_flags{dup = false, qos = 1, retain = false, topic_id_type = 2#01}, PubMsg = #mqtt_sn_message{type = ?SN_PUBLISH, variable = {Flags, 1, 2, <<"Payload">>}}, - ?assertEqual({ok, PubMsg}, parse(serialize(PubMsg))). + ?assertEqual(PubMsg, parse(serialize_pkt(PubMsg))). t_puback(_) -> PubAck = #mqtt_sn_message{type = ?SN_PUBACK, variable = {1, 2, 0}}, - ?assertEqual({ok, PubAck}, parse(serialize(PubAck))). + ?assertEqual(PubAck, parse(serialize_pkt(PubAck))). t_pubrec(_) -> PubRec = #mqtt_sn_message{type = ?SN_PUBREC, variable = 16#1234}, - ?assertEqual({ok, PubRec}, parse(serialize(PubRec))). + ?assertEqual(PubRec, parse(serialize_pkt(PubRec))). t_pubrel(_) -> PubRel = #mqtt_sn_message{type = ?SN_PUBREL, variable = 16#1234}, - ?assertEqual({ok, PubRel}, parse(serialize(PubRel))). + ?assertEqual(PubRel, parse(serialize_pkt(PubRel))). t_pubcomp(_) -> PubComp = #mqtt_sn_message{type = ?SN_PUBCOMP, variable = 16#1234}, - ?assertEqual({ok, PubComp}, parse(serialize(PubComp))). + ?assertEqual(PubComp, parse(serialize_pkt(PubComp))). t_subscribe(_) -> Flags = #mqtt_sn_flags{dup = false, qos = 1, topic_id_type = 16#01}, SubMsg = #mqtt_sn_message{type = ?SN_SUBSCRIBE, variable = {Flags, 16#4321, 16}}, - ?assertEqual({ok, SubMsg}, parse(serialize(SubMsg))). + ?assertEqual(SubMsg, parse(serialize_pkt(SubMsg))). t_suback(_) -> Flags = #mqtt_sn_flags{qos = 1}, SubAck = #mqtt_sn_message{type = ?SN_SUBACK, variable = {Flags, 98, 89, 0}}, - ?assertEqual({ok, SubAck}, parse(serialize(SubAck))). + ?assertEqual(SubAck, parse(serialize_pkt(SubAck))). t_unsubscribe(_) -> Flags = #mqtt_sn_flags{dup = false, qos = 1, topic_id_type = 16#01}, UnSub = #mqtt_sn_message{type = ?SN_UNSUBSCRIBE, variable = {Flags, 16#4321, 16}}, - ?assertEqual({ok, UnSub}, parse(serialize(UnSub))). + ?assertEqual(UnSub, parse(serialize_pkt(UnSub))). t_unsuback(_) -> UnsubAck = #mqtt_sn_message{type = ?SN_UNSUBACK, variable = 72}, - ?assertEqual({ok, UnsubAck}, parse(serialize(UnsubAck))). + ?assertEqual(UnsubAck, parse(serialize_pkt(UnsubAck))). t_pingreq(_) -> Ping = #mqtt_sn_message{type = ?SN_PINGREQ, variable = <<>>}, - ?assertEqual({ok, Ping}, parse(serialize(Ping))), + ?assertEqual(Ping, parse(serialize_pkt(Ping))), Ping1 = #mqtt_sn_message{type = ?SN_PINGREQ, variable = <<"ClientId">>}, - ?assertEqual({ok, Ping1}, parse(serialize(Ping1))). + ?assertEqual(Ping1, parse(serialize_pkt(Ping1))). t_pingresp(_) -> PingResp = #mqtt_sn_message{type = ?SN_PINGRESP}, - ?assertEqual({ok, PingResp}, parse(serialize(PingResp))). + ?assertEqual(PingResp, parse(serialize_pkt(PingResp))). t_disconnect(_) -> Disconn = #mqtt_sn_message{type = ?SN_DISCONNECT}, - ?assertEqual({ok, Disconn}, parse(serialize(Disconn))). + ?assertEqual(Disconn, parse(serialize_pkt(Disconn))). t_willtopicupd(_) -> Flags = #mqtt_sn_flags{qos = 1, retain = true}, WtUpd = #mqtt_sn_message{type = ?SN_WILLTOPICUPD, variable = {Flags, <<"Topic">>}}, - ?assertEqual({ok, WtUpd}, parse(serialize(WtUpd))). + ?assertEqual(WtUpd, parse(serialize_pkt(WtUpd))). t_willmsgupd(_) -> WlMsgUpd = #mqtt_sn_message{type = ?SN_WILLMSGUPD, variable = <<"WillMsg">>}, - ?assertEqual({ok, WlMsgUpd}, parse(serialize(WlMsgUpd))). + ?assertEqual(WlMsgUpd, parse(serialize_pkt(WlMsgUpd))). t_willmsgresp(_) -> UpdResp = #mqtt_sn_message{type = ?SN_WILLMSGRESP, variable = 0}, - ?assertEqual({ok, UpdResp}, parse(serialize(UpdResp))). + ?assertEqual(UpdResp, parse(serialize_pkt(UpdResp))). t_random_test(_) -> random_test_body(), diff --git a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl similarity index 67% rename from apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl rename to apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl index 0947bdaca..0c60d964f 100644 --- a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl @@ -19,7 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("emqx_sn/include/emqx_sn.hrl"). +-include_lib("emqx_gateway/src/mqttsn/include/emqx_sn.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -50,6 +50,31 @@ %% erlang:system_time should be unique and random enough -define(CLIENTID, iolist_to_binary([atom_to_list(?FUNCTION_NAME), "-", integer_to_list(erlang:system_time())])). + +-define(CONF_DEFAULT, <<""" +gateway: { + mqttsn.1: { + gateway_id: 1 + broadcast: true + enable_stats: true + enable_qos3: true + predefined: [ + {id: 1, topic: \"/predefined/topic/name/hello\"}, + {id: 2, topic: \"/predefined/topic/name/nice\"} + ] + clientinfo_override: { + username: \"user1\" + password: \"pw123\" + } + listener.udp.1: { + bind: 1884 + max_connections: 10240000 + max_conn_rate: 1000 + } + } +} +""">>). + %%-------------------------------------------------------------------- %% Setups %%-------------------------------------------------------------------- @@ -58,24 +83,39 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - logger:set_module_level(emqx_sn_gateway, debug), - emqx_ct_helpers:start_apps([emqx_sn], fun set_special_confs/1), + ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT), + emqx_ct_helpers:start_apps([emqx_gateway]), Config. end_per_suite(_) -> - emqx_ct_helpers:stop_apps([emqx_sn]). + emqx_ct_helpers:stop_apps([emqx_gateway]). + +set_special_confs(emqx_gateway) -> + emqx_config:put( + [gateway], + #{ mqttsn => + #{'1' => + #{broadcast => true, + clientinfo_override => + #{password => "pw123", + username => "user1" + }, + enable_qos3 => true, + enable_stats => true, + gateway_id => 1, + idle_timeout => 30000, + listener => + #{udp => + #{'1' => + #{acceptors => 8,active_n => 100,backlog => 1024,bind => 1884, + high_watermark => 1048576,max_conn_rate => 1000, + max_connections => 10240000,send_timeout => 15000, + send_timeout_close => true}}}, + predefined => + [#{id => ?PREDEF_TOPIC_ID1, topic => ?PREDEF_TOPIC_NAME1}, + #{id => ?PREDEF_TOPIC_ID2, topic => ?PREDEF_TOPIC_NAME2}]}} + }); -set_special_confs(emqx) -> - application:set_env(emqx, plugins_loaded_file, - emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins")); -set_special_confs(emqx_sn) -> - application:set_env(emqx_sn, enable_qos3, ?ENABLE_QOS3), - application:set_env(emqx_sn, enable_stats, true), - application:set_env(emqx_sn, username, <<"user1">>), - application:set_env(emqx_sn, password, <<"pw123">>), - application:set_env(emqx_sn, predefined, - [{?PREDEF_TOPIC_ID1, ?PREDEF_TOPIC_NAME1}, - {?PREDEF_TOPIC_ID2, ?PREDEF_TOPIC_NAME2}]); set_special_confs(_App) -> ok. @@ -87,7 +127,7 @@ set_special_confs(_App) -> %% Connect t_connect(_) -> - SockName = {'mqttsn:udp', {{0,0,0,0}, 1884}}, + SockName = {'mqttsn#1:udp', 1884}, ?assertEqual(true, lists:keymember(SockName, 1, esockd:listeners())), {ok, Socket} = gen_udp:open(0, [binary]), @@ -170,7 +210,6 @@ t_subscribe_case02(_) -> ReturnCode = 0, {ok, Socket} = gen_udp:open(0, [binary]), - ClientId = ?CLIENTID, send_connect_msg(Socket, ?CLIENTID), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), @@ -342,7 +381,7 @@ t_subscribe_case07(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, TopicId1, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, ?SN_INVALID_TOPIC_ID:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, receive_response(Socket)), send_unsubscribe_msg_predefined_topic(Socket, TopicId2, MsgId), @@ -365,7 +404,7 @@ t_subscribe_case08(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_reserved_topic(Socket, QoS, TopicId2, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, ?SN_INVALID_TOPIC_ID:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, ?SN_INVALID_TOPIC_ID:16, MsgId:16, ?SN_RC_NOT_SUPPORTED>>, receive_response(Socket)), send_disconnect_msg(Socket, undefined), @@ -843,14 +882,16 @@ t_will_case01(_) -> ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), % wait udp client keepalive timeout - timer:sleep(2000), + timer:sleep(3000), receive - {deliver, WillTopic, #message{payload = WillMsg}} -> ok; - Msg -> ct:print("recevived --- unex: ~p", [Msg]) + {deliver, WillTopic, #message{payload = WillMsg}} -> + ok after 1000 -> ct:fail(wait_willmsg_timeout) end, + ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), + send_disconnect_msg(Socket, undefined), ?assertEqual(udp_receive_timeout, receive_response(Socket)), @@ -895,7 +936,7 @@ t_will_test3(_) -> timer:sleep(4000), - ?assertEqual(udp_receive_timeout, receive_response(Socket)), + ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), send_disconnect_msg(Socket, undefined), ?assertEqual(udp_receive_timeout, receive_response(Socket)), @@ -979,7 +1020,7 @@ t_will_case06(_) -> ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), % wait udp client keepalive timeout - timer:sleep(2000), + timer:sleep(3000), receive {deliver, WillTopic, #message{payload = WillMsg}} -> ok; @@ -991,588 +1032,588 @@ t_will_case06(_) -> gen_udp:close(Socket). -t_asleep_test01_timeout(_) -> - QoS = 1, - Duration = 1, - WillTopic = <<"dead">>, - WillPayload = <<10, 11, 12, 13, 14>>, - {ok, Socket} = gen_udp:open(0, [binary]), - - ClientId = ?CLIENTID, - send_connect_msg_with_will(Socket, Duration, ClientId), - ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), - send_willtopic_msg(Socket, WillTopic, QoS), - ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), - send_willmsg_msg(Socket, WillPayload), - ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), - - send_disconnect_msg(Socket, 1), - ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), - - %% asleep timer should get timeout, and device is lost - timer:sleep(3000), - - gen_udp:close(Socket). - -t_asleep_test02_to_awake_and_back(_) -> - QoS = 1, - Keepalive_Duration = 1, - SleepDuration = 5, - WillTopic = <<"dead">>, - WillPayload = <<10, 11, 12, 13, 14>>, - {ok, Socket} = gen_udp:open(0, [binary]), - - ClientId = ?CLIENTID, - send_connect_msg_with_will(Socket, Keepalive_Duration, ClientId), - ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), - send_willtopic_msg(Socket, WillTopic, QoS), - ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), - send_willmsg_msg(Socket, WillPayload), - ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), - - % goto asleep state - send_disconnect_msg(Socket, SleepDuration), - ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), - - timer:sleep(4500), - - % goto awake state and back - send_pingreq_msg(Socket, ClientId), - ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), - - timer:sleep(4500), - - % goto awake state and back - send_pingreq_msg(Socket, ClientId), - ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), - - %% during above procedure, mqtt keepalive timer should not terminate mqtt-sn process - - %% asleep timer should get timeout, and device should get lost - timer:sleep(8000), - - gen_udp:close(Socket). - -t_asleep_test03_to_awake_qos1_dl_msg(_) -> - QoS = 1, - Duration = 5, - WillTopic = <<"dead">>, - WillPayload = <<10, 11, 12, 13, 14>>, - MsgId = 1000, - {ok, Socket} = gen_udp:open(0, [binary]), - ClientId = ?CLIENTID, - send_connect_msg_with_will(Socket, Duration, ClientId), - ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), - send_willtopic_msg(Socket, WillTopic, QoS), - ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), - send_willmsg_msg(Socket, WillPayload), - ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), - - % subscribe - TopicName1 = <<"abc">>, - MsgId1 = 25, - TopicId1 = ?MAX_PRED_TOPIC_ID + 1, - WillBit = 0, - Dup = 0, - Retain = 0, - CleanSession = 0, - ReturnCode = 0, - Payload1 = <<55, 66, 77, 88, 99>>, - MsgId2 = 87, - - send_register_msg(Socket, TopicName1, MsgId1), - ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId1:16, 0:8>>, receive_response(Socket)), - send_subscribe_msg_predefined_topic(Socket, QoS, TopicId1, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ReturnCode>>, receive_response(Socket)), - - % goto asleep state - send_disconnect_msg(Socket, 1), - ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), - - timer:sleep(300), - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% send downlink data in asleep state. This message should be send to device once it wake up - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - send_publish_msg_predefined_topic(Socket, QoS, MsgId2, TopicId1, Payload1), - - {ok, C} = emqtt:start_link(), - {ok, _} = emqtt:connect(C), - {ok, _} = emqtt:publish(C, TopicName1, Payload1, QoS), - timer:sleep(100), - ok = emqtt:disconnect(C), - - timer:sleep(50), - - % goto awake state, receive downlink messages, and go back to asleep - send_pingreq_msg(Socket, ClientId), - - %% the broker should sent dl msgs to the awake client before sending the pingresp - UdpData = receive_response(Socket), - MsgId_udp = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicId1, Payload1}, UdpData), - send_puback_msg(Socket, TopicId1, MsgId_udp), - - %% check the pingresp is received at last - ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), - - gen_udp:close(Socket). - -t_asleep_test04_to_awake_qos1_dl_msg(_) -> - QoS = 1, - Duration = 5, - WillTopic = <<"dead">>, - WillPayload = <<10, 11, 12, 13, 14>>, - {ok, Socket} = gen_udp:open(0, [binary]), - ClientId = ?CLIENTID, - send_connect_msg_with_will(Socket, Duration, ClientId), - ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), - send_willtopic_msg(Socket, WillTopic, QoS), - ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), - send_willmsg_msg(Socket, WillPayload), - ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), - - % subscribe - TopicName1 = <<"a/+/c">>, - MsgId1 = 25, - TopicId0 = 0, - WillBit = 0, - Dup = 0, - Retain = 0, - CleanSession = 0, - ReturnCode = 0, - send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, - receive_response(Socket)), - - % goto asleep state - send_disconnect_msg(Socket, 1), - ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), - - timer:sleep(300), - - %% send downlink data in asleep state. This message should be send to device once it wake up - Payload1 = <<55, 66, 77, 88, 99>>, - Payload2 = <<55, 66, 77, 88, 100>>, - - {ok, C} = emqtt:start_link(), - {ok, _} = emqtt:connect(C), - {ok, _} = emqtt:publish(C, <<"a/b/c">>, Payload1, QoS), - {ok, _} = emqtt:publish(C, <<"a/b/c">>, Payload2, QoS), - timer:sleep(100), - ok = emqtt:disconnect(C), - - timer:sleep(300), - - % goto awake state, receive downlink messages, and go back to asleep - send_pingreq_msg(Socket, ClientId), - - %% 1. get REGISTER first, since this topic has never been registered - UdpData1 = receive_response(Socket), - {TopicIdNew, MsgId3} = check_register_msg_on_udp(<<"a/b/c">>, UdpData1), - - %% 2. but before we reply the REGACK, the sn-gateway should not send any PUBLISH - ?assertError(_, receive_publish(Socket)), - - send_regack_msg(Socket, TopicIdNew, MsgId3), - - UdpData2 = receive_response(Socket), - MsgId_udp2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload1}, UdpData2), - send_puback_msg(Socket, TopicIdNew, MsgId_udp2), - - UdpData3 = receive_response(Socket), - MsgId_udp3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData3), - send_puback_msg(Socket, TopicIdNew, MsgId_udp3), - - ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), - - gen_udp:close(Socket). +%t_asleep_test01_timeout(_) -> +% QoS = 1, +% Duration = 1, +% WillTopic = <<"dead">>, +% WillPayload = <<10, 11, 12, 13, 14>>, +% {ok, Socket} = gen_udp:open(0, [binary]), +% +% ClientId = ?CLIENTID, +% send_connect_msg_with_will(Socket, Duration, ClientId), +% ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), +% send_willtopic_msg(Socket, WillTopic, QoS), +% ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), +% send_willmsg_msg(Socket, WillPayload), +% ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), +% +% send_disconnect_msg(Socket, 1), +% ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), +% +% %% asleep timer should get timeout, and device is lost +% timer:sleep(3000), +% +% gen_udp:close(Socket). +% +%t_asleep_test02_to_awake_and_back(_) -> +% QoS = 1, +% Keepalive_Duration = 1, +% SleepDuration = 5, +% WillTopic = <<"dead">>, +% WillPayload = <<10, 11, 12, 13, 14>>, +% {ok, Socket} = gen_udp:open(0, [binary]), +% +% ClientId = ?CLIENTID, +% send_connect_msg_with_will(Socket, Keepalive_Duration, ClientId), +% ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), +% send_willtopic_msg(Socket, WillTopic, QoS), +% ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), +% send_willmsg_msg(Socket, WillPayload), +% ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), +% +% % goto asleep state +% send_disconnect_msg(Socket, SleepDuration), +% ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), +% +% timer:sleep(4500), +% +% % goto awake state and back +% send_pingreq_msg(Socket, ClientId), +% ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), +% +% timer:sleep(4500), +% +% % goto awake state and back +% send_pingreq_msg(Socket, ClientId), +% ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), +% +% %% during above procedure, mqtt keepalive timer should not terminate mqtt-sn process +% +% %% asleep timer should get timeout, and device should get lost +% timer:sleep(8000), +% +% gen_udp:close(Socket). +% +%t_asleep_test03_to_awake_qos1_dl_msg(_) -> +% QoS = 1, +% Duration = 5, +% WillTopic = <<"dead">>, +% WillPayload = <<10, 11, 12, 13, 14>>, +% MsgId = 1000, +% {ok, Socket} = gen_udp:open(0, [binary]), +% ClientId = ?CLIENTID, +% send_connect_msg_with_will(Socket, Duration, ClientId), +% ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), +% send_willtopic_msg(Socket, WillTopic, QoS), +% ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), +% send_willmsg_msg(Socket, WillPayload), +% ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), +% +% % subscribe +% TopicName1 = <<"abc">>, +% MsgId1 = 25, +% TopicId1 = ?MAX_PRED_TOPIC_ID + 1, +% WillBit = 0, +% Dup = 0, +% Retain = 0, +% CleanSession = 0, +% ReturnCode = 0, +% Payload1 = <<55, 66, 77, 88, 99>>, +% MsgId2 = 87, +% +% send_register_msg(Socket, TopicName1, MsgId1), +% ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId1:16, 0:8>>, receive_response(Socket)), +% send_subscribe_msg_predefined_topic(Socket, QoS, TopicId1, MsgId), +% ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ReturnCode>>, receive_response(Socket)), +% +% % goto asleep state +% send_disconnect_msg(Socket, 1), +% ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), +% +% timer:sleep(300), +% +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% %% send downlink data in asleep state. This message should be send to device once it wake up +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% send_publish_msg_predefined_topic(Socket, QoS, MsgId2, TopicId1, Payload1), +% +% {ok, C} = emqtt:start_link(), +% {ok, _} = emqtt:connect(C), +% {ok, _} = emqtt:publish(C, TopicName1, Payload1, QoS), +% timer:sleep(100), +% ok = emqtt:disconnect(C), +% +% timer:sleep(50), +% +% % goto awake state, receive downlink messages, and go back to asleep +% send_pingreq_msg(Socket, ClientId), +% +% %% the broker should sent dl msgs to the awake client before sending the pingresp +% UdpData = receive_response(Socket), +% MsgId_udp = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicId1, Payload1}, UdpData), +% send_puback_msg(Socket, TopicId1, MsgId_udp), +% +% %% check the pingresp is received at last +% ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), +% +% gen_udp:close(Socket). +% +%t_asleep_test04_to_awake_qos1_dl_msg(_) -> +% QoS = 1, +% Duration = 5, +% WillTopic = <<"dead">>, +% WillPayload = <<10, 11, 12, 13, 14>>, +% {ok, Socket} = gen_udp:open(0, [binary]), +% ClientId = ?CLIENTID, +% send_connect_msg_with_will(Socket, Duration, ClientId), +% ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), +% send_willtopic_msg(Socket, WillTopic, QoS), +% ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), +% send_willmsg_msg(Socket, WillPayload), +% ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), +% +% % subscribe +% TopicName1 = <<"a/+/c">>, +% MsgId1 = 25, +% TopicId0 = 0, +% WillBit = 0, +% Dup = 0, +% Retain = 0, +% CleanSession = 0, +% ReturnCode = 0, +% send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), +% ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, +% receive_response(Socket)), +% +% % goto asleep state +% send_disconnect_msg(Socket, 1), +% ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), +% +% timer:sleep(300), +% +% %% send downlink data in asleep state. This message should be send to device once it wake up +% Payload1 = <<55, 66, 77, 88, 99>>, +% Payload2 = <<55, 66, 77, 88, 100>>, +% +% {ok, C} = emqtt:start_link(), +% {ok, _} = emqtt:connect(C), +% {ok, _} = emqtt:publish(C, <<"a/b/c">>, Payload1, QoS), +% {ok, _} = emqtt:publish(C, <<"a/b/c">>, Payload2, QoS), +% timer:sleep(100), +% ok = emqtt:disconnect(C), +% +% timer:sleep(300), +% +% % goto awake state, receive downlink messages, and go back to asleep +% send_pingreq_msg(Socket, ClientId), +% +% %% 1. get REGISTER first, since this topic has never been registered +% UdpData1 = receive_response(Socket), +% {TopicIdNew, MsgId3} = check_register_msg_on_udp(<<"a/b/c">>, UdpData1), +% +% %% 2. but before we reply the REGACK, the sn-gateway should not send any PUBLISH +% ?assertError(_, receive_publish(Socket)), +% +% send_regack_msg(Socket, TopicIdNew, MsgId3), +% +% UdpData2 = receive_response(Socket), +% MsgId_udp2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload1}, UdpData2), +% send_puback_msg(Socket, TopicIdNew, MsgId_udp2), +% +% UdpData3 = receive_response(Socket), +% MsgId_udp3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData3), +% send_puback_msg(Socket, TopicIdNew, MsgId_udp3), +% +% ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), +% +% gen_udp:close(Socket). receive_publish(Socket) -> UdpData3 = receive_response(Socket, 1000), <> = UdpData3, <<_:8, ?SN_PUBLISH, _/binary>> = HeaderUdp. -t_asleep_test05_to_awake_qos1_dl_msg(_) -> - QoS = 1, - Duration = 5, - WillTopic = <<"dead">>, - WillPayload = <<10, 11, 12, 13, 14>>, - {ok, Socket} = gen_udp:open(0, [binary]), - ClientId = ?CLIENTID, - send_connect_msg_with_will(Socket, Duration, ClientId), - ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), - send_willtopic_msg(Socket, WillTopic, QoS), - ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), - send_willmsg_msg(Socket, WillPayload), - ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), - - % subscribe - TopicName1 = <<"u/+/w">>, - MsgId1 = 25, - TopicId0 = 0, - WillBit = 0, - Dup = 0, - Retain = 0, - CleanSession = 0, - ReturnCode = 0, - send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, - receive_response(Socket)), - - % goto asleep state - SleepDuration = 30, - send_disconnect_msg(Socket, SleepDuration), - ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), - - timer:sleep(300), - - %% send downlink data in asleep state. This message should be send to device once it wake up - Payload2 = <<55, 66, 77, 88, 99>>, - Payload3 = <<61, 71, 81>>, - Payload4 = <<100, 101, 102, 103, 104, 105, 106, 107>>, - TopicName_test5 = <<"u/v/w">>, - {ok, C} = emqtt:start_link(), - {ok, _} = emqtt:connect(C), - {ok, _} = emqtt:publish(C, TopicName_test5, Payload2, QoS), - timer:sleep(100), - {ok, _} = emqtt:publish(C, TopicName_test5, Payload3, QoS), - timer:sleep(100), - {ok, _} = emqtt:publish(C, TopicName_test5, Payload4, QoS), - timer:sleep(200), - ok = emqtt:disconnect(C), - timer:sleep(50), - - % goto awake state, receive downlink messages, and go back to asleep - send_pingreq_msg(Socket, ClientId), - - UdpData_reg = receive_response(Socket), - {TopicIdNew, MsgId_reg} = check_register_msg_on_udp(TopicName_test5, UdpData_reg), - send_regack_msg(Socket, TopicIdNew, MsgId_reg), - - UdpData2 = receive_response(Socket), - MsgId2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData2), - send_puback_msg(Socket, TopicIdNew, MsgId2), - timer:sleep(50), - - UdpData3 = wrap_receive_response(Socket), - MsgId3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload3}, UdpData3), - send_puback_msg(Socket, TopicIdNew, MsgId3), - timer:sleep(50), - - case receive_response(Socket) of - <<2,23>> -> ok; - UdpData4 -> - MsgId4 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, - CleanSession, ?SN_NORMAL_TOPIC, - TopicIdNew, Payload4}, UdpData4), - send_puback_msg(Socket, TopicIdNew, MsgId4) - end, - ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), - gen_udp:close(Socket). - -t_asleep_test06_to_awake_qos2_dl_msg(_) -> - QoS = 2, - Duration = 1, - WillTopic = <<"dead">>, - WillPayload = <<10, 11, 12, 13, 14>>, - {ok, Socket} = gen_udp:open(0, [binary]), - ClientId = ?CLIENTID, - send_connect_msg_with_will(Socket, Duration, ClientId), - ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), - send_willtopic_msg(Socket, WillTopic, QoS), - ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), - send_willmsg_msg(Socket, WillPayload), - ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), - - % subscribe - TopicName_tom = <<"tom">>, - MsgId1 = 25, - WillBit = 0, - Dup = 0, - Retain = 0, - CleanSession = 0, - ReturnCode = 0, - send_register_msg(Socket, TopicName_tom, MsgId1), - timer:sleep(50), - TopicId_tom = check_regack_msg_on_udp(MsgId1, receive_response(Socket)), - send_subscribe_msg_predefined_topic(Socket, QoS, TopicId_tom, MsgId1), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, - ?SN_NORMAL_TOPIC:2, TopicId_tom:16, MsgId1:16, ReturnCode>>, - receive_response(Socket)), - - % goto asleep state - SleepDuration = 11, - send_disconnect_msg(Socket, SleepDuration), - ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), - - timer:sleep(100), - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% send downlink data in asleep state. This message should be send to device once it wake up - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - Payload1 = <<55, 66, 77, 88, 99>>, - {ok, C} = emqtt:start_link(), - {ok, _} = emqtt:connect(C), - {ok, _} = emqtt:publish(C, TopicName_tom, Payload1, QoS), - timer:sleep(100), - ok = emqtt:disconnect(C), - timer:sleep(300), - - % goto awake state, receive downlink messages, and go back to asleep - send_pingreq_msg(Socket, ClientId), - - UdpData = wrap_receive_response(Socket), - MsgId_udp = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicId_tom, Payload1}, UdpData), - send_pubrec_msg(Socket, MsgId_udp), - ?assertMatch(<<_:8, ?SN_PUBREL:8, _/binary>>, receive_response(Socket)), - send_pubcomp_msg(Socket, MsgId_udp), - - %% verify the pingresp is received after receiving all the buffered qos2 msgs - ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), - gen_udp:close(Socket). - -t_asleep_test07_to_connected(_) -> - QoS = 1, - Keepalive_Duration = 10, - SleepDuration = 1, - WillTopic = <<"dead">>, - WillPayload = <<10, 11, 12, 13, 14>>, - {ok, Socket} = gen_udp:open(0, [binary]), - ClientId = ?CLIENTID, - send_connect_msg_with_will(Socket, Keepalive_Duration, ClientId), - ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), - send_willtopic_msg(Socket, WillTopic, QoS), - ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), - send_willmsg_msg(Socket, WillPayload), - ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), - - % subscribe - TopicName_tom = <<"tom">>, - MsgId1 = 25, - WillBit = 0, - Dup = 0, - Retain = 0, - CleanSession = 0, - ReturnCode = 0, - send_register_msg(Socket, TopicName_tom, MsgId1), - TopicId_tom = check_regack_msg_on_udp(MsgId1, receive_response(Socket)), - send_subscribe_msg_predefined_topic(Socket, QoS, TopicId_tom, MsgId1), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId_tom:16, MsgId1:16, ReturnCode>>, - receive_response(Socket)), - - % goto asleep state - send_disconnect_msg(Socket, SleepDuration), - ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), - - timer:sleep(100), - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% send connect message, and goto connected state - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - send_connect_msg(Socket, ClientId), - ?assertEqual(<<3, ?SN_CONNACK, ?SN_RC_ACCEPTED>>, receive_response(Socket)), - - timer:sleep(1500), - % asleep timer should get timeout, without any effect - - timer:sleep(4000), - % keepalive timer should get timeout - - gen_udp:close(Socket). - -t_asleep_test08_to_disconnected(_) -> - QoS = 1, - Keepalive_Duration = 3, - SleepDuration = 1, - WillTopic = <<"dead">>, - WillPayload = <<10, 11, 12, 13, 14>>, - {ok, Socket} = gen_udp:open(0, [binary]), - ClientId = ?CLIENTID, - send_connect_msg_with_will(Socket, Keepalive_Duration, ClientId), - ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), - send_willtopic_msg(Socket, WillTopic, QoS), - ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), - send_willmsg_msg(Socket, WillPayload), - ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), - - % goto asleep state - send_disconnect_msg(Socket, SleepDuration), - ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), - - timer:sleep(100), - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% send disconnect message, and goto disconnected state - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - send_disconnect_msg(Socket, undefined), - ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), - - timer:sleep(100), - % it is a normal termination, without will message - - gen_udp:close(Socket). - -t_asleep_test09_to_awake_again_qos1_dl_msg(_) -> - QoS = 1, - Duration = 5, - WillTopic = <<"dead">>, - WillPayload = <<10, 11, 12, 13, 14>>, - {ok, Socket} = gen_udp:open(0, [binary]), - ClientId = ?CLIENTID, - send_connect_msg_with_will(Socket, Duration, ClientId), - ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), - send_willtopic_msg(Socket, WillTopic, QoS), - ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), - send_willmsg_msg(Socket, WillPayload), - ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), - - % subscribe - TopicName1 = <<"u/+/k">>, - MsgId1 = 25, - TopicId0 = 0, - WillBit = 0, - Dup = 0, - Retain = 0, - CleanSession = 0, - ReturnCode = 0, - send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, - receive_response(Socket)), - % goto asleep state - SleepDuration = 30, - send_disconnect_msg(Socket, SleepDuration), - ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), - - timer:sleep(1000), - - %% send downlink data in asleep state. This message should be send to device once it wake up - Payload2 = <<55, 66, 77, 88, 99>>, - Payload3 = <<61, 71, 81>>, - Payload4 = <<100, 101, 102, 103, 104, 105, 106, 107>>, - TopicName_test9 = <<"u/v/k">>, - {ok, C} = emqtt:start_link(), - {ok, _} = emqtt:connect(C), - {ok, _} = emqtt:publish(C, TopicName_test9, Payload2, QoS), - timer:sleep(100), - {ok, _} = emqtt:publish(C, TopicName_test9, Payload3, QoS), - timer:sleep(100), - {ok, _} = emqtt:publish(C, TopicName_test9, Payload4, QoS), - timer:sleep(1000), - ok = emqtt:disconnect(C), - - % goto awake state, receive downlink messages, and go back to asleep - send_pingreq_msg(Socket, ClientId), - - UdpData_reg = receive_response(Socket), - {TopicIdNew, MsgId_reg} = check_register_msg_on_udp(TopicName_test9, UdpData_reg), - send_regack_msg(Socket, TopicIdNew, MsgId_reg), - - case wrap_receive_response(Socket) of - udp_receive_timeout -> - ok; - UdpData2 -> - MsgId2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData2), - send_puback_msg(Socket, TopicIdNew, MsgId2) - end, - timer:sleep(100), - - case wrap_receive_response(Socket) of - udp_receive_timeout -> - ok; - UdpData3 -> - MsgId3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload3}, UdpData3), - send_puback_msg(Socket, TopicIdNew, MsgId3) - end, - timer:sleep(100), - - case wrap_receive_response(Socket) of - udp_receive_timeout -> - ok; - UdpData4 -> - MsgId4 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, - CleanSession, ?SN_NORMAL_TOPIC, - TopicIdNew, Payload4}, UdpData4), - send_puback_msg(Socket, TopicIdNew, MsgId4) - end, - ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), - - %% send PINGREQ again to enter awake state - send_pingreq_msg(Socket, ClientId), - %% will not receive any buffered PUBLISH messages buffered before last awake, only receive PINGRESP here - ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), - - gen_udp:close(Socket). - -t_awake_test01_to_connected(_) -> - QoS = 1, - Keepalive_Duration = 3, - SleepDuration = 1, - WillTopic = <<"dead">>, - WillPayload = <<10, 11, 12, 13, 14>>, - {ok, Socket} = gen_udp:open(0, [binary]), - ClientId = ?CLIENTID, - send_connect_msg_with_will(Socket, Keepalive_Duration, ClientId), - ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), - send_willtopic_msg(Socket, WillTopic, QoS), - ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), - send_willmsg_msg(Socket, WillPayload), - ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), - - % goto asleep state - send_disconnect_msg(Socket, SleepDuration), - ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), - - timer:sleep(100), - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% send connect message, and goto connected state - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - send_connect_msg(Socket, ClientId), - ?assertEqual(<<3, ?SN_CONNACK, ?SN_RC_ACCEPTED>>, receive_response(Socket)), - - timer:sleep(1500), - % asleep timer should get timeout - - timer:sleep(4000), - % keepalive timer should get timeout - gen_udp:close(Socket). - -t_awake_test02_to_disconnected(_) -> - QoS = 1, - Keepalive_Duration = 3, - SleepDuration = 1, - WillTopic = <<"dead">>, - WillPayload = <<10, 11, 12, 13, 14>>, - {ok, Socket} = gen_udp:open(0, [binary]), - ClientId = ?CLIENTID, - send_connect_msg_with_will(Socket, Keepalive_Duration, ClientId), - ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), - send_willtopic_msg(Socket, WillTopic, QoS), - ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), - send_willmsg_msg(Socket, WillPayload), - ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), - - - % goto asleep state - send_disconnect_msg(Socket, SleepDuration), - ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), - - timer:sleep(100), - - % goto awake state - send_pingreq_msg(Socket, ClientId), - ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% send disconnect message, and goto disconnected state - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - send_disconnect_msg(Socket, undefined), - ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), - - timer:sleep(100), - % it is a normal termination, no will message will be send - - gen_udp:close(Socket). +%t_asleep_test05_to_awake_qos1_dl_msg(_) -> +% QoS = 1, +% Duration = 5, +% WillTopic = <<"dead">>, +% WillPayload = <<10, 11, 12, 13, 14>>, +% {ok, Socket} = gen_udp:open(0, [binary]), +% ClientId = ?CLIENTID, +% send_connect_msg_with_will(Socket, Duration, ClientId), +% ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), +% send_willtopic_msg(Socket, WillTopic, QoS), +% ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), +% send_willmsg_msg(Socket, WillPayload), +% ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), +% +% % subscribe +% TopicName1 = <<"u/+/w">>, +% MsgId1 = 25, +% TopicId0 = 0, +% WillBit = 0, +% Dup = 0, +% Retain = 0, +% CleanSession = 0, +% ReturnCode = 0, +% send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), +% ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, +% receive_response(Socket)), +% +% % goto asleep state +% SleepDuration = 30, +% send_disconnect_msg(Socket, SleepDuration), +% ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), +% +% timer:sleep(300), +% +% %% send downlink data in asleep state. This message should be send to device once it wake up +% Payload2 = <<55, 66, 77, 88, 99>>, +% Payload3 = <<61, 71, 81>>, +% Payload4 = <<100, 101, 102, 103, 104, 105, 106, 107>>, +% TopicName_test5 = <<"u/v/w">>, +% {ok, C} = emqtt:start_link(), +% {ok, _} = emqtt:connect(C), +% {ok, _} = emqtt:publish(C, TopicName_test5, Payload2, QoS), +% timer:sleep(100), +% {ok, _} = emqtt:publish(C, TopicName_test5, Payload3, QoS), +% timer:sleep(100), +% {ok, _} = emqtt:publish(C, TopicName_test5, Payload4, QoS), +% timer:sleep(200), +% ok = emqtt:disconnect(C), +% timer:sleep(50), +% +% % goto awake state, receive downlink messages, and go back to asleep +% send_pingreq_msg(Socket, ClientId), +% +% UdpData_reg = receive_response(Socket), +% {TopicIdNew, MsgId_reg} = check_register_msg_on_udp(TopicName_test5, UdpData_reg), +% send_regack_msg(Socket, TopicIdNew, MsgId_reg), +% +% UdpData2 = receive_response(Socket), +% MsgId2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData2), +% send_puback_msg(Socket, TopicIdNew, MsgId2), +% timer:sleep(50), +% +% UdpData3 = wrap_receive_response(Socket), +% MsgId3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload3}, UdpData3), +% send_puback_msg(Socket, TopicIdNew, MsgId3), +% timer:sleep(50), +% +% case receive_response(Socket) of +% <<2,23>> -> ok; +% UdpData4 -> +% MsgId4 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, +% CleanSession, ?SN_NORMAL_TOPIC, +% TopicIdNew, Payload4}, UdpData4), +% send_puback_msg(Socket, TopicIdNew, MsgId4) +% end, +% ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), +% gen_udp:close(Socket). +% +%t_asleep_test06_to_awake_qos2_dl_msg(_) -> +% QoS = 2, +% Duration = 1, +% WillTopic = <<"dead">>, +% WillPayload = <<10, 11, 12, 13, 14>>, +% {ok, Socket} = gen_udp:open(0, [binary]), +% ClientId = ?CLIENTID, +% send_connect_msg_with_will(Socket, Duration, ClientId), +% ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), +% send_willtopic_msg(Socket, WillTopic, QoS), +% ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), +% send_willmsg_msg(Socket, WillPayload), +% ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), +% +% % subscribe +% TopicName_tom = <<"tom">>, +% MsgId1 = 25, +% WillBit = 0, +% Dup = 0, +% Retain = 0, +% CleanSession = 0, +% ReturnCode = 0, +% send_register_msg(Socket, TopicName_tom, MsgId1), +% timer:sleep(50), +% TopicId_tom = check_regack_msg_on_udp(MsgId1, receive_response(Socket)), +% send_subscribe_msg_predefined_topic(Socket, QoS, TopicId_tom, MsgId1), +% ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, +% ?SN_NORMAL_TOPIC:2, TopicId_tom:16, MsgId1:16, ReturnCode>>, +% receive_response(Socket)), +% +% % goto asleep state +% SleepDuration = 11, +% send_disconnect_msg(Socket, SleepDuration), +% ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), +% +% timer:sleep(100), +% +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% %% send downlink data in asleep state. This message should be send to device once it wake up +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Payload1 = <<55, 66, 77, 88, 99>>, +% {ok, C} = emqtt:start_link(), +% {ok, _} = emqtt:connect(C), +% {ok, _} = emqtt:publish(C, TopicName_tom, Payload1, QoS), +% timer:sleep(100), +% ok = emqtt:disconnect(C), +% timer:sleep(300), +% +% % goto awake state, receive downlink messages, and go back to asleep +% send_pingreq_msg(Socket, ClientId), +% +% UdpData = wrap_receive_response(Socket), +% MsgId_udp = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicId_tom, Payload1}, UdpData), +% send_pubrec_msg(Socket, MsgId_udp), +% ?assertMatch(<<_:8, ?SN_PUBREL:8, _/binary>>, receive_response(Socket)), +% send_pubcomp_msg(Socket, MsgId_udp), +% +% %% verify the pingresp is received after receiving all the buffered qos2 msgs +% ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), +% gen_udp:close(Socket). +% +%t_asleep_test07_to_connected(_) -> +% QoS = 1, +% Keepalive_Duration = 10, +% SleepDuration = 1, +% WillTopic = <<"dead">>, +% WillPayload = <<10, 11, 12, 13, 14>>, +% {ok, Socket} = gen_udp:open(0, [binary]), +% ClientId = ?CLIENTID, +% send_connect_msg_with_will(Socket, Keepalive_Duration, ClientId), +% ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), +% send_willtopic_msg(Socket, WillTopic, QoS), +% ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), +% send_willmsg_msg(Socket, WillPayload), +% ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), +% +% % subscribe +% TopicName_tom = <<"tom">>, +% MsgId1 = 25, +% WillBit = 0, +% Dup = 0, +% Retain = 0, +% CleanSession = 0, +% ReturnCode = 0, +% send_register_msg(Socket, TopicName_tom, MsgId1), +% TopicId_tom = check_regack_msg_on_udp(MsgId1, receive_response(Socket)), +% send_subscribe_msg_predefined_topic(Socket, QoS, TopicId_tom, MsgId1), +% ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId_tom:16, MsgId1:16, ReturnCode>>, +% receive_response(Socket)), +% +% % goto asleep state +% send_disconnect_msg(Socket, SleepDuration), +% ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), +% +% timer:sleep(100), +% +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% %% send connect message, and goto connected state +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% send_connect_msg(Socket, ClientId), +% ?assertEqual(<<3, ?SN_CONNACK, ?SN_RC_ACCEPTED>>, receive_response(Socket)), +% +% timer:sleep(1500), +% % asleep timer should get timeout, without any effect +% +% timer:sleep(4000), +% % keepalive timer should get timeout +% +% gen_udp:close(Socket). +% +%t_asleep_test08_to_disconnected(_) -> +% QoS = 1, +% Keepalive_Duration = 3, +% SleepDuration = 1, +% WillTopic = <<"dead">>, +% WillPayload = <<10, 11, 12, 13, 14>>, +% {ok, Socket} = gen_udp:open(0, [binary]), +% ClientId = ?CLIENTID, +% send_connect_msg_with_will(Socket, Keepalive_Duration, ClientId), +% ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), +% send_willtopic_msg(Socket, WillTopic, QoS), +% ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), +% send_willmsg_msg(Socket, WillPayload), +% ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), +% +% % goto asleep state +% send_disconnect_msg(Socket, SleepDuration), +% ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), +% +% timer:sleep(100), +% +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% %% send disconnect message, and goto disconnected state +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% send_disconnect_msg(Socket, undefined), +% ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), +% +% timer:sleep(100), +% % it is a normal termination, without will message +% +% gen_udp:close(Socket). +% +%t_asleep_test09_to_awake_again_qos1_dl_msg(_) -> +% QoS = 1, +% Duration = 5, +% WillTopic = <<"dead">>, +% WillPayload = <<10, 11, 12, 13, 14>>, +% {ok, Socket} = gen_udp:open(0, [binary]), +% ClientId = ?CLIENTID, +% send_connect_msg_with_will(Socket, Duration, ClientId), +% ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), +% send_willtopic_msg(Socket, WillTopic, QoS), +% ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), +% send_willmsg_msg(Socket, WillPayload), +% ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), +% +% % subscribe +% TopicName1 = <<"u/+/k">>, +% MsgId1 = 25, +% TopicId0 = 0, +% WillBit = 0, +% Dup = 0, +% Retain = 0, +% CleanSession = 0, +% ReturnCode = 0, +% send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), +% ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, +% receive_response(Socket)), +% % goto asleep state +% SleepDuration = 30, +% send_disconnect_msg(Socket, SleepDuration), +% ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), +% +% timer:sleep(1000), +% +% %% send downlink data in asleep state. This message should be send to device once it wake up +% Payload2 = <<55, 66, 77, 88, 99>>, +% Payload3 = <<61, 71, 81>>, +% Payload4 = <<100, 101, 102, 103, 104, 105, 106, 107>>, +% TopicName_test9 = <<"u/v/k">>, +% {ok, C} = emqtt:start_link(), +% {ok, _} = emqtt:connect(C), +% {ok, _} = emqtt:publish(C, TopicName_test9, Payload2, QoS), +% timer:sleep(100), +% {ok, _} = emqtt:publish(C, TopicName_test9, Payload3, QoS), +% timer:sleep(100), +% {ok, _} = emqtt:publish(C, TopicName_test9, Payload4, QoS), +% timer:sleep(1000), +% ok = emqtt:disconnect(C), +% +% % goto awake state, receive downlink messages, and go back to asleep +% send_pingreq_msg(Socket, ClientId), +% +% UdpData_reg = receive_response(Socket), +% {TopicIdNew, MsgId_reg} = check_register_msg_on_udp(TopicName_test9, UdpData_reg), +% send_regack_msg(Socket, TopicIdNew, MsgId_reg), +% +% case wrap_receive_response(Socket) of +% udp_receive_timeout -> +% ok; +% UdpData2 -> +% MsgId2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData2), +% send_puback_msg(Socket, TopicIdNew, MsgId2) +% end, +% timer:sleep(100), +% +% case wrap_receive_response(Socket) of +% udp_receive_timeout -> +% ok; +% UdpData3 -> +% MsgId3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload3}, UdpData3), +% send_puback_msg(Socket, TopicIdNew, MsgId3) +% end, +% timer:sleep(100), +% +% case wrap_receive_response(Socket) of +% udp_receive_timeout -> +% ok; +% UdpData4 -> +% MsgId4 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, +% CleanSession, ?SN_NORMAL_TOPIC, +% TopicIdNew, Payload4}, UdpData4), +% send_puback_msg(Socket, TopicIdNew, MsgId4) +% end, +% ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), +% +% %% send PINGREQ again to enter awake state +% send_pingreq_msg(Socket, ClientId), +% %% will not receive any buffered PUBLISH messages buffered before last awake, only receive PINGRESP here +% ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), +% +% gen_udp:close(Socket). +% +%t_awake_test01_to_connected(_) -> +% QoS = 1, +% Keepalive_Duration = 3, +% SleepDuration = 1, +% WillTopic = <<"dead">>, +% WillPayload = <<10, 11, 12, 13, 14>>, +% {ok, Socket} = gen_udp:open(0, [binary]), +% ClientId = ?CLIENTID, +% send_connect_msg_with_will(Socket, Keepalive_Duration, ClientId), +% ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), +% send_willtopic_msg(Socket, WillTopic, QoS), +% ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), +% send_willmsg_msg(Socket, WillPayload), +% ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), +% +% % goto asleep state +% send_disconnect_msg(Socket, SleepDuration), +% ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), +% +% timer:sleep(100), +% +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% %% send connect message, and goto connected state +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% send_connect_msg(Socket, ClientId), +% ?assertEqual(<<3, ?SN_CONNACK, ?SN_RC_ACCEPTED>>, receive_response(Socket)), +% +% timer:sleep(1500), +% % asleep timer should get timeout +% +% timer:sleep(4000), +% % keepalive timer should get timeout +% gen_udp:close(Socket). +% +%t_awake_test02_to_disconnected(_) -> +% QoS = 1, +% Keepalive_Duration = 3, +% SleepDuration = 1, +% WillTopic = <<"dead">>, +% WillPayload = <<10, 11, 12, 13, 14>>, +% {ok, Socket} = gen_udp:open(0, [binary]), +% ClientId = ?CLIENTID, +% send_connect_msg_with_will(Socket, Keepalive_Duration, ClientId), +% ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), +% send_willtopic_msg(Socket, WillTopic, QoS), +% ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), +% send_willmsg_msg(Socket, WillPayload), +% ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), +% +% +% % goto asleep state +% send_disconnect_msg(Socket, SleepDuration), +% ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), +% +% timer:sleep(100), +% +% % goto awake state +% send_pingreq_msg(Socket, ClientId), +% ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), +% +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% %% send disconnect message, and goto disconnected state +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% send_disconnect_msg(Socket, undefined), +% ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), +% +% timer:sleep(100), +% % it is a normal termination, no will message will be send +% +% gen_udp:close(Socket). t_broadcast_test1(_) -> {ok, Socket} = gen_udp:open( 0, [binary]), diff --git a/apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl b/apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl new file mode 100644 index 000000000..6161687f2 --- /dev/null +++ b/apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl @@ -0,0 +1,127 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_sn_registry_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-define(REGISTRY, emqx_sn_registry). +-define(MAX_PREDEF_ID, 2). +-define(PREDEF_TOPICS, [#{id => 1, topic => <<"/predefined/topic/name/hello">>}, + #{id => 2, topic => <<"/predefined/topic/name/nice">>}]). + +-define(INSTA_ID, 'mqttsn#1'). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + application:ensure_all_started(ekka), + ekka_mnesia:start(), + Config. + +end_per_suite(_Config) -> + application:stop(ekka), + ok. + +init_per_testcase(_TestCase, Config) -> + {ok, Pid} = ?REGISTRY:start_link(?INSTA_ID, ?PREDEF_TOPICS), + {Tab, Pid} = ?REGISTRY:lookup_name(Pid), + [{reg, {Tab, Pid}} | Config]. + +end_per_testcase(_TestCase, Config) -> + {Tab, _Pid} = proplists:get_value(reg, Config), + ekka_mnesia:clear_table(Tab), + Config. + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_register(Config) -> + Reg = proplists:get_value(reg, Config), + ?assertEqual(?MAX_PREDEF_ID+1, ?REGISTRY:register_topic(Reg, <<"ClientId">>, <<"Topic1">>)), + ?assertEqual(?MAX_PREDEF_ID+2, ?REGISTRY:register_topic(Reg, <<"ClientId">>, <<"Topic2">>)), + ?assertEqual(<<"Topic1">>, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID+1)), + ?assertEqual(<<"Topic2">>, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID+2)), + ?assertEqual(?MAX_PREDEF_ID+1, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, <<"Topic1">>)), + ?assertEqual(?MAX_PREDEF_ID+2, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, <<"Topic2">>)), + emqx_sn_registry:unregister_topic(Reg, <<"ClientId">>), + ?assertEqual(undefined, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID+1)), + ?assertEqual(undefined, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID+2)), + ?assertEqual(undefined, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, <<"Topic1">>)), + ?assertEqual(undefined, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, <<"Topic2">>)). + +t_register_case2(Config) -> + Reg = proplists:get_value(reg, Config), + ?assertEqual(?MAX_PREDEF_ID+1, ?REGISTRY:register_topic(Reg, <<"ClientId">>, <<"Topic1">>)), + ?assertEqual(?MAX_PREDEF_ID+2, ?REGISTRY:register_topic(Reg, <<"ClientId">>, <<"Topic2">>)), + ?assertEqual(?MAX_PREDEF_ID+1, ?REGISTRY:register_topic(Reg, <<"ClientId">>, <<"Topic1">>)), + ?assertEqual(<<"Topic1">>, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID+1)), + ?assertEqual(<<"Topic2">>, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID+2)), + ?assertEqual(?MAX_PREDEF_ID+1, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, <<"Topic1">>)), + ?assertEqual(?MAX_PREDEF_ID+2, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, <<"Topic2">>)), + ?assertEqual(undefined, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, <<"Topic3">>)), + ?REGISTRY:unregister_topic(Reg, <<"ClientId">>), + ?assertEqual(undefined, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID+1)), + ?assertEqual(undefined, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID+2)), + ?assertEqual(undefined, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, <<"Topic1">>)), + ?assertEqual(undefined, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, <<"Topic2">>)). + +t_reach_maximum(Config) -> + Reg = proplists:get_value(reg, Config), + register_a_lot(?MAX_PREDEF_ID+1, 16#ffff, Reg), + ?assertEqual({error, too_large}, ?REGISTRY:register_topic(Reg, <<"ClientId">>, <<"TopicABC">>)), + Topic1 = iolist_to_binary(io_lib:format("Topic~p", [?MAX_PREDEF_ID+1])), + Topic2 = iolist_to_binary(io_lib:format("Topic~p", [?MAX_PREDEF_ID+2])), + ?assertEqual(?MAX_PREDEF_ID+1, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, Topic1)), + ?assertEqual(?MAX_PREDEF_ID+2, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, Topic2)), + ?REGISTRY:unregister_topic(Reg, <<"ClientId">>), + ?assertEqual(undefined, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID+1)), + ?assertEqual(undefined, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID+2)), + ?assertEqual(undefined, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, Topic1)), + ?assertEqual(undefined, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, Topic2)). + +t_register_case4(Config) -> + Reg = proplists:get_value(reg, Config), + ?assertEqual(?MAX_PREDEF_ID+1, ?REGISTRY:register_topic(Reg, <<"ClientId">>, <<"TopicA">>)), + ?assertEqual(?MAX_PREDEF_ID+2, ?REGISTRY:register_topic(Reg, <<"ClientId">>, <<"TopicB">>)), + ?assertEqual(?MAX_PREDEF_ID+3, ?REGISTRY:register_topic(Reg, <<"ClientId">>, <<"TopicC">>)), + ?REGISTRY:unregister_topic(Reg, <<"ClientId">>), + ?assertEqual(?MAX_PREDEF_ID+1, ?REGISTRY:register_topic(Reg, <<"ClientId">>, <<"TopicD">>)). + +t_deny_wildcard_topic(Config) -> + Reg = proplists:get_value(reg, Config), + ?assertEqual({error, wildcard_topic}, ?REGISTRY:register_topic(Reg, <<"ClientId">>, <<"/TopicA/#">>)), + ?assertEqual({error, wildcard_topic}, ?REGISTRY:register_topic(Reg, <<"ClientId">>, <<"/+/TopicB">>)). + +%%-------------------------------------------------------------------- +%% Helper funcs +%%-------------------------------------------------------------------- + +register_a_lot(Max, Max, _Reg) -> + ok; +register_a_lot(N, Max, Reg) when N < Max -> + Topic = iolist_to_binary(["Topic", integer_to_list(N)]), + ?assertEqual(N, ?REGISTRY:register_topic(Reg, <<"ClientId">>, Topic)), + register_a_lot(N+1, Max, Reg). diff --git a/apps/emqx_stomp/test/emqx_stomp_SUITE.erl b/apps/emqx_gateway/test/emqx_stomp_SUITE.erl similarity index 84% rename from apps/emqx_stomp/test/emqx_stomp_SUITE.erl rename to apps/emqx_gateway/test/emqx_stomp_SUITE.erl index 9a5d9698e..261601b3e 100644 --- a/apps/emqx_stomp/test/emqx_stomp_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_stomp_SUITE.erl @@ -16,25 +16,42 @@ -module(emqx_stomp_SUITE). --include_lib("emqx_stomp/include/emqx_stomp.hrl"). +-include_lib("emqx_gateway/src/stomp/include/emqx_stomp.hrl"). -compile(export_all). -compile(nowarn_export_all). -define(HEARTBEAT, <<$\n>>). +-define(CONF_DEFAULT, <<""" +gateway: { + stomp.1: { + authenticator: allow_anonymous + clientinfo_override: { + username: \"${Packet.headers.login}\" + password: \"${Packet.headers.passcode}\" + } + listener.tcp.1: { + bind: 61613 + } + } +} +""">>). + all() -> emqx_ct:all(?MODULE). %%-------------------------------------------------------------------- %% Setups %%-------------------------------------------------------------------- -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_stomp]), - Config. +init_per_suite(Cfg) -> + ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT), + emqx_ct_helpers:start_apps([emqx_gateway]), + Cfg. -end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_stomp]). +end_per_suite(_Cfg) -> + emqx_ct_helpers:stop_apps([emqx_gateway]), + ok. %%-------------------------------------------------------------------- %% Test Cases @@ -52,7 +69,7 @@ t_connect(_) -> {ok, Data} = gen_tcp:recv(Sock, 0), {ok, Frame = #stomp_frame{command = <<"CONNECTED">>, headers = _, - body = _}, _} = parse(Data), + body = _}, _, _} = parse(Data), <<"2000,1000">> = proplists:get_value(<<"heart-beat">>, Frame#stomp_frame.headers), gen_tcp:send(Sock, serialize(<<"DISCONNECT">>, @@ -61,22 +78,23 @@ t_connect(_) -> {ok, Data1} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"RECEIPT">>, headers = [{<<"receipt-id">>, <<"12345">>}], - body = _}, _} = parse(Data1) + body = _}, _, _} = parse(Data1) end), %% Connect will be failed, because of bad login or passcode - with_connection(fun(Sock) -> - gen_tcp:send(Sock, serialize(<<"CONNECT">>, - [{<<"accept-version">>, ?STOMP_VER}, - {<<"host">>, <<"127.0.0.1:61613">>}, - {<<"login">>, <<"admin">>}, - {<<"passcode">>, <<"admin">>}, - {<<"heart-beat">>, <<"1000,2000">>}])), - {ok, Data} = gen_tcp:recv(Sock, 0), - {ok, #stomp_frame{command = <<"ERROR">>, - headers = _, - body = <<"Login or passcode error!">>}, _} = parse(Data) - end), + %% FIXME: Waiting for authentication works + %with_connection(fun(Sock) -> + % gen_tcp:send(Sock, serialize(<<"CONNECT">>, + % [{<<"accept-version">>, ?STOMP_VER}, + % {<<"host">>, <<"127.0.0.1:61613">>}, + % {<<"login">>, <<"admin">>}, + % {<<"passcode">>, <<"admin">>}, + % {<<"heart-beat">>, <<"1000,2000">>}])), + % {ok, Data} = gen_tcp:recv(Sock, 0), + % {ok, #stomp_frame{command = <<"ERROR">>, + % headers = _, + % body = <<"Login or passcode error!">>}, _, _} = parse(Data) + % end), %% Connect will be failed, because of bad version with_connection(fun(Sock) -> @@ -89,7 +107,7 @@ t_connect(_) -> {ok, Data} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"ERROR">>, headers = _, - body = <<"Supported protocol versions < 1.2">>}, _} = parse(Data) + body = <<"Login Failed: Supported protocol versions < 1.2">>}, _, _} = parse(Data) end). t_heartbeat(_) -> @@ -104,7 +122,7 @@ t_heartbeat(_) -> {ok, Data} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"CONNECTED">>, headers = _, - body = _}, _} = parse(Data), + body = _}, _, _} = parse(Data), {ok, ?HEARTBEAT} = gen_tcp:recv(Sock, 0), %% Server will close the connection because never receive the heart beat from client @@ -122,7 +140,7 @@ t_subscribe(_) -> {ok, Data} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"CONNECTED">>, headers = _, - body = _}, _} = parse(Data), + body = _}, _, _} = parse(Data), %% Subscribe gen_tcp:send(Sock, serialize(<<"SUBSCRIBE">>, @@ -139,7 +157,7 @@ t_subscribe(_) -> {ok, Data1} = gen_tcp:recv(Sock, 0, 1000), {ok, Frame = #stomp_frame{command = <<"MESSAGE">>, headers = _, - body = <<"hello">>}, _} = parse(Data1), + body = <<"hello">>}, _, _} = parse(Data1), lists:foreach(fun({Key, Val}) -> Val = proplists:get_value(Key, Frame#stomp_frame.headers) end, [{<<"destination">>, <<"/queue/foo">>}, @@ -155,7 +173,7 @@ t_subscribe(_) -> {ok, #stomp_frame{command = <<"RECEIPT">>, headers = [{<<"receipt-id">>, <<"12345">>}], - body = _}, _} = parse(Data2), + body = _}, _, _} = parse(Data2), gen_tcp:send(Sock, serialize(<<"SEND">>, [{<<"destination">>, <<"/queue/foo">>}], @@ -175,7 +193,7 @@ t_transaction(_) -> {ok, Data} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"CONNECTED">>, headers = _, - body = _}, _} = parse(Data), + body = _}, _, _} = parse(Data), %% Subscribe gen_tcp:send(Sock, serialize(<<"SUBSCRIBE">>, @@ -208,12 +226,12 @@ t_transaction(_) -> {ok, #stomp_frame{command = <<"MESSAGE">>, headers = _, - body = <<"hello">>}, Rest1} = parse(Data1), + body = <<"hello">>}, Rest1, _} = parse(Data1), %{ok, Data2} = gen_tcp:recv(Sock, 0, 500), {ok, #stomp_frame{command = <<"MESSAGE">>, headers = _, - body = <<"hello again">>}, _Rest2} = parse(Rest1), + body = <<"hello again">>}, _Rest2, _} = parse(Rest1), %% Transaction: tx2 gen_tcp:send(Sock, serialize(<<"BEGIN">>, @@ -236,7 +254,7 @@ t_transaction(_) -> {ok, Data3} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"RECEIPT">>, headers = [{<<"receipt-id">>, <<"12345">>}], - body = _}, _} = parse(Data3) + body = _}, _, _} = parse(Data3) end). t_receipt_in_error(_) -> @@ -250,7 +268,7 @@ t_receipt_in_error(_) -> {ok, Data} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"CONNECTED">>, headers = _, - body = _}, _} = parse(Data), + body = _}, _, _} = parse(Data), gen_tcp:send(Sock, serialize(<<"ABORT">>, [{<<"transaction">>, <<"tx1">>}, @@ -259,7 +277,7 @@ t_receipt_in_error(_) -> {ok, Data1} = gen_tcp:recv(Sock, 0), {ok, Frame = #stomp_frame{command = <<"ERROR">>, headers = _, - body = <<"Transaction tx1 not found">>}, _} = parse(Data1), + body = <<"Transaction tx1 not found">>}, _, _} = parse(Data1), <<"12345">> = proplists:get_value(<<"receipt-id">>, Frame#stomp_frame.headers) end). @@ -275,7 +293,7 @@ t_ack(_) -> {ok, Data} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"CONNECTED">>, headers = _, - body = _}, _} = parse(Data), + body = _}, _, _} = parse(Data), %% Subscribe gen_tcp:send(Sock, serialize(<<"SUBSCRIBE">>, @@ -290,7 +308,7 @@ t_ack(_) -> {ok, Data1} = gen_tcp:recv(Sock, 0), {ok, Frame = #stomp_frame{command = <<"MESSAGE">>, headers = _, - body = <<"ack test">>}, _} = parse(Data1), + body = <<"ack test">>}, _, _} = parse(Data1), AckId = proplists:get_value(<<"ack">>, Frame#stomp_frame.headers), @@ -301,7 +319,7 @@ t_ack(_) -> {ok, Data2} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"RECEIPT">>, headers = [{<<"receipt-id">>, <<"12345">>}], - body = _}, _} = parse(Data2), + body = _}, _, _} = parse(Data2), gen_tcp:send(Sock, serialize(<<"SEND">>, [{<<"destination">>, <<"/queue/foo">>}], @@ -310,7 +328,7 @@ t_ack(_) -> {ok, Data3} = gen_tcp:recv(Sock, 0), {ok, Frame1 = #stomp_frame{command = <<"MESSAGE">>, headers = _, - body = <<"nack test">>}, _} = parse(Data3), + body = <<"nack test">>}, _, _} = parse(Data3), AckId1 = proplists:get_value(<<"ack">>, Frame1#stomp_frame.headers), @@ -321,9 +339,16 @@ t_ack(_) -> {ok, Data4} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"RECEIPT">>, headers = [{<<"receipt-id">>, <<"12345">>}], - body = _}, _} = parse(Data4) + body = _}, _, _} = parse(Data4) end). +%% TODO: Mountpoint, AuthChain, Authorization + Mountpoint, ClientInfoOverride, +%% Listeners, Metrics, Stats, ClientInfo +%% +%% TODO: Start/Stop, List Instace +%% +%% TODO: RateLimit, OOM, + with_connection(DoFun) -> {ok, Sock} = gen_tcp:connect({127, 0, 0, 1}, 61613, @@ -336,14 +361,15 @@ with_connection(DoFun) -> end. serialize(Command, Headers) -> - emqx_stomp_frame:serialize(emqx_stomp_frame:make(Command, Headers)). + emqx_stomp_frame:serialize_pkt(emqx_stomp_frame:make(Command, Headers), #{}). serialize(Command, Headers, Body) -> - emqx_stomp_frame:serialize(emqx_stomp_frame:make(Command, Headers, Body)). + emqx_stomp_frame:serialize_pkt(emqx_stomp_frame:make(Command, Headers, Body), #{}). parse(Data) -> - ProtoEnv = [{max_headers, 10}, - {max_header_length, 1024}, - {max_body_length, 8192}], - Parser = emqx_stomp_frame:init_parer_state(ProtoEnv), + ProtoEnv = #{max_headers => 10, + max_header_length => 1024, + max_body_length => 8192 + }, + Parser = emqx_stomp_frame:initial_parse_state(ProtoEnv), emqx_stomp_frame:parse(Data, Parser). diff --git a/apps/emqx_stomp/test/emqx_stomp_heartbeat_SUITE.erl b/apps/emqx_gateway/test/emqx_stomp_heartbeat_SUITE.erl similarity index 100% rename from apps/emqx_stomp/test/emqx_stomp_heartbeat_SUITE.erl rename to apps/emqx_gateway/test/emqx_stomp_heartbeat_SUITE.erl diff --git a/apps/emqx_sn/intergration_test/Makefile b/apps/emqx_gateway/test/intergration_test/Makefile similarity index 100% rename from apps/emqx_sn/intergration_test/Makefile rename to apps/emqx_gateway/test/intergration_test/Makefile diff --git a/apps/emqx_sn/intergration_test/README.md b/apps/emqx_gateway/test/intergration_test/README.md similarity index 100% rename from apps/emqx_sn/intergration_test/README.md rename to apps/emqx_gateway/test/intergration_test/README.md diff --git a/apps/emqx_sn/intergration_test/add_emqx_sn_to_project.py b/apps/emqx_gateway/test/intergration_test/add_emqx_sn_to_project.py similarity index 100% rename from apps/emqx_sn/intergration_test/add_emqx_sn_to_project.py rename to apps/emqx_gateway/test/intergration_test/add_emqx_sn_to_project.py diff --git a/apps/emqx_sn/intergration_test/client/case1_qos0pub.c b/apps/emqx_gateway/test/intergration_test/client/case1_qos0pub.c similarity index 100% rename from apps/emqx_sn/intergration_test/client/case1_qos0pub.c rename to apps/emqx_gateway/test/intergration_test/client/case1_qos0pub.c diff --git a/apps/emqx_sn/intergration_test/client/case1_qos0sub.c b/apps/emqx_gateway/test/intergration_test/client/case1_qos0sub.c similarity index 100% rename from apps/emqx_sn/intergration_test/client/case1_qos0sub.c rename to apps/emqx_gateway/test/intergration_test/client/case1_qos0sub.c diff --git a/apps/emqx_sn/intergration_test/client/case2_qos0pub.c b/apps/emqx_gateway/test/intergration_test/client/case2_qos0pub.c similarity index 100% rename from apps/emqx_sn/intergration_test/client/case2_qos0pub.c rename to apps/emqx_gateway/test/intergration_test/client/case2_qos0pub.c diff --git a/apps/emqx_sn/intergration_test/client/case2_qos0sub.c b/apps/emqx_gateway/test/intergration_test/client/case2_qos0sub.c similarity index 100% rename from apps/emqx_sn/intergration_test/client/case2_qos0sub.c rename to apps/emqx_gateway/test/intergration_test/client/case2_qos0sub.c diff --git a/apps/emqx_sn/intergration_test/client/case3_qos0pub.c b/apps/emqx_gateway/test/intergration_test/client/case3_qos0pub.c similarity index 100% rename from apps/emqx_sn/intergration_test/client/case3_qos0pub.c rename to apps/emqx_gateway/test/intergration_test/client/case3_qos0pub.c diff --git a/apps/emqx_sn/intergration_test/client/case3_qos0sub.c b/apps/emqx_gateway/test/intergration_test/client/case3_qos0sub.c similarity index 100% rename from apps/emqx_sn/intergration_test/client/case3_qos0sub.c rename to apps/emqx_gateway/test/intergration_test/client/case3_qos0sub.c diff --git a/apps/emqx_sn/intergration_test/client/case4_qos3pub.c b/apps/emqx_gateway/test/intergration_test/client/case4_qos3pub.c similarity index 100% rename from apps/emqx_sn/intergration_test/client/case4_qos3pub.c rename to apps/emqx_gateway/test/intergration_test/client/case4_qos3pub.c diff --git a/apps/emqx_sn/intergration_test/client/case4_qos3sub.c b/apps/emqx_gateway/test/intergration_test/client/case4_qos3sub.c similarity index 100% rename from apps/emqx_sn/intergration_test/client/case4_qos3sub.c rename to apps/emqx_gateway/test/intergration_test/client/case4_qos3sub.c diff --git a/apps/emqx_sn/intergration_test/client/case5_qos3pub.c b/apps/emqx_gateway/test/intergration_test/client/case5_qos3pub.c similarity index 100% rename from apps/emqx_sn/intergration_test/client/case5_qos3pub.c rename to apps/emqx_gateway/test/intergration_test/client/case5_qos3pub.c diff --git a/apps/emqx_sn/intergration_test/client/case5_qos3sub.c b/apps/emqx_gateway/test/intergration_test/client/case5_qos3sub.c similarity index 100% rename from apps/emqx_sn/intergration_test/client/case5_qos3sub.c rename to apps/emqx_gateway/test/intergration_test/client/case5_qos3sub.c diff --git a/apps/emqx_sn/intergration_test/client/case6_sleep.c b/apps/emqx_gateway/test/intergration_test/client/case6_sleep.c similarity index 100% rename from apps/emqx_sn/intergration_test/client/case6_sleep.c rename to apps/emqx_gateway/test/intergration_test/client/case6_sleep.c diff --git a/apps/emqx_sn/intergration_test/client/case7_double_connect.c b/apps/emqx_gateway/test/intergration_test/client/case7_double_connect.c similarity index 100% rename from apps/emqx_sn/intergration_test/client/case7_double_connect.c rename to apps/emqx_gateway/test/intergration_test/client/case7_double_connect.c diff --git a/apps/emqx_sn/intergration_test/client/int_test_result.c b/apps/emqx_gateway/test/intergration_test/client/int_test_result.c similarity index 100% rename from apps/emqx_sn/intergration_test/client/int_test_result.c rename to apps/emqx_gateway/test/intergration_test/client/int_test_result.c diff --git a/apps/emqx_sn/intergration_test/client/int_test_result.h b/apps/emqx_gateway/test/intergration_test/client/int_test_result.h similarity index 100% rename from apps/emqx_sn/intergration_test/client/int_test_result.h rename to apps/emqx_gateway/test/intergration_test/client/int_test_result.h diff --git a/apps/emqx_sn/intergration_test/disable_qos3.py b/apps/emqx_gateway/test/intergration_test/disable_qos3.py similarity index 100% rename from apps/emqx_sn/intergration_test/disable_qos3.py rename to apps/emqx_gateway/test/intergration_test/disable_qos3.py diff --git a/apps/emqx_sn/intergration_test/enable_qos3.py b/apps/emqx_gateway/test/intergration_test/enable_qos3.py similarity index 100% rename from apps/emqx_sn/intergration_test/enable_qos3.py rename to apps/emqx_gateway/test/intergration_test/enable_qos3.py diff --git a/apps/emqx_sn/test/props/emqx_sn_proper_types.erl b/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl similarity index 99% rename from apps/emqx_sn/test/props/emqx_sn_proper_types.erl rename to apps/emqx_gateway/test/props/emqx_sn_proper_types.erl index 8d4dae357..96318788d 100644 --- a/apps/emqx_sn/test/props/emqx_sn_proper_types.erl +++ b/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl @@ -16,7 +16,7 @@ -module(emqx_sn_proper_types). --include_lib("emqx_sn/include/emqx_sn.hrl"). +-include_lib("emqx_gateway/src/mqttsn/include/emqx_sn.hrl"). -include_lib("proper/include/proper.hrl"). -compile({no_auto_import, [register/1]}). diff --git a/apps/emqx_sn/test/props/prop_emqx_sn_frame.erl b/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl similarity index 91% rename from apps/emqx_sn/test/props/prop_emqx_sn_frame.erl rename to apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl index 0135ebac7..9e12a7bd4 100644 --- a/apps/emqx_sn/test/props/prop_emqx_sn_frame.erl +++ b/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl @@ -16,22 +16,24 @@ -module(prop_emqx_sn_frame). --include_lib("emqx_sn/include/emqx_sn.hrl"). +-include_lib("src/mqttsn/include/emqx_sn.hrl"). -include_lib("proper/include/proper.hrl"). -compile({no_auto_import, [register/1]}). --import(emqx_sn_frame, - [ parse/1 - , serialize/1 - ]). - -define(ALL(Vars, Types, Exprs), ?SETUP(fun() -> State = do_setup(), fun() -> do_teardown(State) end end, ?FORALL(Vars, Types, Exprs))). +parse(D) -> + {ok, P, _Rest, _State} = emqx_sn_frame:parse(D, #{}), + P. + +serialize(P) -> + emqx_sn_frame:serialize_pkt(P, #{}). + %%-------------------------------------------------------------------- %% Properties %%-------------------------------------------------------------------- @@ -39,7 +41,7 @@ prop_parse_and_serialize() -> ?ALL(Msg, mqtt_sn_message(), begin - {ok, Msg} = parse(serialize(Msg)), + Msg = parse(serialize(Msg)), true end). diff --git a/apps/emqx_lwm2m/.formatter.exs b/apps/emqx_lwm2m/.formatter.exs deleted file mode 100644 index d2cda26ed..000000000 --- a/apps/emqx_lwm2m/.formatter.exs +++ /dev/null @@ -1,4 +0,0 @@ -# Used by "mix format" -[ - inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] -] diff --git a/apps/emqx_lwm2m/integration_test/Makefile b/apps/emqx_lwm2m/integration_test/Makefile deleted file mode 100644 index dcad28900..000000000 --- a/apps/emqx_lwm2m/integration_test/Makefile +++ /dev/null @@ -1,128 +0,0 @@ - -.PHONY: clean, clean_result, start_broker stop_broker case1 case2 case3 - -EMQX_DIR = emqx-enterprise-rel -EMQ = $(EMQX_DIR)/relx.config -WAKAAMA = build_wakaama/lightclient -PAHO_PYTHON = paho/mqtt/client.py - -all: clean_result $(EMQ) $(WAKAAMA) $(PAHO_PYTHON) start_broker clean_result case1 case2 case3 stop_broker - @echo " " - @echo " test complete" - @echo " " - -clean_result: - -rm -f case*.txt - -start_broker: - -rm -f $(EMQX_DIR)/_rel/emqx/log/* - -$(EMQX_DIR)/_rel/emqx/bin/emqx stop - sleep 3 - $(EMQX_DIR)/_rel/emqx/bin/emqx start - sleep 1 - $(EMQX_DIR)/_rel/emqx/bin/emqx_ctl plugins load emqx_lwm2m - - -stop_broker: - -$(EMQX_DIR)/_rel/emqx/bin/emqx stop - - -case1: - -build_wakaama/lightclient -4 -n jXtestlwm2m & - python case1.py - -ps aux|grep lightclient|awk '{print $$2}'|xargs kill -2 - -case2: - -build_wakaama/lightclient -4 -n jXtestlwm2m & - python case2.py - -ps aux|grep lightclient|awk '{print $$2}'|xargs kill -2 - -case3: - -build_wakaama/lightclient -4 -n jXtestlwm2m & - python case3.py - -ps aux|grep lightclient|awk '{print $$2}'|xargs kill -2 - - -$(EMQ): - git clone https://github.com/emqx/emqx-enterprise-rel - git clone https://github.com/emqx/emqx-lwm2m.git - @echo "update emqx-lwm2m with this development code" - mv emqx-lwm2m emqx_lwm2m - -rm -rf emqx_lwm2m/etc - -rm -rf emqx_lwm2m/include - -rm -rf emqx_lwm2m/priv - -rm -rf emqx_lwm2m/src - -rm -rf emqx_lwm2m/Makefile - -rm -rf emqx_lwm2m/erlang.mk - cp -rf ../etc emqx_lwm2m/ - cp -rf ../include emqx_lwm2m/ - cp -rf ../priv emqx_lwm2m/ - cp -rf ../src emqx_lwm2m/ - cp -rf ../Makefile emqx_lwm2m/Makefile - cp -rf ../erlang.mk emqx_lwm2m/erlang.mk - -mkdir $(EMQX_DIR)/deps - mv emqx_lwm2m $(EMQX_DIR)/deps/ - @echo "start building ..." - python insert_lwm2m_plugin.py - make -C emqx-rel -f Makefile - -cp -rf ../lwm2m_xml $(EMQX_DIR)/_rel/emqx/etc/ - - -w: - cd build_wakaama && cmake -DCMAKE_BUILD_TYPE=Debug ../wakaama/examples/lightclient && make - -$(WAKAAMA): - git clone https://github.com/eclipse/wakaama - -mkdir build_wakaama - # replace lightclient's source code, change port from 5683 to 5683, since 5683 is the default port of emqx-lwm2m - cp -f object_security.c wakaama/examples/lightclient/object_security.c - cd build_wakaama && cmake -DCMAKE_BUILD_TYPE=Debug ../wakaama/examples/lightclient && make - - -mqtt: $(PAHO_PYTHON) - # short for paho python client - -$(PAHO_PYTHON): - git clone https://github.com/eclipse/paho.mqtt.python.git - mv paho.mqtt.python/src/paho ./ - rm -rf paho.mqtt.python - - -r: rebuild_emq - # r short for rebuild_emq - @echo " rebuild complete " - - -rebuild_emq: - -$(EMQX_DIR)/_rel/emqx/bin/emqx stop - -mkdir $(EMQX_DIR)/deps - -rm -rf $(EMQX_DIR)/deps/emqx_lwm2m/etc - -rm -rf $(EMQX_DIR)/deps/emqx_lwm2m/include - -rm -rf $(EMQX_DIR)/deps/emqx_lwm2m/priv - -rm -rf $(EMQX_DIR)/deps/emqx_lwm2m/src - -rm -rf $(EMQX_DIR)/deps/emqx_lwm2m/Makefile - -rm -rf $(EMQX_DIR)/deps/emqx_lwm2m/erlang.mk - cp -rf ../etc $(EMQX_DIR)/deps/emqx_lwm2m/ - cp -rf ../include $(EMQX_DIR)/deps/emqx_lwm2m/ - cp -rf ../priv $(EMQX_DIR)/deps/emqx_lwm2m/ - cp -rf ../src $(EMQX_DIR)/deps/emqx_lwm2m/ - cp -rf ../Makefile $(EMQX_DIR)/deps/emqx_lwm2m/Makefile - cp -rf ../erlang.mk $(EMQX_DIR)/deps/emqx_lwm2m/erlang.mk - make -C $(EMQX_DIR) -f Makefile - - -clean: clean_result - -rm -f client/*.exe - -rm -f client/*.o - -rm -rf emqx-rel - -rm -rf build_wakaama - -rm -rf wakaama - - - -lazy: clean_result start_broker case1 case2 case3 stop_broker - # custom your command here - @echo "you are so lazy" - - - diff --git a/apps/emqx_lwm2m/integration_test/case1.py b/apps/emqx_lwm2m/integration_test/case1.py deleted file mode 100644 index b3f3a3df6..000000000 --- a/apps/emqx_lwm2m/integration_test/case1.py +++ /dev/null @@ -1,65 +0,0 @@ - - - -import sys, time -import paho.mqtt.client as mqtt - - -quit_now = False -DeviceId = "jXtestlwm2m" -test_step = 0 - -def on_connect(mqttc, userdata, flags, rc): - global DeviceId - json = "{\"CmdID\":5,\"Command\":\"Discover\",\"BaseName\":\"/3/0\"}" - mqttc.publish("lwm2m/"+DeviceId+"/command", json) - -def on_message(mqttc, userdata, msg): - global quit_now, conclusion, test_step - if msg: - print(["incoming message topic ", msg.topic, " payload ", msg.payload]) - - if msg.topic == 'lwm2m/jXtestlwm2m/response': - if test_step == 0: - if msg.payload == '{"CmdID":5,"Command":"Discover","Result":",,,,"}': - json = "{\"CmdID\":6,\"Command\":\"Read\",\"BaseName\":\"/3/0\"}" - mqttc.publish("lwm2m/"+DeviceId+"/command", json) - test_step = 1 - elif test_step == 1: - if msg.payload == '{"CmdID":6,"Command":"Read","Result":{"bn":"/3/0","e":[{"n":"0","sv":"Open Mobile Alliance"},{"n":"1","sv":"Lightweight M2M Client"},{"n":"16","sv":"U"}]}}': - test_step = 2 - quit_now = True - - -def on_publish(mqttc, userdata, mid): - pass - - - -def main(): - global DeviceId, test_step - timeout = 7 - mqttc = mqtt.Client("test_coap_lwm2m_c02334") - mqttc.on_message = on_message - mqttc.on_publish = on_publish - mqttc.on_connect = on_connect - - mqttc.connect("127.0.0.1", 1883, 120) - mqttc.subscribe("lwm2m/"+DeviceId+"/response", qos=1) - mqttc.loop_start() - while quit_now == False and timeout > 0: - time.sleep(1) - timeout = timeout - 1 - mqttc.disconnect() - mqttc.loop_stop() - if test_step == 2: - print("\n\n CASE1 PASS\n\n") - else: - print("\n\n CASE1 FAIL\n\n") - - -if __name__ == "__main__": - main() - - - diff --git a/apps/emqx_lwm2m/integration_test/case2.py b/apps/emqx_lwm2m/integration_test/case2.py deleted file mode 100644 index d4a5013a0..000000000 --- a/apps/emqx_lwm2m/integration_test/case2.py +++ /dev/null @@ -1,60 +0,0 @@ - - - -import sys, time -import paho.mqtt.client as mqtt - - -quit_now = False -DeviceId = "jXtestlwm2m" -test_step = 0 - -def on_connect(mqttc, userdata, flags, rc): - global DeviceId - json = "{\"CmdID\":5,\"Command\":\"Execute\",\"BaseName\":\"/3/0/4\"}" - mqttc.publish("lwm2m/"+DeviceId+"/command", json) - -def on_message(mqttc, userdata, msg): - global quit_now, conclusion, test_step - if msg: - print(["incoming message topic ", msg.topic, " payload ", msg.payload]) - - if msg.topic == 'lwm2m/jXtestlwm2m/response': - if test_step == 0: - if msg.payload == '{"CmdID":5,"Command":"Execute","Result":"Changed"}': - test_step = 1 - quit_now = True - - -def on_publish(mqttc, userdata, mid): - pass - - - -def main(): - global DeviceId, test_step - timeout = 7 - mqttc = mqtt.Client("test_coap_lwm2m_c02334") - mqttc.on_message = on_message - mqttc.on_publish = on_publish - mqttc.on_connect = on_connect - - mqttc.connect("127.0.0.1", 1883, 120) - mqttc.subscribe("lwm2m/"+DeviceId+"/response", qos=1) - mqttc.loop_start() - while quit_now == False and timeout > 0: - time.sleep(1) - timeout = timeout - 1 - mqttc.disconnect() - mqttc.loop_stop() - if test_step == 1: - print("\n\n CASE2 PASS\n\n") - else: - print("\n\n CASE2 FAIL\n\n") - - -if __name__ == "__main__": - main() - - - diff --git a/apps/emqx_lwm2m/integration_test/case3.py b/apps/emqx_lwm2m/integration_test/case3.py deleted file mode 100644 index 2ce2c99f8..000000000 --- a/apps/emqx_lwm2m/integration_test/case3.py +++ /dev/null @@ -1,60 +0,0 @@ - - - -import sys, time -import paho.mqtt.client as mqtt - - -quit_now = False -DeviceId = "jXtestlwm2m" -test_step = 0 - -def on_connect(mqttc, userdata, flags, rc): - global DeviceId - json = '{"CmdID":5,"Command":"Write","Value":{"bn":"/1/0/1","e":[{"v":123}]}}' - mqttc.publish("lwm2m/"+DeviceId+"/command", json) - -def on_message(mqttc, userdata, msg): - global quit_now, conclusion, test_step - if msg: - print(["incoming message topic ", msg.topic, " payload ", msg.payload]) - - if msg.topic == 'lwm2m/jXtestlwm2m/response': - if test_step == 0: - if msg.payload == '{"CmdID":5,"Command":"Write","Result":"Changed"}': - test_step = 1 - quit_now = True - - -def on_publish(mqttc, userdata, mid): - pass - - - -def main(): - global DeviceId, test_step - timeout = 7 - mqttc = mqtt.Client("test_coap_lwm2m_c02334") - mqttc.on_message = on_message - mqttc.on_publish = on_publish - mqttc.on_connect = on_connect - - mqttc.connect("127.0.0.1", 1883, 120) - mqttc.subscribe("lwm2m/"+DeviceId+"/response", qos=1) - mqttc.loop_start() - while quit_now == False and timeout > 0: - time.sleep(1) - timeout = timeout - 1 - mqttc.disconnect() - mqttc.loop_stop() - if test_step == 1: - print("\n\n CASE3 PASS\n\n") - else: - print("\n\n CASE3 FAIL\n\n") - - -if __name__ == "__main__": - main() - - - diff --git a/apps/emqx_lwm2m/integration_test/insert_lwm2m_plugin.py b/apps/emqx_lwm2m/integration_test/insert_lwm2m_plugin.py deleted file mode 100644 index 7769a3af7..000000000 --- a/apps/emqx_lwm2m/integration_test/insert_lwm2m_plugin.py +++ /dev/null @@ -1,52 +0,0 @@ - - -def change_makefile(): - f = open("emqx-rel/Makefile", "rb") - data = f.read() - f.close() - - if data.find("emqx_lwm2m") < 0: - data = data.replace("emqx_auth_pgsql emqx_auth_mongo", "emqx_auth_pgsql emqx_auth_mongo emqx_lwm2m\n\ndep_emqx_lwm2m = git https://github.com/emqx/emqx-lwm2m\n\n") - f = open("emqx-rel/Makefile", "wb") - f.write(data) - f.close() - - - f = open("emqx-rel/relx.config", "rb") - data = f.read() - f.close() - - if data.find("emq_lwm2m") < 0: - f = open("emqx-rel/relx.config", "wb") - data = data.replace("{emqx_auth_mongo, load}", "{emqx_auth_mongo, load},\n{emqx_lwm2m, load}") - data = data.replace('{template, "rel/conf/emqx.conf", "etc/emqx.conf"},', \ - '{template, "rel/conf/emqx.conf", "etc/emqx.conf"},'+ \ - '\n {template, "rel/conf/plugins/emqx_lwm2m.conf", "etc/plugins/emqx_lwm2m.conf"},'+ \ - '\n {copy, "deps/emqx_lwm2m/lwm2m_xml", "etc/"},') - f.write(data) - f.close() - - - -def change_lwm2m_config(): - f = open("emqx-rel/deps/emqx_lwm2m/etc/emqx_lwm2m.conf", "rb") - data = f.read() - f.close() - - if data.find("5683") > 0: - data = data.replace("5683", "5683") - f = open("emqx-rel/deps/emqx_lwm2m/etc/emqx_lwm2m.conf", "wb") - f.write(data) - f.close() - - - -def main(): - change_makefile() - change_lwm2m_config() - - -if __name__ == "__main__": - main() - - \ No newline at end of file diff --git a/apps/emqx_lwm2m/integration_test/object_security.c b/apps/emqx_lwm2m/integration_test/object_security.c deleted file mode 100644 index ee12ea7b7..000000000 --- a/apps/emqx_lwm2m/integration_test/object_security.c +++ /dev/null @@ -1,253 +0,0 @@ -/******************************************************************************* - * - * Copyright (c) 2013, 2014, 2015 Intel Corporation and others. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * and Eclipse Distribution License v1.0 which accompany this distribution. - * - * The Eclipse Public License is available at - * http://www.eclipse.org/legal/epl-v10.html - * The Eclipse Distribution License is available at - * http://www.eclipse.org/org/documents/edl-v10.php. - * - * Contributors: - * David Navarro, Intel Corporation - initial API and implementation - * Bosch Software Innovations GmbH - Please refer to git log - * Pascal Rieux - Please refer to git log - * - *******************************************************************************/ - -/* - * Resources: - * - * Name | ID | Operations | Instances | Mandatory | Type | Range | Units | - * Server URI | 0 | | Single | Yes | String | | | - * Bootstrap Server | 1 | | Single | Yes | Boolean | | | - * Security Mode | 2 | | Single | Yes | Integer | 0-3 | | - * Public Key or ID | 3 | | Single | Yes | Opaque | | | - * Server Public Key or ID | 4 | | Single | Yes | Opaque | | | - * Secret Key | 5 | | Single | Yes | Opaque | | | - * SMS Security Mode | 6 | | Single | Yes | Integer | 0-255 | | - * SMS Binding Key Param. | 7 | | Single | Yes | Opaque | 6 B | | - * SMS Binding Secret Keys | 8 | | Single | Yes | Opaque | 32-48 B | | - * Server SMS Number | 9 | | Single | Yes | Integer | | | - * Short Server ID | 10 | | Single | No | Integer | 1-65535 | | - * Client Hold Off Time | 11 | | Single | Yes | Integer | | s | - * - */ - -/* - * Here we implement a very basic LWM2M Security Object which only knows NoSec security mode. - */ - -#include "liblwm2m.h" - -#include -#include -#include - - -typedef struct _security_instance_ -{ - struct _security_instance_ * next; // matches lwm2m_list_t::next - uint16_t instanceId; // matches lwm2m_list_t::id - char * uri; - bool isBootstrap; - uint16_t shortID; - uint32_t clientHoldOffTime; -} security_instance_t; - -static uint8_t prv_get_value(lwm2m_data_t * dataP, - security_instance_t * targetP) -{ - - switch (dataP->id) - { - case LWM2M_SECURITY_URI_ID: - lwm2m_data_encode_string(targetP->uri, dataP); - return COAP_205_CONTENT; - - case LWM2M_SECURITY_BOOTSTRAP_ID: - lwm2m_data_encode_bool(targetP->isBootstrap, dataP); - return COAP_205_CONTENT; - - case LWM2M_SECURITY_SECURITY_ID: - lwm2m_data_encode_int(LWM2M_SECURITY_MODE_NONE, dataP); - return COAP_205_CONTENT; - - case LWM2M_SECURITY_PUBLIC_KEY_ID: - // Here we return an opaque of 1 byte containing 0 - { - uint8_t value = 0; - - lwm2m_data_encode_opaque(&value, 1, dataP); - } - return COAP_205_CONTENT; - - case LWM2M_SECURITY_SERVER_PUBLIC_KEY_ID: - // Here we return an opaque of 1 byte containing 0 - { - uint8_t value = 0; - - lwm2m_data_encode_opaque(&value, 1, dataP); - } - return COAP_205_CONTENT; - - case LWM2M_SECURITY_SECRET_KEY_ID: - // Here we return an opaque of 1 byte containing 0 - { - uint8_t value = 0; - - lwm2m_data_encode_opaque(&value, 1, dataP); - } - return COAP_205_CONTENT; - - case LWM2M_SECURITY_SMS_SECURITY_ID: - lwm2m_data_encode_int(LWM2M_SECURITY_MODE_NONE, dataP); - return COAP_205_CONTENT; - - case LWM2M_SECURITY_SMS_KEY_PARAM_ID: - // Here we return an opaque of 6 bytes containing a buggy value - { - char * value = "12345"; - lwm2m_data_encode_opaque((uint8_t *)value, 6, dataP); - } - return COAP_205_CONTENT; - - case LWM2M_SECURITY_SMS_SECRET_KEY_ID: - // Here we return an opaque of 32 bytes containing a buggy value - { - char * value = "1234567890abcdefghijklmnopqrstu"; - lwm2m_data_encode_opaque((uint8_t *)value, 32, dataP); - } - return COAP_205_CONTENT; - - case LWM2M_SECURITY_SMS_SERVER_NUMBER_ID: - lwm2m_data_encode_int(0, dataP); - return COAP_205_CONTENT; - - case LWM2M_SECURITY_SHORT_SERVER_ID: - lwm2m_data_encode_int(targetP->shortID, dataP); - return COAP_205_CONTENT; - - case LWM2M_SECURITY_HOLD_OFF_ID: - lwm2m_data_encode_int(targetP->clientHoldOffTime, dataP); - return COAP_205_CONTENT; - - default: - return COAP_404_NOT_FOUND; - } -} - -static uint8_t prv_security_read(uint16_t instanceId, - int * numDataP, - lwm2m_data_t ** dataArrayP, - lwm2m_object_t * objectP) -{ - security_instance_t * targetP; - uint8_t result; - int i; - - targetP = (security_instance_t *)lwm2m_list_find(objectP->instanceList, instanceId); - if (NULL == targetP) return COAP_404_NOT_FOUND; - - // is the server asking for the full instance ? - if (*numDataP == 0) - { - uint16_t resList[] = {LWM2M_SECURITY_URI_ID, - LWM2M_SECURITY_BOOTSTRAP_ID, - LWM2M_SECURITY_SECURITY_ID, - LWM2M_SECURITY_PUBLIC_KEY_ID, - LWM2M_SECURITY_SERVER_PUBLIC_KEY_ID, - LWM2M_SECURITY_SECRET_KEY_ID, - LWM2M_SECURITY_SMS_SECURITY_ID, - LWM2M_SECURITY_SMS_KEY_PARAM_ID, - LWM2M_SECURITY_SMS_SECRET_KEY_ID, - LWM2M_SECURITY_SMS_SERVER_NUMBER_ID, - LWM2M_SECURITY_SHORT_SERVER_ID, - LWM2M_SECURITY_HOLD_OFF_ID}; - int nbRes = sizeof(resList)/sizeof(uint16_t); - - *dataArrayP = lwm2m_data_new(nbRes); - if (*dataArrayP == NULL) return COAP_500_INTERNAL_SERVER_ERROR; - *numDataP = nbRes; - for (i = 0 ; i < nbRes ; i++) - { - (*dataArrayP)[i].id = resList[i]; - } - } - - i = 0; - do - { - result = prv_get_value((*dataArrayP) + i, targetP); - i++; - } while (i < *numDataP && result == COAP_205_CONTENT); - - return result; -} - -lwm2m_object_t * get_security_object() -{ - lwm2m_object_t * securityObj; - - securityObj = (lwm2m_object_t *)lwm2m_malloc(sizeof(lwm2m_object_t)); - - if (NULL != securityObj) - { - security_instance_t * targetP; - - memset(securityObj, 0, sizeof(lwm2m_object_t)); - - securityObj->objID = 0; - - // Manually create an hardcoded instance - targetP = (security_instance_t *)lwm2m_malloc(sizeof(security_instance_t)); - if (NULL == targetP) - { - lwm2m_free(securityObj); - return NULL; - } - - memset(targetP, 0, sizeof(security_instance_t)); - targetP->instanceId = 0; - targetP->uri = strdup("coap://localhost:5683"); - targetP->isBootstrap = false; - targetP->shortID = 123; - targetP->clientHoldOffTime = 10; - - securityObj->instanceList = LWM2M_LIST_ADD(securityObj->instanceList, targetP); - - securityObj->readFunc = prv_security_read; - } - - return securityObj; -} - -void free_security_object(lwm2m_object_t * objectP) -{ - while (objectP->instanceList != NULL) - { - security_instance_t * securityInstance = (security_instance_t *)objectP->instanceList; - objectP->instanceList = objectP->instanceList->next; - if (NULL != securityInstance->uri) - { - lwm2m_free(securityInstance->uri); - } - lwm2m_free(securityInstance); - } - lwm2m_free(objectP); -} - -char * get_server_uri(lwm2m_object_t * objectP, - uint16_t secObjInstID) -{ - security_instance_t * targetP = (security_instance_t *)LWM2M_LIST_FIND(objectP->instanceList, secObjInstID); - - if (NULL != targetP) - { - return lwm2m_strdup(targetP->uri); - } - - return NULL; -} diff --git a/apps/emqx_lwm2m/mix.exs b/apps/emqx_lwm2m/mix.exs deleted file mode 100644 index 2b45596d2..000000000 --- a/apps/emqx_lwm2m/mix.exs +++ /dev/null @@ -1,32 +0,0 @@ -defmodule EMQXLwm2m.MixProject do - use Mix.Project - - def project do - [ - app: :emqx_lwm2m, - version: "4.3.1", - build_path: "../../_build", - config_path: "../../config/config.exs", - deps_path: "../../deps", - lockfile: "../../mix.lock", - elixir: "~> 1.12", - start_permanent: Mix.env() == :prod, - deps: deps(), - description: "EMQ X LwM2M Gateway" - ] - end - - def application do - [ - registered: [:emqx_lwm2m_sup], - extra_applications: [:logger] - ] - end - - defp deps do - [ - {:emqx, in_umbrella: true, runtime: false}, - {:lwm2m_coap, github: "emqx/lwm2m-coap", tag: "v1.1.2"} - ] - end -end diff --git a/apps/emqx_lwm2m/rebar.config b/apps/emqx_lwm2m/rebar.config deleted file mode 100644 index 7700071ed..000000000 --- a/apps/emqx_lwm2m/rebar.config +++ /dev/null @@ -1,29 +0,0 @@ -{deps, - %% lwm2m-coap v.1.* is for emqx v4.3.* - %% lwm2m-coap v.2.* is for emqx v5.* - [{lwm2m_coap, {git, "https://github.com/emqx/lwm2m-coap", {tag, "v2.0.0"}}} - ]}. - -{profiles, - [{test, - [{deps, [{er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}}, - {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.0"}}} - ]} - ]} - ]}. - -{edoc_opts, [{preprocess, true}]}. -{erl_opts, [warn_unused_vars, - warn_shadow_vars, - warn_unused_import, - warn_obsolete_guard, - debug_info, - {parse_transform}]}. - -{xref_checks, [undefined_function_calls, undefined_functions, - locals_not_used, deprecated_function_calls, - warnings_as_errors, deprecated_functions]}. -{cover_enabled, true}. -{cover_opts, [verbose]}. -{cover_export_enabled, true}. -{extra_src_dirs, [{"lwm2m_xml", [{recursive,true}]}]}. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src deleted file mode 100644 index b6d49302f..000000000 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src +++ /dev/null @@ -1,13 +0,0 @@ -%% -*-: erlang -*- -{VSN, - [ - {<<"4.3.[0-1]">>, [ - {restart_application, emqx_lwm2m} - ]} - ], - [ - {<<"4.3.[0-1]">>, [ - {restart_application, emqx_lwm2m} - ]} - ] -}. diff --git a/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl deleted file mode 100644 index 7f2f37466..000000000 --- a/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl +++ /dev/null @@ -1,1953 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_lwm2m_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --define(PORT, 5683). - --define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). - --include("emqx_lwm2m.hrl"). --include_lib("lwm2m_coap/include/coap.hrl"). --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - -%%-------------------------------------------------------------------- -%% Setups -%%-------------------------------------------------------------------- - -all() -> - [ {group, test_grp_0_register} - , {group, test_grp_1_read} - , {group, test_grp_2_write} - , {group, test_grp_3_execute} - , {group, test_grp_4_discover} - , {group, test_grp_5_write_attr} - , {group, test_grp_6_observe} - , {group, test_grp_8_object_19} - ]. - -suite() -> [{timetrap, {seconds, 90}}]. - -groups() -> - RepeatOpt = {repeat_until_all_ok, 1}, - [ - {test_grp_0_register, [RepeatOpt], [ - case01_register, - case01_register_additional_opts, - case01_register_incorrect_opts, - case01_register_report, - case02_update_deregister, - case03_register_wrong_version, - case04_register_and_lifetime_timeout, - case05_register_wrong_epn, - case06_register_wrong_lifetime, - case07_register_alternate_path_01, - case07_register_alternate_path_02, - case08_reregister - ]}, - {test_grp_1_read, [RepeatOpt], [ - case10_read, - case10_read_separate_ack, - case11_read_object_tlv, - case11_read_object_json, - case12_read_resource_opaque, - case13_read_no_xml - ]}, - {test_grp_2_write, [RepeatOpt], [ - case20_write, - case21_write_object, - case22_write_error, - case20_single_write - ]}, - {test_grp_create, [RepeatOpt], [ - case_create_basic - ]}, - {test_grp_delete, [RepeatOpt], [ - case_delete_basic - ]}, - {test_grp_3_execute, [RepeatOpt], [ - case30_execute, case31_execute_error - ]}, - {test_grp_4_discover, [RepeatOpt], [ - case40_discover - ]}, - {test_grp_5_write_attr, [RepeatOpt], [ - case50_write_attribute - ]}, - {test_grp_6_observe, [RepeatOpt], [ - case60_observe - ]}, - {test_grp_7_block_wize_transfer, [RepeatOpt], [ - case70_read_large, case70_write_large - ]}, - {test_grp_8_object_19, [RepeatOpt], [ - case80_specail_object_19_1_0_write, - case80_specail_object_19_0_0_notify - %case80_specail_object_19_0_0_response, - %case80_normal_object_19_0_0_read - ]}, - {test_grp_9_psm_queue_mode, [RepeatOpt], [ - case90_psm_mode, - case90_queue_mode - ]} - ]. - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx]), - Config. - -end_per_suite(Config) -> - timer:sleep(300), - emqx_ct_helpers:stop_apps([emqx]), - Config. - -init_per_testcase(_AllTestCase, Config) -> - application:set_env(emqx_lwm2m, bind_udp, [{5683, []}]), - application:set_env(emqx_lwm2m, bind_dtls, [{5684, []}]), - application:set_env(emqx_lwm2m, xml_dir, emqx_ct_helpers:deps_path(emqx_lwm2m, "lwm2m_xml")), - application:set_env(emqx_lwm2m, lifetime_max, 86400), - application:set_env(emqx_lwm2m, lifetime_min, 1), - application:set_env(emqx_lwm2m, mountpoint, "lwm2m/%e/"), - {ok, _Started} = application:ensure_all_started(emqx_lwm2m), - {ok, ClientUdpSock} = gen_udp:open(0, [binary, {active, false}]), - - {ok, C} = emqtt:start_link([{host, "localhost"},{port, 1883},{clientid, <<"c1">>}]), - {ok, _} = emqtt:connect(C), - timer:sleep(100), - - [{sock, ClientUdpSock}, {emqx_c, C} | Config]. - -end_per_testcase(_AllTestCase, Config) -> - timer:sleep(300), - gen_udp:close(?config(sock, Config)), - emqtt:disconnect(?config(emqx_c, Config)), - ok = application:stop(emqx_lwm2m), - ok = application:stop(lwm2m_coap). - -%%-------------------------------------------------------------------- -%% Cases -%%-------------------------------------------------------------------- - -case01_register(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- - UdpSock = ?config(sock, Config), - Epn = "urn:oma:lwm2m:oma:3", - MsgId = 12, - SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), - - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, - [], - MsgId), - - %% checkpoint 1 - response - #coap_message{type = Type, method = Method, id = RspId, options = Opts} = - test_recv_coap_response(UdpSock), - ack = Type, - {ok, created} = Method, - RspId = MsgId, - Location = proplists:get_value(location_path, Opts), - ?assertNotEqual(undefined, Location), - - %% checkpoint 2 - verify subscribed topics - timer:sleep(50), - ?LOGT("all topics: ~p", [test_mqtt_broker:get_subscrbied_topics()]), - true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), - - % ---------------------------------------- - % DE-REGISTER command - % ---------------------------------------- - ?LOGT("start to send DE-REGISTER command", []), - MsgId3 = 52, - test_send_coap_request( UdpSock, - delete, - sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]), - #coap_content{payload = <<>>}, - [], - MsgId3), - #coap_message{type = ack, id = RspId3, method = Method3} = test_recv_coap_response(UdpSock), - {ok,deleted} = Method3, - MsgId3 = RspId3, - timer:sleep(50), - false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). - -case01_register_additional_opts(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- - UdpSock = ?config(sock, Config), - Epn = "urn:oma:lwm2m:oma:3", - MsgId = 12, - SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), - - AddOpts = "ep=~s<=345&lwm2m=1&apn=psmA.eDRX0.ctnb&cust_opt=shawn&im=123&ct=1.4&mt=mdm9620&mv=1.2", - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?" ++ AddOpts, [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, - [], - MsgId), - - %% checkpoint 1 - response - #coap_message{type = Type, method = Method, id = RspId, options = Opts} = - test_recv_coap_response(UdpSock), - Type = ack, - Method = {ok, created}, - RspId = MsgId, - Location = proplists:get_value(location_path, Opts), - ?assertNotEqual(undefined, Location), - - %% checkpoint 2 - verify subscribed topics - timer:sleep(50), - - true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), - - % ---------------------------------------- - % DE-REGISTER command - % ---------------------------------------- - ?LOGT("start to send DE-REGISTER command", []), - MsgId3 = 52, - test_send_coap_request( UdpSock, - delete, - sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]), - #coap_content{payload = <<>>}, - [], - MsgId3), - #coap_message{type = ack, id = RspId3, method = Method3} = test_recv_coap_response(UdpSock), - {ok,deleted} = Method3, - MsgId3 = RspId3, - timer:sleep(50), - false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). - -case01_register_incorrect_opts(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- - UdpSock = ?config(sock, Config), - Epn = "urn:oma:lwm2m:oma:3", - MsgId = 12, - - - AddOpts = "ep=~s<=345&lwm2m=1&incorrect_opt", - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?" ++ AddOpts, [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, - [], - MsgId), - - %% checkpoint 1 - response - #coap_message{type = ack, method = Method, id = MsgId} = - test_recv_coap_response(UdpSock), - ?assertEqual({error,bad_request}, Method). - -case01_register_report(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- - UdpSock = ?config(sock, Config), - Epn = "urn:oma:lwm2m:oma:3", - MsgId = 12, - SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), - ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0), - timer:sleep(200), - - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, - [], - MsgId), - - #coap_message{type = Type, method = Method, id = RspId, options = Opts} = - test_recv_coap_response(UdpSock), - Type = ack, - Method = {ok, created}, - RspId = MsgId, - Location = proplists:get_value(location_path, Opts), - ?assertNotEqual(undefined, Location), - - timer:sleep(50), - true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), - - ReadResult = emqx_json:encode(#{ - <<"msgType">> => <<"register">>, - <<"data">> => #{ - <<"alternatePath">> => <<"/">>, - <<"ep">> => list_to_binary(Epn), - <<"lt">> => 345, - <<"lwm2m">> => <<"1">>, - <<"objectList">> => [<<"/1">>, <<"/2">>, <<"/3">>, <<"/4">>, <<"/5">>] - } - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(ReportTopic)), - - % ---------------------------------------- - % DE-REGISTER command - % ---------------------------------------- - ?LOGT("start to send DE-REGISTER command", []), - MsgId3 = 52, - test_send_coap_request( UdpSock, - delete, - sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]), - #coap_content{payload = <<>>}, - [], - MsgId3), - #coap_message{type = ack, id = RspId3, method = Method3} = test_recv_coap_response(UdpSock), - {ok,deleted} = Method3, - MsgId3 = RspId3, - timer:sleep(50), - false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). - -case02_update_deregister(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- - UdpSock = ?config(sock, Config), - Epn = "urn:oma:lwm2m:oma:3", - MsgId = 12, - SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), - ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0), - timer:sleep(200), - - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, - [], - MsgId), - timer:sleep(100), - #coap_message{type = ack, method = Method, options = Opts} = test_recv_coap_response(UdpSock), - ?assertEqual({ok,created}, Method), - - ?LOGT("Options got: ~p", [Opts]), - Location = proplists:get_value(location_path, Opts), - Register = emqx_json:encode(#{ - <<"msgType">> => <<"register">>, - <<"data">> => #{ - <<"alternatePath">> => <<"/">>, - <<"ep">> => list_to_binary(Epn), - <<"lt">> => 345, - <<"lwm2m">> => <<"1">>, - <<"objectList">> => [<<"/1">>, <<"/2">>, <<"/3">>, <<"/4">>, <<"/5">>] - } - }), - ?assertEqual(Register, test_recv_mqtt_response(ReportTopic)), - - % ---------------------------------------- - % UPDATE command - % ---------------------------------------- - ?LOGT("start to send UPDATE command", []), - MsgId2 = 27, - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b~s?lt=789", [?PORT, join_path(Location, <<>>)]), - #coap_content{content_format = <<"text/plain">>, payload = <<", , , , , ">>}, - [], - MsgId2), - #coap_message{type = ack, id = RspId2, method = Method2} = test_recv_coap_response(UdpSock), - {ok,changed} = Method2, - MsgId2 = RspId2, - Update = emqx_json:encode(#{ - <<"msgType">> => <<"update">>, - <<"data">> => #{ - <<"alternatePath">> => <<"/">>, - <<"ep">> => list_to_binary(Epn), - <<"lt">> => 789, - <<"lwm2m">> => <<"1">>, - <<"objectList">> => [<<"/1">>, <<"/2">>, <<"/3">>, <<"/4">>, <<"/5">>, <<"/6">>] - } - }), - ?assertEqual(Update, test_recv_mqtt_response(ReportTopic)), - - % ---------------------------------------- - % DE-REGISTER command - % ---------------------------------------- - ?LOGT("start to send DE-REGISTER command", []), - MsgId3 = 52, - test_send_coap_request( UdpSock, - delete, - sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]), - #coap_content{payload = <<>>}, - [], - MsgId3), - #coap_message{type = ack, id = RspId3, method = Method3} = test_recv_coap_response(UdpSock), - {ok,deleted} = Method3, - MsgId3 = RspId3, - - timer:sleep(50), - false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). - -case03_register_wrong_version(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- - UdpSock = ?config(sock, Config), - Epn = "urn:oma:lwm2m:oma:3", - MsgId = 12, - SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=8.3", [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, - [], - MsgId), - #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock), - ?assertEqual({error,precondition_failed}, Method), - timer:sleep(50), - - false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). - -case04_register_and_lifetime_timeout(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- - UdpSock = ?config(sock, Config), - Epn = "urn:oma:lwm2m:oma:3", - MsgId = 12, - SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), - - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=2&lwm2m=1", [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, - [], - MsgId), - timer:sleep(100), - #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock), - ?assertEqual({ok,created}, Method), - - true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), - - % ---------------------------------------- - % lifetime timeout - % ---------------------------------------- - timer:sleep(4000), - - false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). - -case05_register_wrong_epn(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- - MsgId = 12, - UdpSock = ?config(sock, Config), - - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?lt=345&lwm2m=1.0", [?PORT]), - #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, - [], - MsgId), - #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock), - ?assertEqual({error,bad_request}, Method). - -case06_register_wrong_lifetime(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- - UdpSock = ?config(sock, Config), - Epn = "urn:oma:lwm2m:oma:3", - MsgId = 12, - - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s&lwm2m=1", [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, - [], - MsgId), - #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock), - ?assertEqual({error,bad_request}, Method), - timer:sleep(50), - ?assertEqual([], test_mqtt_broker:get_subscrbied_topics()). - -case07_register_alternate_path_01(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- - UdpSock = ?config(sock, Config), - Epn = "urn:oma:lwm2m:oma:3", - MsgId = 12, - SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), - ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0), - timer:sleep(200), - - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, - payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, - [], - MsgId), - timer:sleep(50), - true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). - -case07_register_alternate_path_02(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- - UdpSock = ?config(sock, Config), - Epn = "urn:oma:lwm2m:oma:3", - MsgId = 12, - SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), - ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0), - timer:sleep(200), - - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, - payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, - [], - MsgId), - timer:sleep(50), - true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). - -case08_reregister(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- - UdpSock = ?config(sock, Config), - Epn = "urn:oma:lwm2m:oma:3", - MsgId = 12, - SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), - ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0), - timer:sleep(200), - - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, - payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, - [], - MsgId), - timer:sleep(50), - true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), - - ReadResult = emqx_json:encode( - #{ - <<"msgType">> => <<"register">>, - <<"data">> => #{ - <<"alternatePath">> => <<"/lwm2m">>, - <<"ep">> => list_to_binary(Epn), - <<"lt">> => 345, - <<"lwm2m">> => <<"1">>, - <<"objectList">> => [<<"/1/0">>, <<"/2/0">>, <<"/3/0">>] - } - } - ), - ?assertEqual(ReadResult, test_recv_mqtt_response(ReportTopic)), - timer:sleep(1000), - - %% the same lwm2mc client registers to server again - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, - payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, - [], - MsgId + 1), - %% verify the lwm2m client is still online - ?assertEqual(ReadResult, test_recv_mqtt_response(ReportTopic)). - -case10_read(Config) -> - UdpSock = ?config(sock, Config), - Epn = "urn:oma:lwm2m:oma:3", - MsgId1 = 15, - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - % step 1, device register ... - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, - payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, - [], - MsgId1), - #coap_message{method = Method1} = test_recv_coap_response(UdpSock), - ?assertEqual({ok,created}, Method1), - test_recv_mqtt_response(RespTopic), - - % step2, send a READ command to device - CmdId = 206, - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - Command = #{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"path">> => <<"/3/0/0">> - } - }, - CommandJson = emqx_json:encode(Command), - ?LOGT("CommandJson=~p", [CommandJson]), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, - ?LOGT("LwM2M client got ~p", [Request2]), - - ?assertEqual(get, Method2), - ?assertEqual(<<"/lwm2m/3/0/0">>, get_coap_path(Options2)), - ?assertEqual(<<>>, Payload2), - timer:sleep(50), - - test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request2, true), - timer:sleep(100), - - ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"reqPath">> => <<"/3/0/0">>, - <<"content">> => [#{ - path => <<"/3/0/0">>, - value => <<"EMQ">> - }] - } - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). - -case10_read_separate_ack(Config) -> - UdpSock = ?config(sock, Config), - Epn = "urn:oma:lwm2m:oma:3", - MsgId1 = 15, - ObjectList = <<", , , , ">>, - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - - % step 1, device register ... - std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - - % step2, send a READ command to device - CmdId = 206, - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - Command = #{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"path">> => <<"/3/0/0">> - } - }, - CommandJson = emqx_json:encode(Command), - ?LOGT("CommandJson=~p", [CommandJson]), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, - ?LOGT("LwM2M client got ~p", [Request2]), - - ?assertEqual(get, Method2), - ?assertEqual(<<"/3/0/0">>, get_coap_path(Options2)), - ?assertEqual(<<>>, Payload2), - - test_send_empty_ack(UdpSock, "127.0.0.1", ?PORT, Request2), - ReadResultACK = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"ack">>, - <<"data">> => #{ - <<"path">> => <<"/3/0/0">> - } - }), - ?assertEqual(ReadResultACK, test_recv_mqtt_response(RespTopic)), - timer:sleep(100), - - test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request2, false), - timer:sleep(100), - - ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"reqPath">> => <<"/3/0/0">>, - <<"content">> => [#{ - path => <<"/3/0/0">>, - value => <<"EMQ">> - }] - } - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). - -case11_read_object_tlv(Config) -> - % step 1, device register ... - Epn = "urn:oma:lwm2m:oma:3", - MsgId1 = 15, - UdpSock = ?config(sock, Config), - ObjectList = <<", , , , ">>, - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - - std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - - % step2, send a READ command to device - CmdId = 207, - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - Command = #{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"path">> => <<"/3/0">> - } - }, - CommandJson = emqx_json:encode(Command), - ?LOGT("CommandJson=~p", [CommandJson]), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2} = Request2, - ?LOGT("LwM2M client got ~p", [Request2]), - - ?assertEqual(get, Method2), - timer:sleep(50), - - Tlv = <<16#08, 16#00, 16#3C, 16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33>>, - test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"application/vnd.oma.lwm2m+tlv">>, payload = Tlv}, Request2, true), - timer:sleep(100), - - ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"reqPath">> => <<"/3/0">>, - <<"content">> => [ - #{ - path => <<"/3/0/0">>, - value => <<"Open Mobile Alliance">> - }, - #{ - path => <<"/3/0/1">>, - value => <<"Lightweight M2M Client">> - }, - #{ - path => <<"/3/0/2">>, - value => <<"345000123">> - } - ] - } - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). - -case11_read_object_json(Config) -> - % step 1, device register ... - UdpSock = ?config(sock, Config), - Epn = "urn:oma:lwm2m:oma:3", - MsgId1 = 15, - - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - ObjectList = <<", , , , ">>, - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - - std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - - % step2, send a READ command to device - CmdId = 206, - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - Command = #{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"path">> => <<"/3/0">> - } - }, - CommandJson = emqx_json:encode(Command), - ?LOGT("CommandJson=~p", [CommandJson]), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2} = Request2, - ?LOGT("LwM2M client got ~p", [Request2]), - - ?assertEqual(get, Method2), - timer:sleep(50), - - Json = <<"{\"bn\":\"/3/0\",\"e\":[{\"n\":\"0\",\"sv\":\"Open Mobile Alliance\"},{\"n\":\"1\",\"sv\":\"Lightweight M2M Client\"},{\"n\":\"2\",\"sv\":\"345000123\"}]}">>, - test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"application/vnd.oma.lwm2m+json">>, payload = Json}, Request2, true), - timer:sleep(100), - - ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"reqPath">> => <<"/3/0">>, - <<"content">> => [ - #{ - path => <<"/3/0/0">>, - value => <<"Open Mobile Alliance">> - }, - #{ - path => <<"/3/0/1">>, - value => <<"Lightweight M2M Client">> - }, - #{ - path => <<"/3/0/2">>, - value => <<"345000123">> - } - ] - } - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). - -case12_read_resource_opaque(Config) -> - % step 1, device register ... - UdpSock = ?config(sock, Config), - Epn = "urn:oma:lwm2m:oma:3", - MsgId1 = 15, - ObjectList = <<", , , , ">>, - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - - std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - - % step2, send a READ command to device - CmdId = 206, - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - Command = #{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"path">> => <<"/3/0/8">> - } - }, - CommandJson = emqx_json:encode(Command), - ?LOGT("CommandJson=~p", [CommandJson]), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2} = Request2, - ?LOGT("LwM2M client got ~p", [Request2]), - - ?assertEqual(get, Method2), - timer:sleep(50), - - Opaque = <<20, 21, 22, 23>>, - test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"application/octet-stream">>, payload = Opaque}, Request2, true), - timer:sleep(100), - - ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"reqPath">> => <<"/3/0/8">>, - <<"content">> => [ - #{ - path => <<"/3/0/8">>, - value => base64:encode(Opaque) - } - ] - } - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). - -case13_read_no_xml(Config) -> - % step 1, device register ... - Epn = "urn:oma:lwm2m:oma:3", - MsgId1 = 15, - UdpSock = ?config(sock, Config), - ObjectList = <<", , , , ">>, - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - - std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - - % step2, send a READ command to device - CmdId = 206, - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - Command = #{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"path">> => <<"/9723/0/0">> - } - }, - CommandJson = emqx_json:encode(Command), - ?LOGT("CommandJson=~p", [CommandJson]), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2} = Request2, - ?LOGT("LwM2M client got ~p", [Request2]), - - ?assertEqual(get, Method2), - timer:sleep(50), - - test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request2, true), - timer:sleep(100), - - ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"reqPath">> => <<"/9723/0/0">>, - <<"code">> => <<"4.00">>, - <<"codeMsg">> => <<"bad_request">> - } - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). - -case20_single_write(Config) -> - % step 1, device register ... - Epn = "urn:oma:lwm2m:oma:3", - MsgId1 = 15, - UdpSock = ?config(sock, Config), - ObjectList = <<", , , , ">>, - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - - std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - - % step2, send a WRITE command to device - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - CmdId = 307, - Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"write">>, - <<"data">> => #{ - <<"path">> => <<"/3/0/13">>, - <<"type">> => <<"Integer">>, - <<"value">> => <<"12345">> - } - }, - CommandJson = emqx_json:encode(Command), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, - Path2 = get_coap_path(Options2), - ?assertEqual(put, Method2), - ?assertEqual(<<"/3/0/13">>, Path2), - Tlv_Value = <<3:2, 0:1, 0:2, 2:3, 13, 12345:16>>, - ?assertEqual(Tlv_Value, Payload2), - timer:sleep(50), - - test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true), - timer:sleep(100), - - ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/13">>, - <<"code">> => <<"2.04">>, - <<"codeMsg">> => <<"changed">> - }, - <<"msgType">> => <<"write">> - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). - -case20_write(Config) -> - % step 1, device register ... - Epn = "urn:oma:lwm2m:oma:3", - MsgId1 = 15, - UdpSock = ?config(sock, Config), - ObjectList = <<", , , , ">>, - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - - std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - - % step2, send a WRITE command to device - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - CmdId = 307, - Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"write">>, - <<"data">> => #{ - <<"basePath">> => <<"/3/0/13">>, - <<"content">> => [#{ - type => <<"Float">>, - value => <<"12345.0">> - }] - } - }, - CommandJson = emqx_json:encode(Command), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, - Path2 = get_coap_path(Options2), - ?assertEqual(put, Method2), - ?assertEqual(<<"/3/0/13">>, Path2), - Tlv_Value = <<200, 13, 8, 64,200,28,128,0,0,0,0>>, - ?assertEqual(Tlv_Value, Payload2), - timer:sleep(50), - - test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true), - timer:sleep(100), - - WriteResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/13">>, - <<"code">> => <<"2.04">>, - <<"codeMsg">> => <<"changed">> - }, - <<"msgType">> => <<"write">> - }), - ?assertEqual(WriteResult, test_recv_mqtt_response(RespTopic)). - -case21_write_object(Config) -> - % step 1, device register ... - Epn = "urn:oma:lwm2m:oma:3", - MsgId1 = 15, - UdpSock = ?config(sock, Config), - ObjectList = <<", , , , ">>, - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - - std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - - % step2, send a WRITE command to device - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - CmdId = 307, - Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"write">>, - <<"data">> => #{ - <<"basePath">> => <<"/3/0/">>, - <<"content">> => [#{ - path => <<"13">>, - type => <<"Integer">>, - value => <<"12345">> - },#{ - path => <<"14">>, - type => <<"String">>, - value => <<"87x">> - }] - } - }, - CommandJson = emqx_json:encode(Command), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, - Path2 = get_coap_path(Options2), - ?assertEqual(post, Method2), - ?assertEqual(<<"/3/0">>, Path2), - Tlv_Value = <<3:2, 0:1, 0:2, 2:3, 13, 12345:16, - 3:2, 0:1, 0:2, 3:3, 14, "87x">>, - ?assertEqual(Tlv_Value, Payload2), - timer:sleep(50), - - test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true), - timer:sleep(100), - - - ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"write">>, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/">>, - <<"code">> => <<"2.04">>, - <<"codeMsg">> => <<"changed">> - } - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). - -case22_write_error(Config) -> - % step 1, device register ... - Epn = "urn:oma:lwm2m:oma:3", - MsgId1 = 15, - UdpSock = ?config(sock, Config), - ObjectList = <<", , , , ">>, - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - - std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - - % step2, send a WRITE command to device - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - CmdId = 307, - Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"write">>, - <<"data">> => #{ - <<"basePath">> => <<"/3/0/1">>, - <<"content">> => [ - #{ - type => <<"Integer">>, - value => <<"12345">> - } - ] - } - }, - CommandJson = emqx_json:encode(Command), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2, options=Options2} = Request2, - Path2 = get_coap_path(Options2), - ?assertEqual(put, Method2), - ?assertEqual(<<"/3/0/1">>, Path2), - timer:sleep(50), - - test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {error, bad_request}, #coap_content{}, Request2, true), - timer:sleep(100), - - ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/1">>, - <<"code">> => <<"4.00">>, - <<"codeMsg">> => <<"bad_request">> - }, - <<"msgType">> => <<"write">> - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). - -case_create_basic(Config) -> - % step 1, device register ... - Epn = "urn:oma:lwm2m:oma:3", - MsgId1 = 15, - UdpSock = ?config(sock, Config), - ObjectList = <<", , , ">>, - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - - std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - - % step2, send a CREATE command to device - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - CmdId = 307, - Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"create">>, - <<"data">> => #{ - <<"path">> => <<"/5">> - } - }, - CommandJson = emqx_json:encode(Command), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, - Path2 = get_coap_path(Options2), - ?assertEqual(post, Method2), - ?assertEqual(<<"/5">>, Path2), - ?assertEqual(<<"">>, Payload2), - timer:sleep(50), - - test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, created}, #coap_content{}, Request2, true), - timer:sleep(100), - - ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"data">> => #{ - <<"reqPath">> => <<"/5">>, - <<"code">> => <<"2.01">>, - <<"codeMsg">> => <<"created">> - }, - <<"msgType">> => <<"create">> - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). - -case_delete_basic(Config) -> - % step 1, device register ... - Epn = "urn:oma:lwm2m:oma:3", - MsgId1 = 15, - UdpSock = ?config(sock, Config), - ObjectList = <<", , , , , ">>, - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - - std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - - % step2, send a CREATE command to device - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - CmdId = 307, - Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"delete">>, - <<"data">> => #{ - <<"path">> => <<"/5/0">> - } - }, - CommandJson = emqx_json:encode(Command), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, - Path2 = get_coap_path(Options2), - ?assertEqual(delete, Method2), - ?assertEqual(<<"/5/0">>, Path2), - ?assertEqual(<<"">>, Payload2), - timer:sleep(50), - - test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, deleted}, #coap_content{}, Request2, true), - timer:sleep(100), - - ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"data">> => #{ - <<"reqPath">> => <<"/5/0">>, - <<"code">> => <<"2.02">>, - <<"codeMsg">> => <<"deleted">> - }, - <<"msgType">> => <<"delete">> - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). - -case30_execute(Config) -> - % step 1, device register ... - Epn = "urn:oma:lwm2m:oma:3", - MsgId1 = 15, - UdpSock = ?config(sock, Config), - ObjectList = <<", , , , ">>, - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - - std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - - % step2, send a WRITE command to device - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - CmdId = 307, - Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"execute">>, - <<"data">> => #{ - <<"path">> => <<"/3/0/4">>, - %% "args" should not be present for "/3/0/4", only for testing the encoding here - <<"args">> => <<"2,7">> - } - }, - CommandJson = emqx_json:encode(Command), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, - Path2 = get_coap_path(Options2), - ?assertEqual(post, Method2), - ?assertEqual(<<"/3/0/4">>, Path2), - ?assertEqual(<<"2,7">>, Payload2), - timer:sleep(50), - - test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true), - timer:sleep(100), - - ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/4">>, - <<"code">> => <<"2.04">>, - <<"codeMsg">> => <<"changed">> - }, - <<"msgType">> => <<"execute">> - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). - -case31_execute_error(Config) -> - % step 1, device register ... - Epn = "urn:oma:lwm2m:oma:3", - MsgId1 = 15, - UdpSock = ?config(sock, Config), - ObjectList = <<", , , , ">>, - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - - std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - - % step2, send a WRITE command to device - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - CmdId = 307, - Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"execute">>, - <<"data">> => #{ - <<"path">> => <<"/3/0/4">>, - <<"args">> => <<"2,7">> - } - }, - CommandJson = emqx_json:encode(Command), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, - Path2 = get_coap_path(Options2), - ?assertEqual(post, Method2), - ?assertEqual(<<"/3/0/4">>, Path2), - ?assertEqual(<<"2,7">>, Payload2), - timer:sleep(50), - - test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {error, uauthorized}, #coap_content{}, Request2, true), - timer:sleep(100), - - ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/4">>, - <<"code">> => <<"4.01">>, - <<"codeMsg">> => <<"uauthorized">> - }, - <<"msgType">> => <<"execute">> - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). - -case40_discover(Config) -> - % step 1, device register ... - Epn = "urn:oma:lwm2m:oma:3", - MsgId1 = 15, - UdpSock = ?config(sock, Config), - ObjectList = <<", , , , ">>, - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - - std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - - % step2, send a WRITE command to device - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - CmdId = 307, - Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"discover">>, - <<"data">> => #{ - <<"path">> => <<"/3/0/7">> - } }, - CommandJson = emqx_json:encode(Command), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, - Path2 = get_coap_path(Options2), - ?assertEqual(get, Method2), - ?assertEqual(<<"/3/0/7">>, Path2), - ?assertEqual(<<>>, Payload2), - timer:sleep(50), - - PayloadDiscover = <<";dim=8;pmin=10;pmax=60;gt=50;lt=42.2,">>, - test_send_coap_response(UdpSock, - "127.0.0.1", - ?PORT, - {ok, content}, - #coap_content{content_format = <<"application/link-format">>, payload = PayloadDiscover}, - Request2, - true), - timer:sleep(100), - - ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"discover">>, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/7">>, - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"content">> => - [<<";dim=8;pmin=10;pmax=60;gt=50;lt=42.2">>, <<"">>] - } - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). - -case50_write_attribute(Config) -> - % step 1, device register ... - Epn = "urn:oma:lwm2m:oma:3", - MsgId1 = 15, - UdpSock = ?config(sock, Config), - ObjectList = <<", , , , ">>, - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - - std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - - % step2, send a WRITE command to device - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - CmdId = 307, - Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"write-attr">>, - <<"data">> => #{ - <<"path">> => <<"/3/0/9">>, - <<"pmin">> => <<"1">>, - <<"pmax">> => <<"5">>, - <<"lt">> => <<"5">> - } }, - CommandJson = emqx_json:encode(Command), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(100), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, - ?LOGT("got options: ~p", [Options2]), - Path2 = get_coap_path(Options2), - Query2 = lists:sort(get_coap_query(Options2)), - ?assertEqual(put, Method2), - ?assertEqual(<<"/3/0/9">>, Path2), - ?assertEqual(lists:sort([<<"pmax=5">>,<<"lt=5">>,<<"pmin=1">>]), Query2), - ?assertEqual(<<>>, Payload2), - timer:sleep(50), - - test_send_coap_response(UdpSock, - "127.0.0.1", - ?PORT, - {ok, changed}, - #coap_content{}, - Request2, - true), - timer:sleep(100), - - ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/9">>, - <<"code">> => <<"2.04">>, - <<"codeMsg">> => <<"changed">> - }, - <<"msgType">> => <<"write-attr">> - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). - -case60_observe(Config) -> - % step 1, device register ... - Epn = "urn:oma:lwm2m:oma:3", - MsgId1 = 15, - UdpSock = ?config(sock, Config), - ObjectList = <<", , , , ">>, - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - RespTopicAD = list_to_binary("lwm2m/"++Epn++"/up/notify"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - emqtt:subscribe(?config(emqx_c, Config), RespTopicAD, qos0), - timer:sleep(200), - - std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - - % step2, send a OBSERVE command to device - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - CmdId = 307, - Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"observe">>, - <<"data">> => #{ - <<"path">> => <<"/3/0/10">> - } - }, - CommandJson = emqx_json:encode(Command), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, - Path2 = get_coap_path(Options2), - Observe = get_coap_observe(Options2), - ?assertEqual(get, Method2), - ?assertEqual(<<"/3/0/10">>, Path2), - ?assertEqual(Observe, 0), - ?assertEqual(<<>>, Payload2), - timer:sleep(50), - - test_send_coap_observe_ack( UdpSock, - "127.0.0.1", - ?PORT, - {ok, content}, - #coap_content{content_format = <<"text/plain">>, payload = <<"2048">>}, - Request2), - timer:sleep(100), - - ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"observe">>, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/10">>, - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"content">> => [#{ - path => <<"/3/0/10">>, - value => 2048 - }] - } - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)), - - %% step3 the notifications - timer:sleep(200), - ObSeq = 3, - test_send_coap_notif( UdpSock, - "127.0.0.1", - ?PORT, - #coap_content{content_format = <<"text/plain">>, payload = <<"4096">>}, - ObSeq, - Request2), - timer:sleep(100), - #coap_message{} = test_recv_coap_response(UdpSock), - - ReadResult2 = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"notify">>, - <<"seqNum">> => ObSeq, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/10">>, - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"content">> => [#{ - path => <<"/3/0/10">>, - value => 4096 - }] - } - }), - ?assertEqual(ReadResult2, test_recv_mqtt_response(RespTopicAD)), - - %% Step3. cancel observe - CmdId3 = 308, - Command3 = #{<<"requestID">> => CmdId3, <<"cacheID">> => CmdId3, - <<"msgType">> => <<"cancel-observe">>, - <<"data">> => #{ - <<"path">> => <<"/3/0/10">> - } - }, - CommandJson3 = emqx_json:encode(Command3), - test_mqtt_broker:publish(CommandTopic, CommandJson3, 0), - timer:sleep(50), - Request3 = test_recv_coap_request(UdpSock), - #coap_message{method = Method3, options=Options3, payload=Payload3} = Request3, - Path3 = get_coap_path(Options3), - Observe3 = get_coap_observe(Options3), - ?assertEqual(get, Method3), - ?assertEqual(<<"/3/0/10">>, Path3), - ?assertEqual(Observe3, 1), - ?assertEqual(<<>>, Payload3), - timer:sleep(50), - - test_send_coap_observe_ack( UdpSock, - "127.0.0.1", - ?PORT, - {ok, content}, - #coap_content{content_format = <<"text/plain">>, payload = <<"1150">>}, - Request3), - timer:sleep(100), - - ReadResult3 = emqx_json:encode(#{ - <<"requestID">> => CmdId3, <<"cacheID">> => CmdId3, - <<"msgType">> => <<"cancel-observe">>, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/10">>, - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"content">> => [#{ - path => <<"/3/0/10">>, - value => 1150 - }] - } - }), - ?assertEqual(ReadResult3, test_recv_mqtt_response(RespTopic)). - -case80_specail_object_19_0_0_notify(Config) -> - % step 1, device register, with extra register options - Epn = "urn:oma:lwm2m:oma:3", - RegOptionWangYi = "&apn=psmA.eDRX0.ctnb&im=13456&ct=2.0&mt=MDM9206&mv=4.0", - MsgId1 = 15, - UdpSock = ?config(sock, Config), - - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1"++RegOptionWangYi, [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, - [], - MsgId1), - #coap_message{method = Method1} = test_recv_coap_response(UdpSock), - ?assertEqual({ok,created}, Method1), - ReadResult = emqx_json:encode(#{ - <<"msgType">> => <<"register">>, - <<"data">> => #{ - <<"alternatePath">> => <<"/">>, - <<"ep">> => list_to_binary(Epn), - <<"lt">> => 345, - <<"lwm2m">> => <<"1">>, - <<"objectList">> => [<<"/1">>, <<"/2">>, <<"/3">>, <<"/4">>, <<"/5">>], - <<"apn">> => <<"psmA.eDRX0.ctnb">>, - <<"im">> => <<"13456">>, - <<"ct">> => <<"2.0">>, - <<"mt">> => <<"MDM9206">>, - <<"mv">> => <<"4.0">> - } - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)), - - % step2, send a OBSERVE command to device - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - CmdId = 307, - Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"observe">>, - <<"data">> => #{ - <<"path">> => <<"/19/0/0">> - } - }, - CommandJson = emqx_json:encode(Command), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, - Path2 = get_coap_path(Options2), - Observe = get_coap_observe(Options2), - ?assertEqual(get, Method2), - ?assertEqual(<<"/19/0/0">>, Path2), - ?assertEqual(Observe, 0), - ?assertEqual(<<>>, Payload2), - timer:sleep(50), - - test_send_coap_observe_ack( UdpSock, - "127.0.0.1", - ?PORT, - {ok, content}, - #coap_content{content_format = <<"text/plain">>, payload = <<"2048">>}, - Request2), - timer:sleep(100). - - %% step 3, device send uplink data notifications - -case80_specail_object_19_1_0_write(Config) -> - Epn = "urn:oma:lwm2m:oma:3", - RegOptionWangYi = "&apn=psmA.eDRX0.ctnb&im=13456&ct=2.0&mt=MDM9206&mv=4.0", - MsgId1 = 15, - UdpSock = ?config(sock, Config), - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1"++RegOptionWangYi, [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, - [], - MsgId1), - #coap_message{method = Method1} = test_recv_coap_response(UdpSock), - ?assertEqual({ok,created}, Method1), - test_recv_mqtt_response(RespTopic), - - % step2, send a WRITE command to device - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - CmdId = 307, - Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"write">>, - <<"data">> => #{ - <<"path">> => <<"/19/1/0">>, - <<"type">> => <<"Opaque">>, - <<"value">> => base64:encode(<<12345:32>>) - } - }, - - CommandJson = emqx_json:encode(Command), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, - Path2 = get_coap_path(Options2), - ?assertEqual(put, Method2), - ?assertEqual(<<"/19/1/0">>, Path2), - ?assertEqual(<<3:2, 0:1, 0:2, 4:3, 0, 12345:32>>, Payload2), - timer:sleep(50), - - test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true), - timer:sleep(100), - - ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"data">> => #{ - <<"reqPath">> => <<"/19/1/0">>, - <<"code">> => <<"2.04">>, - <<"codeMsg">> => <<"changed">> - }, - <<"msgType">> => <<"write">> - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). - -case90_psm_mode(Config) -> - server_cache_mode(Config, "ep=~s<=345&lwm2m=1&apn=psmA.eDRX0.ctnb"). - -case90_queue_mode(Config) -> - server_cache_mode(Config, "ep=~s<=345&lwm2m=1&b=UQ"). - -server_cache_mode(Config, RegOption) -> - application:set_env(?APP, qmode_time_window, 2), - - % step 1, device register, with apn indicates "PSM" mode - Epn = "urn:oma:lwm2m:oma:3", - - MsgId1 = 15, - UdpSock = ?config(sock, Config), - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), - - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?"++RegOption, [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, - [], - MsgId1), - #coap_message{type = ack, method = Method1, options = Opts} = test_recv_coap_response(UdpSock), - ?assertEqual({ok,created}, Method1), - ?LOGT("Options got: ~p", [Opts]), - Location = proplists:get_value(location_path, Opts), - test_recv_mqtt_response(RespTopic), - - %% server not in PSM mode - send_read_command_1(0, UdpSock), - verify_read_response_1(0, UdpSock), - - %% server inters into PSM mode - timer:sleep(2), - - %% verify server caches downlink commands - send_read_command_1(1, UdpSock), - send_read_command_1(2, UdpSock), - send_read_command_1(3, UdpSock), - - ?assertEqual(timeout_test_recv_coap_request, test_recv_coap_request(UdpSock)), - - device_update_1(UdpSock, Location), - - verify_read_response_1(1, UdpSock), - verify_read_response_1(2, UdpSock), - verify_read_response_1(3, UdpSock). - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%%% Internal Functions -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -send_read_command_1(CmdId, _UdpSock) -> - Epn = "urn:oma:lwm2m:oma:3", - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - Command = #{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"path">> => <<"/3/0/0">> - } - }, - CommandJson = emqx_json:encode(Command), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50). - -verify_read_response_1(CmdId, UdpSock) -> - Epn = "urn:oma:lwm2m:oma:3", - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - - %% device receives a command - Request = test_recv_coap_request(UdpSock), - ?LOGT("LwM2M client got ~p", [Request]), - - %% device replies the commond - test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request, true), - - ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"content">> => [#{ - path => <<"/3/0/0">>, - value => <<"EMQ">> - }] - } - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). - -device_update_1(UdpSock, Location) -> - Epn = "urn:oma:lwm2m:oma:3", - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - ?LOGT("send UPDATE command", []), - MsgId2 = 27, - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b~s?lt=789", [?PORT, join_path(Location, <<>>)]), - #coap_content{payload = <<>>}, - [], - MsgId2), - #coap_message{type = ack, id = MsgId2, method = Method2} = test_recv_coap_response(UdpSock), - {ok,changed} = Method2, - test_recv_mqtt_response(RespTopic). - -test_recv_mqtt_response(RespTopic) -> - receive - {publish, #{topic := RespTopic, payload := RM}} -> - ?LOGT("test_recv_mqtt_response Response=~p", [RM]), - RM - after 1000 -> timeout_test_recv_mqtt_response - end. - -test_send_coap_request(UdpSock, Method, Uri, Content, Options, MsgId) -> - is_record(Content, coap_content) orelse error("Content must be a #coap_content!"), - is_list(Options) orelse error("Options must be a list"), - case resolve_uri(Uri) of - {coap, {IpAddr, Port}, Path, Query} -> - Request0 = lwm2m_coap_message:request(con, Method, Content, [{uri_path, Path}, {uri_query, Query} | Options]), - Request = Request0#coap_message{id = MsgId}, - ?LOGT("send_coap_request Request=~p", [Request]), - RequestBinary = lwm2m_coap_message_parser:encode(Request), - ?LOGT("test udp socket send to ~p:~p, data=~p", [IpAddr, Port, RequestBinary]), - ok = gen_udp:send(UdpSock, IpAddr, Port, RequestBinary); - {SchemeDiff, ChIdDiff, _, _} -> - error(lists:flatten(io_lib:format("scheme ~s or ChId ~s does not match with socket", [SchemeDiff, ChIdDiff]))) - end. - -test_recv_coap_response(UdpSock) -> - {ok, {Address, Port, Packet}} = gen_udp:recv(UdpSock, 0, 2000), - Response = lwm2m_coap_message_parser:decode(Packet), - ?LOGT("test udp receive from ~p:~p, data1=~p, Response=~p", [Address, Port, Packet, Response]), - #coap_message{type = ack, method = Method, id=Id, token = Token, options = Options, payload = Payload} = Response, - ?LOGT("receive coap response Method=~p, Id=~p, Token=~p, Options=~p, Payload=~p", [Method, Id, Token, Options, Payload]), - Response. - -test_recv_coap_request(UdpSock) -> - case gen_udp:recv(UdpSock, 0, 2000) of - {ok, {_Address, _Port, Packet}} -> - Request = lwm2m_coap_message_parser:decode(Packet), - #coap_message{type = con, method = Method, id=Id, token = Token, payload = Payload, options = Options} = Request, - ?LOGT("receive coap request Method=~p, Id=~p, Token=~p, Options=~p, Payload=~p", [Method, Id, Token, Options, Payload]), - Request; - {error, Reason} -> - ?LOGT("test_recv_coap_request failed, Reason=~p", [Reason]), - timeout_test_recv_coap_request - end. - -test_send_coap_response(UdpSock, Host, Port, Code, Content, Request, Ack) -> - is_record(Content, coap_content) orelse error("Content must be a #coap_content!"), - is_list(Host) orelse error("Host is not a string"), - - {ok, IpAddr} = inet:getaddr(Host, inet), - Response = lwm2m_coap_message:response(Code, Content, Request), - Response2 = case Ack of - true -> Response#coap_message{type = ack}; - false -> Response - end, - ?LOGT("test_send_coap_response Response=~p", [Response2]), - ok = gen_udp:send(UdpSock, IpAddr, Port, lwm2m_coap_message_parser:encode(Response2)). - -test_send_empty_ack(UdpSock, Host, Port, Request) -> - is_list(Host) orelse error("Host is not a string"), - {ok, IpAddr} = inet:getaddr(Host, inet), - EmptyACK = lwm2m_coap_message:ack(Request), - ?LOGT("test_send_empty_ack EmptyACK=~p", [EmptyACK]), - ok = gen_udp:send(UdpSock, IpAddr, Port, lwm2m_coap_message_parser:encode(EmptyACK)). - -test_send_coap_observe_ack(UdpSock, Host, Port, Code, Content, Request) -> - is_record(Content, coap_content) orelse error("Content must be a #coap_content!"), - is_list(Host) orelse error("Host is not a string"), - - {ok, IpAddr} = inet:getaddr(Host, inet), - Response = lwm2m_coap_message:response(Code, Content, Request), - Response1 = lwm2m_coap_message:set(observe, 0, Response), - Response2 = Response1#coap_message{type = ack}, - - ?LOGT("test_send_coap_observe_ack Response=~p", [Response2]), - ResponseBinary = lwm2m_coap_message_parser:encode(Response2), - ok = gen_udp:send(UdpSock, IpAddr, Port, ResponseBinary). - -test_send_coap_notif(UdpSock, Host, Port, Content, ObSeq, Request) -> - is_record(Content, coap_content) orelse error("Content must be a #coap_content!"), - is_list(Host) orelse error("Host is not a string"), - - {ok, IpAddr} = inet:getaddr(Host, inet), - Notif = lwm2m_coap_message:response({ok, content}, Content, Request), - NewNotif = lwm2m_coap_message:set(observe, ObSeq, Notif), - ?LOGT("test_send_coap_notif Response=~p", [NewNotif]), - NotifBinary = lwm2m_coap_message_parser:encode(NewNotif), - ?LOGT("test udp socket send to ~p:~p, data=~p", [IpAddr, Port, NotifBinary]), - ok = gen_udp:send(UdpSock, IpAddr, Port, NotifBinary). - -std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic) -> - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, payload = ObjectList}, - [], - MsgId1), - #coap_message{method = {ok,created}} = test_recv_coap_response(UdpSock), - test_recv_mqtt_response(RespTopic), - timer:sleep(100). - -resolve_uri(Uri) -> - {ok, #{scheme := Scheme, - host := Host, - port := PortNo, - path := Path} = URIMap} = emqx_http_lib:uri_parse(Uri), - Query = maps:get(query, URIMap, ""), - {ok, PeerIP} = inet:getaddr(Host, inet), - {Scheme, {PeerIP, PortNo}, split_path(Path), split_query(Query)}. - -split_path([]) -> []; -split_path([$/]) -> []; -split_path([$/ | Path]) -> split_segments(Path, $/, []). - -split_query([]) -> []; -split_query(Path) -> split_segments(Path, $&, []). - -split_segments(Path, Char, Acc) -> - case string:rchr(Path, Char) of - 0 -> - [make_segment(Path) | Acc]; - N when N > 0 -> - split_segments(string:substr(Path, 1, N-1), Char, - [make_segment(string:substr(Path, N+1)) | Acc]) - end. - -make_segment(Seg) -> - list_to_binary(emqx_http_lib:uri_decode(Seg)). - - -get_coap_path(Options) -> - get_path(Options, <<>>). - -get_coap_query(Options) -> - proplists:get_value(uri_query, Options, []). - -get_coap_observe(Options) -> - get_observe(Options). - - -get_path([], Acc) -> - %?LOGT("get_path Acc=~p", [Acc]), - Acc; -get_path([{uri_path, Path1}|T], Acc) -> - %?LOGT("Path=~p, Acc=~p", [Path1, Acc]), - get_path(T, join_path(Path1, Acc)); -get_path([{_, _}|T], Acc) -> - get_path(T, Acc). - -get_observe([]) -> - undefined; -get_observe([{observe, V}|_T]) -> - V; -get_observe([{_, _}|T]) -> - get_observe(T). - -join_path([], Acc) -> Acc; -join_path([<<"/">>|T], Acc) -> - join_path(T, Acc); -join_path([H|T], Acc) -> - join_path(T, <>). - -sprintf(Format, Args) -> - lists:flatten(io_lib:format(Format, Args)). diff --git a/apps/emqx_lwm2m/test/emqx_tlv_SUITE.erl b/apps/emqx_lwm2m/test/emqx_tlv_SUITE.erl deleted file mode 100644 index bfb85d832..000000000 --- a/apps/emqx_lwm2m/test/emqx_tlv_SUITE.erl +++ /dev/null @@ -1,240 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_tlv_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --define(LOGT(Format, Args), logger:debug("TEST_SUITE: " ++ Format, Args)). - --include("emqx_lwm2m.hrl"). --include_lib("lwm2m_coap/include/coap.hrl"). --include_lib("eunit/include/eunit.hrl"). - - -all() -> [case01, case02, case03, case03_0, case04, case05, case06, case07, case08, case09]. - - - -init_per_suite(Config) -> - Config. - -end_per_suite(Config) -> - Config. - - -case01(_Config) -> - Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). - -case02(_Config) -> - Data = <<16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_multiple_resource => 16#06, value => [ - #{tlv_resource_instance => 16#00, value => <<1>>}, - #{tlv_resource_instance => 16#01, value => <<5>>} - ]} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). - -case03(_Config) -> - Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>}, - #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>}, - #{tlv_resource_with_value => 16#02, value => <<"345000123">>} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). - -case03_0(_Config) -> - Data = <<16#87, 16#02, 16#41, 16#7F, 16#07, 16#61, 16#01, 16#36, 16#01>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_multiple_resource => 16#02, value => [ - #{tlv_resource_instance => 16#7F, value => <<16#07>>}, - #{tlv_resource_instance => 16#0136, value => <<16#01>>} - ]} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). - -case04(_Config) -> - % 6.4.3.1 Single Object Instance Request Example - Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33, 16#C3, 16#03, 16#31, 16#2E, 16#30, 16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05, 16#88, 16#07, 16#08, 16#42, 16#00, 16#0E, 16#D8, 16#42, 16#01, 16#13, 16#88, 16#87, 16#08, 16#41, 16#00, 16#7D, 16#42, 16#01, 16#03, 16#84, 16#C1, 16#09, 16#64, 16#C1, 16#0A, 16#0F, 16#83, 16#0B, 16#41, 16#00, 16#00, 16#C4, 16#0D, 16#51, 16#82, 16#42, 16#8F, 16#C6, 16#0E, 16#2B, 16#30, 16#32, 16#3A, 16#30, 16#30, 16#C1, 16#10, 16#55>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>}, - #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>}, - #{tlv_resource_with_value => 16#02, value => <<"345000123">>}, - #{tlv_resource_with_value => 16#03, value => <<"1.0">>}, - #{tlv_multiple_resource => 16#06, value => [ - #{tlv_resource_instance => 16#00, value => <<1>>}, - #{tlv_resource_instance => 16#01, value => <<5>>} - ]}, - #{tlv_multiple_resource => 16#07, value => [ - #{tlv_resource_instance => 16#00, value => <<16#0ED8:16>>}, - #{tlv_resource_instance => 16#01, value => <<16#1388:16>>} - ]}, - #{tlv_multiple_resource => 16#08, value => [ - #{tlv_resource_instance => 16#00, value => <<16#7d>>}, - #{tlv_resource_instance => 16#01, value => <<16#0384:16>>} - ]}, - #{tlv_resource_with_value => 16#09, value => <<16#64>>}, - #{tlv_resource_with_value => 16#0A, value => <<16#0F>>}, - #{tlv_multiple_resource => 16#0B, value => [ - #{tlv_resource_instance => 16#00, value => <<16#00>>} - ]}, - #{tlv_resource_with_value => 16#0D, value => <<16#5182428F:32>>}, - #{tlv_resource_with_value => 16#0E, value => <<"+02:00">>}, - #{tlv_resource_with_value => 16#10, value => <<"U">>} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). - -case05(_Config) -> - % 6.4.3.2 Multiple Object Instance Request Examples - % A) Request on Single-Instance Object - Data = <<16#08, 16#00, 16#79, 16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33, 16#C3, 16#03, 16#31, 16#2E, 16#30, 16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05, 16#88, 16#07, 16#08, 16#42, 16#00, 16#0E, 16#D8, 16#42, 16#01, 16#13, 16#88, 16#87, 16#08, 16#41, 16#00, 16#7D, 16#42, 16#01, 16#03, 16#84, 16#C1, 16#09, 16#64, 16#C1, 16#0A, 16#0F, 16#83, 16#0B, 16#41, 16#00, 16#00, 16#C4, 16#0D, 16#51, 16#82, 16#42, 16#8F, 16#C6, 16#0E, 16#2B, 16#30, 16#32, 16#3A, 16#30, 16#30, 16#C1, 16#10, 16#55>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_object_instance => 16#00, value => [ - #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>}, - #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>}, - #{tlv_resource_with_value => 16#02, value => <<"345000123">>}, - #{tlv_resource_with_value => 16#03, value => <<"1.0">>}, - #{tlv_multiple_resource => 16#06, value => [ - #{tlv_resource_instance => 16#00, value => <<1>>}, - #{tlv_resource_instance => 16#01, value => <<5>>} - ]}, - #{tlv_multiple_resource => 16#07, value => [ - #{tlv_resource_instance => 16#00, value => <<16#0ED8:16>>}, - #{tlv_resource_instance => 16#01, value => <<16#1388:16>>} - ]}, - #{tlv_multiple_resource => 16#08, value => [ - #{tlv_resource_instance => 16#00, value => <<16#7d>>}, - #{tlv_resource_instance => 16#01, value => <<16#0384:16>>} - ]}, - #{tlv_resource_with_value => 16#09, value => <<16#64>>}, - #{tlv_resource_with_value => 16#0A, value => <<16#0F>>}, - #{tlv_multiple_resource => 16#0B, value => [ - #{tlv_resource_instance => 16#00, value => <<16#00>>} - ]}, - #{tlv_resource_with_value => 16#0D, value => <<16#5182428F:32>>}, - #{tlv_resource_with_value => 16#0E, value => <<"+02:00">>}, - #{tlv_resource_with_value => 16#10, value => <<"U">>} - ]} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). - -case06(_Config) -> - % 6.4.3.2 Multiple Object Instance Request Examples - % B) Request on Multiple-Instances Object having 2 instances - Data = <<16#08, 16#00, 16#0E, 16#C1, 16#00, 16#01, 16#C1, 16#01, 16#00, 16#83, 16#02, 16#41, 16#7F, 16#07, 16#C1, 16#03, 16#7F, 16#08, 16#02, 16#12, 16#C1, 16#00, 16#03, 16#C1, 16#01, 16#00, 16#87, 16#02, 16#41, 16#7F, 16#07, 16#61, 16#01, 16#36, 16#01, 16#C1, 16#03, 16#7F>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_object_instance => 16#00, value => [ - #{tlv_resource_with_value => 16#00, value => <<16#01>>}, - #{tlv_resource_with_value => 16#01, value => <<16#00>>}, - #{tlv_multiple_resource => 16#02, value => [ - #{tlv_resource_instance => 16#7F, value => <<16#07>>} - ]}, - #{tlv_resource_with_value => 16#03, value => <<16#7F>>} - ]}, - #{tlv_object_instance => 16#02, value => [ - #{tlv_resource_with_value => 16#00, value => <<16#03>>}, - #{tlv_resource_with_value => 16#01, value => <<16#00>>}, - #{tlv_multiple_resource => 16#02, value => [ - #{tlv_resource_instance => 16#7F, value => <<16#07>>}, - #{tlv_resource_instance => 16#0136, value => <<16#01>>} - ]}, - #{tlv_resource_with_value => 16#03, value => <<16#7F>>} - ]} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). - -case07(_Config) -> - % 6.4.3.2 Multiple Object Instance Request Examples - % C) Request on Multiple-Instances Object having 1 instance only - Data = <<16#08, 16#00, 16#0F, 16#C1, 16#00, 16#01, 16#C4, 16#01, 16#00, 16#01, 16#51, 16#80, 16#C1, 16#06, 16#01, 16#C1, 16#07, 16#55>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_object_instance => 16#00, value => [ - #{tlv_resource_with_value => 16#00, value => <<16#01>>}, - #{tlv_resource_with_value => 16#01, value => <<86400:32>>}, - #{tlv_resource_with_value => 16#06, value => <<16#01>>}, - #{tlv_resource_with_value => 16#07, value => <<$U>>}]} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). - -case08(_Config) -> - % 6.4.3.3 Example of Request on an Object Instance containing an Object Link Resource - % Example 1) request to Object 65 Instance 0: Read /65/0 - Data = <<16#88, 16#00, 16#0C, 16#44, 16#00, 16#00, 16#42, 16#00, 16#00, 16#44, 16#01, 16#00, 16#42, 16#00, 16#01, 16#C8, 16#01, 16#0D, 16#38, 16#36, 16#31, 16#33, 16#38, 16#30, 16#30, 16#37, 16#35, 16#35, 16#35, 16#30, 16#30, 16#C4, 16#02, 16#12, 16#34, 16#56, 16#78>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_multiple_resource => 16#00, value => [ - #{tlv_resource_instance => 16#00, value => <<16#00, 16#42, 16#00, 16#00>>}, - #{tlv_resource_instance => 16#01, value => <<16#00, 16#42, 16#00, 16#01>>} - ]}, - #{tlv_resource_with_value => 16#01, value => <<"8613800755500">>}, - #{tlv_resource_with_value => 16#02, value => <<16#12345678:32>>} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). - -case09(_Config) -> - % 6.4.3.3 Example of Request on an Object Instance containing an Object Link Resource - % Example 2) request to Object 66: Read /66: TLV payload will contain 2 Object Instances - Data = <<16#08, 16#00, 16#26, 16#C8, 16#00, 16#0B, 16#6D, 16#79, 16#53, 16#65, 16#72, 16#76, 16#69, 16#63, 16#65, 16#20, 16#31, 16#C8, 16#01, 16#0F, 16#49, 16#6E, 16#74, 16#65, 16#72, 16#6E, 16#65, 16#74, 16#2E, 16#31, 16#35, 16#2E, 16#32, 16#33, 16#34, 16#C4, 16#02, 16#00, 16#43, 16#00, 16#00, 16#08, 16#01, 16#26, 16#C8, 16#00, 16#0B, 16#6D, 16#79, 16#53, 16#65, 16#72, 16#76, 16#69, 16#63, 16#65, 16#20, 16#32, 16#C8, 16#01, 16#0F, 16#49, 16#6E, 16#74, 16#65, 16#72, 16#6E, 16#65, 16#74, 16#2E, 16#31, 16#35, 16#2E, 16#32, 16#33, 16#35, 16#C4, 16#02, 16#FF, 16#FF, 16#FF, 16#FF>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_object_instance => 16#00, value => [ - #{tlv_resource_with_value => 16#00, value => <<"myService 1">>}, - #{tlv_resource_with_value => 16#01, value => <<"Internet.15.234">>}, - #{tlv_resource_with_value => 16#02, value => <<16#00, 16#43, 16#00, 16#00>>} - ]}, - #{tlv_object_instance => 16#01, value => [ - #{tlv_resource_with_value => 16#00, value => <<"myService 2">>}, - #{tlv_resource_with_value => 16#01, value => <<"Internet.15.235">>}, - #{tlv_resource_with_value => 16#02, value => <<16#FF, 16#FF, 16#FF, 16#FF>>} - ]} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). - diff --git a/apps/emqx_lwm2m/test/test_mqtt_broker.erl b/apps/emqx_lwm2m/test/test_mqtt_broker.erl deleted file mode 100644 index dd85340b6..000000000 --- a/apps/emqx_lwm2m/test/test_mqtt_broker.erl +++ /dev/null @@ -1,171 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(test_mqtt_broker). - --compile(nowarn_export_all). --compile(export_all). - --define(LOGT(Format, Args), logger:debug("TEST_BROKER: " ++ Format, Args)). - --record(state, {subscriber}). - --include_lib("emqx/include/emqx.hrl"). - --include_lib("emqx/include/emqx_mqtt.hrl"). - --include_lib("eunit/include/eunit.hrl"). - -start(_, <<"attacker">>, _, _, _) -> - {stop, auth_failure}; -start(ClientId, Username, Password, _Channel, KeepaliveInterval) -> - true = is_binary(ClientId), - (true = ( is_binary(Username)) orelse (Username == undefined) ), - (true = ( is_binary(Password)) orelse (Password == undefined) ), - self() ! {keepalive, start, KeepaliveInterval}, - {ok, []}. - -publish(Topic, Payload, Qos) -> - ClientId = <<"lwm2m_test_suite">>, - Msg = emqx_message:make(ClientId, Qos, Topic, Payload), - emqx:publish(Msg). - -subscribe(Topic) -> - gen_server:call(?MODULE, {subscribe, Topic, self()}). - -unsubscribe(Topic) -> - gen_server:call(?MODULE, {unsubscribe, Topic}). - -get_subscrbied_topics() -> - [Topic || {_Client, Topic} <- ets:tab2list(emqx_subscription)]. - -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - -stop() -> - gen_server:stop(?MODULE). - -init(_Param) -> - {ok, #state{subscriber = []}}. - -handle_call({subscribe, Topic, Proc}, _From, State=#state{subscriber = SubList}) -> - ?LOGT("test broker subscribe Topic=~p, Pid=~p~n", [Topic, Proc]), - is_binary(Topic) orelse error("Topic should be a binary"), - {reply, {ok, []}, State#state{subscriber = [{Topic, Proc}|SubList]}}; - -handle_call(get_subscribed_topics, _From, State=#state{subscriber = SubList}) -> - Response = subscribed_topics(SubList, []), - ?LOGT("test broker get subscribed topics=~p~n", [Response]), - {reply, Response, State}; - -handle_call({unsubscribe, Topic}, _From, State=#state{subscriber = SubList}) -> - ?LOGT("test broker unsubscribe Topic=~p~n", [Topic]), - is_binary(Topic) orelse error("Topic should be a binary"), - NewSubList = proplists:delete(Topic, SubList), - {reply, {ok, []}, State#state{subscriber = NewSubList}}; - - -handle_call({publish, {Topic, Msg, MatchedTopicFilter}}, _From, State=#state{subscriber = SubList}) -> - (is_binary(Topic) and is_binary(Msg)) orelse error("Topic and Msg should be binary"), - Pid = proplists:get_value(MatchedTopicFilter, SubList), - ?LOGT("test broker publish topic=~p, Msg=~p, Pid=~p, MatchedTopicFilter=~p, SubList=~p~n", [Topic, Msg, Pid, MatchedTopicFilter, SubList]), - (Pid == undefined) andalso ?LOGT("!!!!! this topic ~p has never been subscribed, please specify a valid topic filter", [MatchedTopicFilter]), - ?assertNotEqual(undefined, Pid), - Pid ! {deliver, #message{topic = Topic, payload = Msg}}, - {reply, ok, State}; - -handle_call(stop, _From, State) -> - {stop, normal, stopped, State}; - -handle_call(Req, _From, State) -> - ?LOGT("test_broker_server: ignore call Req=~p~n", [Req]), - {reply, {error, badreq}, State}. - - -handle_cast(Msg, State) -> - ?LOGT("test_broker_server: ignore cast msg=~p~n", [Msg]), - {noreply, State}. - -handle_info(Info, State) -> - ?LOGT("test_broker_server: ignore info=~p~n", [Info]), - {noreply, State}. - -terminate(Reason, _State) -> - ?LOGT("test_broker_server: terminate Reason=~p~n", [Reason]), - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - - - - -subscribed_topics([], Acc) -> - Acc; -subscribed_topics([{Topic,_Pid}|T], Acc) -> - subscribed_topics(T, [Topic|Acc]). - - - - --record(keepalive, {statfun, statval, tsec, tmsg, tref, repeat = 0}). - --type(keepalive() :: #keepalive{}). - -%% @doc Start a keepalive --spec(start(fun(), integer(), any()) -> undefined | keepalive()). -start(_, 0, _) -> - undefined; -start(StatFun, TimeoutSec, TimeoutMsg) -> - {ok, StatVal} = StatFun(), - #keepalive{statfun = StatFun, statval = StatVal, - tsec = TimeoutSec, tmsg = TimeoutMsg, - tref = timer(TimeoutSec, TimeoutMsg)}. - -%% @doc Check keepalive, called when timeout. --spec(check(keepalive()) -> {ok, keepalive()} | {error, any()}). -check(KeepAlive = #keepalive{statfun = StatFun, statval = LastVal, repeat = Repeat}) -> - case StatFun() of - {ok, NewVal} -> - if NewVal =/= LastVal -> - {ok, resume(KeepAlive#keepalive{statval = NewVal, repeat = 0})}; - Repeat < 1 -> - {ok, resume(KeepAlive#keepalive{statval = NewVal, repeat = Repeat + 1})}; - true -> - {error, timeout} - end; - {error, Error} -> - {error, Error} - end. - -resume(KeepAlive = #keepalive{tsec = TimeoutSec, tmsg = TimeoutMsg}) -> - KeepAlive#keepalive{tref = timer(TimeoutSec, TimeoutMsg)}. - -%% @doc Cancel Keepalive --spec(cancel(keepalive()) -> ok). -cancel(#keepalive{tref = TRef}) -> - cancel(TRef); -cancel(undefined) -> - ok; -cancel(TRef) -> - catch erlang:cancel_timer(TRef). - -timer(Sec, Msg) -> - erlang:send_after(timer:seconds(Sec), self(), Msg). - - -log(Format, Args) -> - logger:debug(Format, Args). diff --git a/apps/emqx_management/README.md b/apps/emqx_management/README.md index c71a47628..52013c025 100644 --- a/apps/emqx_management/README.md +++ b/apps/emqx_management/README.md @@ -7,3 +7,6 @@ EMQ X Management API http://restful-api-design.readthedocs.io/en/latest/scope.html +default application see: +header: +authorization: Basic YWRtaW46cHVibGlj diff --git a/apps/emqx_management/etc/emqx_management.conf b/apps/emqx_management/etc/emqx_management.conf index 05a3008fa..127a21e3b 100644 --- a/apps/emqx_management/etc/emqx_management.conf +++ b/apps/emqx_management/etc/emqx_management.conf @@ -1,53 +1,42 @@ -##-------------------------------------------------------------------- -## EMQ X Management Plugin -##-------------------------------------------------------------------- - -## Max Row Limit -management.max_row_limit = 10000 - -## Application default secret -## -## Value: String -## management.application.default_secret = public - -## Default Application ID -## -## Value: String -management.default_application.id = admin - -## Default Application Secret -## -## Value: String -management.default_application.secret = public - -##-------------------------------------------------------------------- -## HTTP Listener - -management.listener.http.port = 8081 -management.listener.http.acceptors = 2 -management.listener.http.max_clients = 512 -management.listener.http.backlog = 512 -management.listener.http.send_timeout = 15s -management.listener.http.send_timeout_close = on -management.listener.http.inet6 = false -management.listener.http.ipv6_v6only = false - -##-------------------------------------------------------------------- -## HTTPS Listener - -## management.listener.https.port = 8081 -## management.listener.https.acceptors = 2 -## management.listener.https.max_clients = 512 -## management.listener.https.backlog = 512 -## management.listener.https.send_timeout = 15s -## management.listener.https.send_timeout_close = on -## management.listener.https.certfile = "etc/certs/cert.pem" -## management.listener.https.keyfile = "etc/certs/key.pem" -## management.listener.https.cacertfile = "etc/certs/cacert.pem" -## management.listener.https.verify = verify_peer -## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier -## management.listener.https.tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" -## management.listener.https.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" -## management.listener.https.fail_if_no_peer_cert = true -## management.listener.https.inet6 = false -## management.listener.https.ipv6_v6only = false +emqx_management:{ + applications: [ + { + id: "admin", + secret: "public" + } + ] + max_row_limit: 10000 + listeners: [ + { + num_acceptors: 4 + max_connections: 512 + protocol: http + port: 8081 + backlog: 512 + send_timeout: 15s + send_timeout_close: true + inet6: false + ipv6_v6only: false + } +## , +## { +## protocol: https +## port: 8081 +## acceptors: 2 +## backlog: 512 +## send_timeout: 15s +## send_timeout_close: true +## inet6: false +## ipv6_v6only: false +## certfile = "etc/certs/cert.pem" +## keyfile = "etc/certs/key.pem" +## cacertfile = "etc/certs/cacert.pem" +## verify = verify_peer +## tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" +## ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" +## fail_if_no_peer_cert = true +## inet6 = false +## ipv6_v6only = false +## } + ] +} diff --git a/apps/emqx_management/include/emqx_mgmt.hrl b/apps/emqx_management/include/emqx_mgmt.hrl index b952332c5..40baec4e1 100644 --- a/apps/emqx_management/include/emqx_mgmt.hrl +++ b/apps/emqx_management/include/emqx_mgmt.hrl @@ -35,3 +35,29 @@ -define(VERSIONS, ["4.0", "4.1", "4.2", "4.3"]). -define(MANAGEMENT_SHARD, emqx_management_shard). + +-define(GENERATE_API_METADATA(MetaData), + maps:fold( + fun(Method, MethodDef0, NextMetaData) -> + Default = #{ + tags => [?MODULE], + security => [#{application => []}]}, + MethodDef = + lists:foldl( + fun(Key, NMethodDef) -> + case maps:is_key(Key, NMethodDef) of + true -> + NMethodDef; + false -> + maps:put(Key, maps:get(Key, Default), NMethodDef) + end + end, MethodDef0, maps:keys(Default)), + maps:put(Method, MethodDef, NextMetaData) + end, + #{}, MetaData)). + +-define(GENERATE_API(Path, MetaData, Function), + {Path, ?GENERATE_API_METADATA(MetaData), Function}). + +-define(GENERATE_APIS(Apis), + [?GENERATE_API(Path, MetaData, Function) || {Path, MetaData, Function} <- Apis]). diff --git a/apps/emqx_management/priv/emqx_management.schema b/apps/emqx_management/priv/emqx_management.schema deleted file mode 100644 index 4e887809e..000000000 --- a/apps/emqx_management/priv/emqx_management.schema +++ /dev/null @@ -1,239 +0,0 @@ -%%-*- mode: erlang -*- -%% emqx_management config mapping - -{mapping, "management.max_row_limit", "emqx_management.max_row_limit", [ - {default, 10000}, - {datatype, integer} -]}. - -{mapping, "management.default_application.id", "emqx_management.default_application_id", [ - {default, undefined}, - {datatype, string} -]}. - -{mapping, "management.default_application.secret", "emqx_management.default_application_secret", [ - {default, undefined}, - {datatype, string} -]}. - -{mapping, "management.application.default_secret", "emqx_management.application", [ - {default, undefined}, - {datatype, string} -]}. - -{mapping, "management.listener.http.port", "emqx_management.listeners", [ - {datatype, [integer, ip]} -]}. - -{mapping, "management.listener.http.acceptors", "emqx_management.listeners", [ - {default, 4}, - {datatype, integer} -]}. - -{mapping, "management.listener.http.max_clients", "emqx_management.listeners", [ - {default, 512}, - {datatype, integer} -]}. - -{mapping, "management.listener.http.backlog", "emqx_management.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "management.listener.http.send_timeout", "emqx_management.listeners", [ - {datatype, {duration, ms}}, - {default, "15s"} -]}. - -{mapping, "management.listener.http.send_timeout_close", "emqx_management.listeners", [ - {datatype, flag}, - {default, on} -]}. - -{mapping, "management.listener.http.recbuf", "emqx_management.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "management.listener.http.sndbuf", "emqx_management.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "management.listener.http.buffer", "emqx_management.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "management.listener.http.tune_buffer", "emqx_management.listeners", [ - {datatype, flag}, - hidden -]}. - -{mapping, "management.listener.http.nodelay", "emqx_management.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -{mapping, "management.listener.http.inet6", "emqx_management.listeners", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "management.listener.http.ipv6_v6only", "emqx_management.listeners", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "management.listener.https.port", "emqx_management.listeners", [ - {datatype, [integer, ip]} -]}. - -{mapping, "management.listener.https.acceptors", "emqx_management.listeners", [ - {default, 8}, - {datatype, integer} -]}. - -{mapping, "management.listener.https.max_clients", "emqx_management.listeners", [ - {default, 64}, - {datatype, integer} -]}. - -{mapping, "management.listener.https.backlog", "emqx_management.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "management.listener.https.send_timeout", "emqx_management.listeners", [ - {datatype, {duration, ms}}, - {default, "15s"} -]}. - -{mapping, "management.listener.https.send_timeout_close", "emqx_management.listeners", [ - {datatype, flag}, - {default, on} -]}. - -{mapping, "management.listener.https.recbuf", "emqx_management.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "management.listener.https.sndbuf", "emqx_management.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "management.listener.https.buffer", "emqx_management.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "management.listener.https.tune_buffer", "emqx_management.listeners", [ - {datatype, flag}, - hidden -]}. - -{mapping, "management.listener.https.nodelay", "emqx_management.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -{mapping, "management.listener.https.keyfile", "emqx_management.listeners", [ - {datatype, string} -]}. - -{mapping, "management.listener.https.certfile", "emqx_management.listeners", [ - {datatype, string} -]}. - -{mapping, "management.listener.https.cacertfile", "emqx_management.listeners", [ - {datatype, string} -]}. - -{mapping, "management.listener.https.verify", "emqx_management.listeners", [ - {datatype, atom} -]}. - -{mapping, "management.listener.https.ciphers", "emqx_management.listeners", [ - {datatype, string} -]}. - -{mapping, "management.listener.https.tls_versions", "emqx_management.listeners", [ - {datatype, string} -]}. - -{mapping, "management.listener.https.fail_if_no_peer_cert", "emqx_management.listeners", [ - {datatype, {enum, [true, false]}} -]}. - -{mapping, "management.listener.https.inet6", "emqx_management.listeners", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "management.listener.https.ipv6_v6only", "emqx_management.listeners", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -{translation, "emqx_management.application", fun(Conf) -> - Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, - Opts = fun(Prefix) -> - Filter([{default_secret, cuttlefish:conf_get(Prefix ++ ".default_secret", Conf)}]) - end, - Prefix = "management.application", - Transfer = fun(default_secret, V) -> list_to_binary(V); - (_, V) -> V - end, - [{K, Transfer(K, V)}|| {K, V} <- Opts(Prefix)] -end}. - -{translation, "emqx_management.listeners", fun(Conf) -> - Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, - Opts = fun(Prefix) -> - Filter([{num_acceptors, cuttlefish:conf_get(Prefix ++ ".acceptors", Conf)}, - {max_connections, cuttlefish:conf_get(Prefix ++ ".max_clients", Conf)}]) - end, - TcpOpts = fun(Prefix) -> - Filter([{backlog, cuttlefish:conf_get(Prefix ++ ".backlog", Conf, undefined)}, - {send_timeout, cuttlefish:conf_get(Prefix ++ ".send_timeout", Conf, undefined)}, - {send_timeout_close, cuttlefish:conf_get(Prefix ++ ".send_timeout_close", Conf, undefined)}, - {recbuf, cuttlefish:conf_get(Prefix ++ ".recbuf", Conf, undefined)}, - {sndbuf, cuttlefish:conf_get(Prefix ++ ".sndbuf", Conf, undefined)}, - {buffer, cuttlefish:conf_get(Prefix ++ ".buffer", Conf, undefined)}, - {nodelay, cuttlefish:conf_get(Prefix ++ ".nodelay", Conf, true)}, - {inet6, cuttlefish:conf_get(Prefix ++ ".inet6", Conf)}, - {ipv6_v6only, cuttlefish:conf_get(Prefix ++ ".ipv6_v6only", Conf)}]) - end, - - SplitFun = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end, - - SslOpts = fun(Prefix) -> - Versions = case SplitFun(cuttlefish:conf_get(Prefix ++ ".tls_versions", Conf, undefined)) of - undefined -> undefined; - L -> [list_to_atom(V) || V <- L] - end, - Filter([{versions, Versions}, - {ciphers, SplitFun(cuttlefish:conf_get(Prefix ++ ".ciphers", Conf, undefined))}, - {keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, - {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, - {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)}, - {verify, cuttlefish:conf_get(Prefix ++ ".verify", Conf, undefined)}, - {fail_if_no_peer_cert, cuttlefish:conf_get(Prefix ++ ".fail_if_no_peer_cert", Conf, undefined)}]) - end, - lists:foldl( - fun(Proto, Acc) -> - Prefix = "management.listener." ++ atom_to_list(Proto), - case cuttlefish:conf_get(Prefix ++ ".port", Conf, undefined) of - undefined -> Acc; - Port -> - [{Proto, Port, TcpOpts(Prefix) ++ Opts(Prefix) - ++ case Proto of - http -> []; - https -> SslOpts(Prefix) - end} | Acc] - end - end, [], [http, https]) -end}. - diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index fe68fef44..6cd38d928 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.4.0"}, % strict semver, bump manually! + {vsn, "5.0.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/apps/emqx_management/src/emqx_management.appup.src b/apps/emqx_management/src/emqx_management.appup.src deleted file mode 100644 index 06945afad..000000000 --- a/apps/emqx_management/src/emqx_management.appup.src +++ /dev/null @@ -1,13 +0,0 @@ -%% -*- mode: erlang -*- -{VSN, - [ {<<"4.3.[0-2]">>, - [ {restart_application, emqx_management} - ]}, - {<<".*">>, []} - ], - [ {<<"4.3.[0-2]">>, - [ {restart_application, emqx_management} - ]}, - {<<".*">>, []} - ] -}. diff --git a/apps/emqx_management/src/emqx_management_schema.erl b/apps/emqx_management/src/emqx_management_schema.erl new file mode 100644 index 000000000..f9543697f --- /dev/null +++ b/apps/emqx_management/src/emqx_management_schema.erl @@ -0,0 +1,57 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_management_schema). + +-include_lib("typerefl/include/types.hrl"). + +-behaviour(hocon_schema). + +-export([ structs/0 + , fields/1]). + +structs() -> ["emqx_management"]. + +fields("emqx_management") -> + [ {applications, hoconsc:array(hoconsc:ref(?MODULE, "application"))} + , {max_row_limit, fun max_row_limit/1} + , {listeners, hoconsc:array(hoconsc:union([hoconsc:ref(?MODULE, "http"), hoconsc:ref(?MODULE, "https")]))} + ]; + +fields("application") -> + [ {"id", emqx_schema:t(string(), undefined, "admin")} + , {"secret", emqx_schema:t(string(), undefined, "public")} + ]; + + +fields("http") -> + [ {"protocol", hoconsc:enum([http, https])} + , {"port", emqx_schema:t(integer(), undefined, 8081)} + , {"num_acceptors", emqx_schema:t(integer(), undefined, 4)} + , {"max_connections", emqx_schema:t(integer(), undefined, 512)} + , {"backlog", emqx_schema:t(integer(), undefined, 1024)} + , {"send_timeout", emqx_schema:t(emqx_schema:duration(), undefined, "15s")} + , {"send_timeout_close", emqx_schema:t(boolean(), undefined, true)} + , {"inet6", emqx_schema:t(boolean(), undefined, false)} + , {"ipv6_v6only", emqx_schema:t(boolean(), undefined, false)} + ]; + +fields("https") -> + emqx_schema:ssl(#{enable => true}) ++ fields("http"). + +max_row_limit(type) -> integer(); +max_row_limit(default) -> 1000; +max_row_limit(nullable) -> false; +max_row_limit(_) -> undefined. diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index ce83b9b71..20f1aa108 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -42,11 +42,13 @@ -export([ lookup_client/2 , lookup_client/3 , kickout_client/1 - , list_acl_cache/1 - , clean_acl_cache/1 - , clean_acl_cache/2 - , clean_acl_cache_all/0 - , clean_acl_cache_all/1 + , list_authz_cache/1 + , list_client_subscriptions/1 + , client_subscriptions/2 + , clean_authz_cache/1 + , clean_authz_cache/2 + , clean_authz_cache_all/0 + , clean_authz_cache_all/1 , set_ratelimit_policy/2 , set_quota_policy/2 ]). @@ -85,7 +87,10 @@ %% Listeners -export([ list_listeners/0 , list_listeners/1 - , restart_listener/2 + , list_listeners/2 + , list_listeners_by_id/1 + , get_listener/2 + , manage_listener/2 ]). %% Alarms @@ -106,10 +111,19 @@ , max_row_limit/0 ]). +-export([ return/0 + , return/1]). + -define(MAX_ROW_LIMIT, 10000). -define(APP, emqx_management). +%% TODO: remove these function after all api use minirest version 1.X +return() -> + ok. +return(_Response) -> + ok. + %%-------------------------------------------------------------------- %% Node Info %%-------------------------------------------------------------------- @@ -166,7 +180,7 @@ broker_info(Node) -> %%-------------------------------------------------------------------- get_metrics() -> - [{Node, get_metrics(Node)} || Node <- ekka_mnesia:running_nodes()]. + nodes_info_count([get_metrics(Node) || Node <- ekka_mnesia:running_nodes()]). get_metrics(Node) when Node =:= node() -> emqx_metrics:all(); @@ -174,13 +188,44 @@ get_metrics(Node) -> rpc_call(Node, get_metrics, [Node]). get_stats() -> - [{Node, get_stats(Node)} || Node <- ekka_mnesia:running_nodes()]. + GlobalStatsKeys = + [ 'retained.count' + , 'retained.max' + , 'routes.count' + , 'routes.max' + , 'subscriptions.shared.count' + , 'subscriptions.shared.max' + ], + CountStats = nodes_info_count([ + begin + Stats = get_stats(Node), + delete_keys(Stats, GlobalStatsKeys) + end || Node <- ekka_mnesia:running_nodes()]), + GlobalStats = maps:with(GlobalStatsKeys, maps:from_list(get_stats(node()))), + maps:merge(CountStats, GlobalStats). + +delete_keys(List, []) -> + List; +delete_keys(List, [Key | Keys]) -> + delete_keys(proplists:delete(Key, List), Keys). get_stats(Node) when Node =:= node() -> emqx_stats:getstats(); get_stats(Node) -> rpc_call(Node, get_stats, [Node]). +nodes_info_count(PropList) -> + NodeCount = + fun({Key, Value}, Result) -> + Count = maps:get(Key, Result, 0), + Result#{Key => Count + Value} + end, + AllCount = + fun(StatsMap, Result) -> + lists:foldl(NodeCount, Result, StatsMap) + end, + lists:foldl(AllCount, #{}, PropList). + %%-------------------------------------------------------------------- %% Clients %%-------------------------------------------------------------------- @@ -223,39 +268,56 @@ kickout_client(Node, ClientId) when Node =:= node() -> kickout_client(Node, ClientId) -> rpc_call(Node, kickout_client, [Node, ClientId]). -list_acl_cache(ClientId) -> - call_client(ClientId, list_acl_cache). +list_authz_cache(ClientId) -> + call_client(ClientId, list_authz_cache). -clean_acl_cache(ClientId) -> - Results = [clean_acl_cache(Node, ClientId) || Node <- ekka_mnesia:running_nodes()], +list_client_subscriptions(ClientId) -> + Results = [client_subscriptions(Node, ClientId) || Node <- ekka_mnesia:running_nodes()], + Expected = lists:filter(fun({error, _}) -> false; + ([]) -> false; + (_) -> true + end, Results), + case Expected of + [] -> []; + [Result|_] -> Result + end. + +client_subscriptions(Node, ClientId) when Node =:= node() -> + emqx_broker:subscriptions(ClientId); + +client_subscriptions(Node, ClientId) -> + rpc_call(Node, client_subscriptions, [Node, ClientId]). + +clean_authz_cache(ClientId) -> + Results = [clean_authz_cache(Node, ClientId) || Node <- ekka_mnesia:running_nodes()], case lists:any(fun(Item) -> Item =:= ok end, Results) of true -> ok; false -> lists:last(Results) end. -clean_acl_cache(Node, ClientId) when Node =:= node() -> +clean_authz_cache(Node, ClientId) when Node =:= node() -> case emqx_cm:lookup_channels(ClientId) of [] -> {error, not_found}; Pids when is_list(Pids) -> - erlang:send(lists:last(Pids), clean_acl_cache), + erlang:send(lists:last(Pids), clean_authz_cache), ok end; -clean_acl_cache(Node, ClientId) -> - rpc_call(Node, clean_acl_cache, [Node, ClientId]). +clean_authz_cache(Node, ClientId) -> + rpc_call(Node, clean_authz_cache, [Node, ClientId]). -clean_acl_cache_all() -> - Results = [{Node, clean_acl_cache_all(Node)} || Node <- ekka_mnesia:running_nodes()], +clean_authz_cache_all() -> + Results = [{Node, clean_authz_cache_all(Node)} || Node <- ekka_mnesia:running_nodes()], case lists:filter(fun({_Node, Item}) -> Item =/= ok end, Results) of [] -> ok; BadNodes -> {error, BadNodes} end. -clean_acl_cache_all(Node) when Node =:= node() -> - emqx_acl_cache:drain_cache(); +clean_authz_cache_all(Node) when Node =:= node() -> + emqx_authz_cache:drain_cache(); -clean_acl_cache_all(Node) -> - rpc_call(Node, clean_acl_cache_all, [Node]). +clean_authz_cache_all(Node) -> + rpc_call(Node, clean_authz_cache_all, [Node]). set_ratelimit_policy(ClientId, Policy) -> call_client(ClientId, {ratelimit, Policy}). @@ -411,36 +473,39 @@ reload_plugin(Node, Plugin) -> %%-------------------------------------------------------------------- list_listeners() -> - [{Node, list_listeners(Node)} || Node <- ekka_mnesia:running_nodes()]. + lists:append([list_listeners(Node) || Node <- ekka_mnesia:running_nodes()]). + +list_listeners(Node, Identifier) -> + listener_id_filter(Identifier, list_listeners(Node)). list_listeners(Node) when Node =:= node() -> - Tcp = lists:map(fun({{Protocol, ListenOn}, _Pid}) -> - #{protocol => Protocol, - listen_on => ListenOn, - identifier => emqx_listeners:find_id_by_listen_on(ListenOn), - acceptors => esockd:get_acceptors({Protocol, ListenOn}), - max_conns => esockd:get_max_connections({Protocol, ListenOn}), - current_conns => esockd:get_current_connections({Protocol, ListenOn}), - shutdown_count => esockd:get_shutdown_count({Protocol, ListenOn})} - end, esockd:listeners()), - Http = lists:map(fun({Protocol, Opts}) -> - #{protocol => Protocol, - listen_on => proplists:get_value(port, Opts), - acceptors => maps:get(num_acceptors, proplists:get_value(transport_options, Opts, #{}), 0), - max_conns => proplists:get_value(max_connections, Opts), - current_conns => proplists:get_value(all_connections, Opts), - shutdown_count => []} - end, ranch:info()), - Tcp ++ Http; + [{Id, maps:put(node, Node, Conf)} || {Id, Conf} <- emqx_listeners:list()]; list_listeners(Node) -> rpc_call(Node, list_listeners, [Node]). -restart_listener(Node, Identifier) when Node =:= node() -> - emqx_listeners:restart_listener(Identifier); +list_listeners_by_id(Identifier) -> + listener_id_filter(Identifier, list_listeners()). -restart_listener(Node, Identifier) -> - rpc_call(Node, restart_listener, [Node, Identifier]). +get_listener(Node, Identifier) -> + case listener_id_filter(Identifier, list_listeners(Node)) of + [] -> + {error, not_found}; + [Listener] -> + Listener + end. + +listener_id_filter(Identifier, Listeners) -> + Filter = + fun({Id, _}) -> Id =:= Identifier end, + lists:filter(Filter, Listeners). + +-spec manage_listener(Operation :: start_listener|stop_listener|restart_listener, Param :: map()) -> + ok | {error, Reason :: term()}. +manage_listener(Operation, #{identifier := Identifier, node := Node}) when Node =:= node()-> + erlang:apply(emqx_listeners, Operation, [Identifier]); +manage_listener(Operation, Param = #{node := Node}) -> + rpc_call(Node, restart_listener, [Operation, Param]). %%-------------------------------------------------------------------- %% Get Alarms @@ -501,7 +566,7 @@ item(route, {Topic, Node}) -> #{topic => Topic, node => Node}. %%-------------------------------------------------------------------- -%% Internel Functions. +%% Internal Functions. %%-------------------------------------------------------------------- rpc_call(Node, Fun, Args) -> @@ -525,7 +590,7 @@ check_row_limit([Tab|Tables], Limit) -> end. max_row_limit() -> - application:get_env(?APP, max_row_limit, ?MAX_ROW_LIMIT). + emqx_config:get([?APP, max_row_limit], ?MAX_ROW_LIMIT). table_size(Tab) -> ets:info(Tab, size). diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index e068c5384..fbf926540 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -65,10 +65,10 @@ count(Table, Nodes) -> lists:sum([rpc_call(Node, ets, info, [Table, size], 5000) || Node <- Nodes]). page(Params) -> - binary_to_integer(proplists:get_value(<<"_page">>, Params, <<"1">>)). + binary_to_integer(proplists:get_value(<<"page">>, Params, <<"1">>)). limit(Params) -> - case proplists:get_value(<<"_limit">>, Params) of + case proplists:get_value(<<"limit">>, Params) of undefined -> emqx_mgmt:max_row_limit(); Size -> binary_to_integer(Size) end. @@ -204,7 +204,7 @@ params2qs(Params, QsSchema) -> {length(Qs) + length(Fuzzy), {Qs, Fuzzy}}. %%-------------------------------------------------------------------- -%% Intenal funcs +%% Internal funcs pick_params_to_qs([], _, Acc1, Acc2) -> NAcc2 = [E || E <- Acc2, not lists:keymember(element(1, E), 1, Acc1)], @@ -215,12 +215,12 @@ pick_params_to_qs([{Key, Value}|Params], QsKits, Acc1, Acc2) -> undefined -> pick_params_to_qs(Params, QsKits, Acc1, Acc2); Type -> case Key of - <> - when Prefix =:= <<"_gte_">>; - Prefix =:= <<"_lte_">> -> + <> + when Prefix =:= <<"gte_">>; + Prefix =:= <<"lte_">> -> OpposeKey = case Prefix of - <<"_gte_">> -> <<"_lte_", NKey/binary>>; - <<"_lte_">> -> <<"_gte_", NKey/binary>> + <<"gte_">> -> <<"lte_", NKey/binary>>; + <<"lte_">> -> <<"gte_", NKey/binary>> end, case lists:keytake(OpposeKey, 1, Params) of false -> @@ -252,20 +252,20 @@ qs(K, Value0, Type) -> throw({bad_value_type, {K, Type, Value0}}) end. -qs(<<"_gte_", Key/binary>>, Value) -> +qs(<<"gte_", Key/binary>>, Value) -> {binary_to_existing_atom(Key, utf8), '>=', Value}; -qs(<<"_lte_", Key/binary>>, Value) -> +qs(<<"lte_", Key/binary>>, Value) -> {binary_to_existing_atom(Key, utf8), '=<', Value}; -qs(<<"_like_", Key/binary>>, Value) -> +qs(<<"like_", Key/binary>>, Value) -> {binary_to_existing_atom(Key, utf8), like, Value}; -qs(<<"_match_", Key/binary>>, Value) -> +qs(<<"match_", Key/binary>>, Value) -> {binary_to_existing_atom(Key, utf8), match, Value}; qs(Key, Value) -> {binary_to_existing_atom(Key, utf8), '=:=', Value}. -is_fuzzy_key(<<"_like_", _/binary>>) -> +is_fuzzy_key(<<"like_", _/binary>>) -> true; -is_fuzzy_key(<<"_match_", _/binary>>) -> +is_fuzzy_key(<<"match_", _/binary>>) -> true; is_fuzzy_key(_) -> false. @@ -317,18 +317,18 @@ params2qs_test() -> {<<"int">>, integer}, {<<"atom">>, atom}, {<<"ts">>, timestamp}, - {<<"_gte_range">>, integer}, - {<<"_lte_range">>, integer}, - {<<"_like_fuzzy">>, binary}, - {<<"_match_topic">>, binary}], + {<<"gte_range">>, integer}, + {<<"lte_range">>, integer}, + {<<"like_fuzzy">>, binary}, + {<<"match_topic">>, binary}], Params = [{<<"str">>, <<"abc">>}, {<<"int">>, <<"123">>}, {<<"atom">>, <<"connected">>}, {<<"ts">>, <<"156000">>}, - {<<"_gte_range">>, <<"1">>}, - {<<"_lte_range">>, <<"5">>}, - {<<"_like_fuzzy">>, <<"user">>}, - {<<"_match_topic">>, <<"t/#">>}], + {<<"gte_range">>, <<"1">>}, + {<<"lte_range">>, <<"5">>}, + {<<"like_fuzzy">>, <<"user">>}, + {<<"match_topic">>, <<"t/#">>}], ExpectedQs = [{str, '=:=', <<"abc">>}, {int, '=:=', 123}, {atom, '=:=', connected}, diff --git a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl index d8a0f25dc..36a0f3a5b 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl @@ -16,122 +16,119 @@ -module(emqx_mgmt_api_alarms). --include("emqx_mgmt.hrl"). +-behaviour(minirest_api). --include_lib("emqx/include/emqx.hrl"). +-export([api_spec/0]). --rest_api(#{name => list_all_alarms, - method => 'GET', - path => "/alarms", - func => list, - descr => "List all alarms in the cluster"}). +-export([alarms/2]). --rest_api(#{name => list_node_alarms, - method => 'GET', - path => "nodes/:atom:node/alarms", - func => list, - descr => "List all alarms on a node"}). +-export([ query_activated/3 + , query_deactivated/3]). +%% notice: from emqx_alarms +-define(ACTIVATED_ALARM, emqx_activated_alarm). +-define(DEACTIVATED_ALARM, emqx_deactivated_alarm). --rest_api(#{name => list_all_activated_alarms, - method => 'GET', - path => "/alarms/activated", - func => list_activated, - descr => "List all activated alarm in the cluster"}). +api_spec() -> + {[alarms_api()], [alarm_schema()]}. --rest_api(#{name => list_node_activated_alarms, - method => 'GET', - path => "nodes/:atom:node/alarms/activated", - func => list_activated, - descr => "List all activated alarm on a node"}). +alarm_schema() -> + #{ + alarm => #{ + type => object, + properties => #{ + node => #{ + type => string, + description => <<"Alarm in node">>}, + name => #{ + type => string, + description => <<"Alarm name">>}, + message => #{ + type => string, + description => <<"Alarm readable information">>}, + details => #{ + type => object, + description => <<"Alarm detail">>}, + duration => #{ + type => integer, + description => <<"Alarms duration time; UNIX time stamp">>} + } + } + }. --rest_api(#{name => list_all_deactivated_alarms, - method => 'GET', - path => "/alarms/deactivated", - func => list_deactivated, - descr => "List all deactivated alarm in the cluster"}). +alarms_api() -> + Metadata = #{ + get => #{ + description => <<"EMQ X alarms">>, + parameters => [#{ + name => activated, + in => query, + description => <<"All alarms, if not specified">>, + required => false, + schema => #{type => boolean, default => true} + }], + responses => #{ + <<"200">> => + emqx_mgmt_util:response_array_schema(<<"List all alarms">>, alarm)}}, + delete => #{ + description => <<"Remove all deactivated alarms">>, + responses => #{ + <<"200">> => + emqx_mgmt_util:response_schema(<<"Remove all deactivated alarms ok">>)}}}, + {"/alarms", Metadata, alarms}. --rest_api(#{name => list_node_deactivated_alarms, - method => 'GET', - path => "nodes/:atom:node/alarms/deactivated", - func => list_deactivated, - descr => "List all deactivated alarm on a node"}). +%%%============================================================================================== +%% parameters trans +alarms(get, Request) -> + case proplists:get_value(<<"activated">>, cowboy_req:parse_qs(Request), undefined) of + undefined -> + list(#{activated => undefined}); + <<"true">> -> + list(#{activated => true}); + <<"false">> -> + list(#{activated => false}) + end; --rest_api(#{name => deactivate_alarm, - method => 'POST', - path => "/alarms/deactivated", - func => deactivate, - descr => "Delete the special alarm on a node"}). +alarms(delete, _Request) -> + delete(). --rest_api(#{name => delete_all_deactivated_alarms, - method => 'DELETE', - path => "/alarms/deactivated", - func => delete_deactivated, - descr => "Delete all deactivated alarm in the cluster"}). +%%%============================================================================================== +%% api apply +list(#{activated := true}) -> + do_list(activated); +list(#{activated := false}) -> + do_list(deactivated); +list(#{activated := undefined}) -> + do_list(activated). --rest_api(#{name => delete_node_deactivated_alarms, - method => 'DELETE', - path => "nodes/:atom:node/alarms/deactivated", - func => delete_deactivated, - descr => "Delete all deactivated alarm on a node"}). - --export([ list/2 - , deactivate/2 - , list_activated/2 - , list_deactivated/2 - , delete_deactivated/2 - ]). - -list(Bindings, _Params) when map_size(Bindings) == 0 -> - {ok, #{code => ?SUCCESS, - data => [#{node => Node, alarms => Alarms} || {Node, Alarms} <- emqx_mgmt:get_alarms(all)]}}; - -list(#{node := Node}, _Params) -> - {ok, #{code => ?SUCCESS, - data => emqx_mgmt:get_alarms(Node, all)}}. - -list_activated(Bindings, _Params) when map_size(Bindings) == 0 -> - {ok, #{code => ?SUCCESS, - data => [#{node => Node, alarms => Alarms} || {Node, Alarms} <- emqx_mgmt:get_alarms(activated)]}}; - -list_activated(#{node := Node}, _Params) -> - {ok, #{code => ?SUCCESS, - data => emqx_mgmt:get_alarms(Node, activated)}}. - -list_deactivated(Bindings, _Params) when map_size(Bindings) == 0 -> - {ok, #{code => ?SUCCESS, - data => [#{node => Node, alarms => Alarms} || {Node, Alarms} <- emqx_mgmt:get_alarms(deactivated)]}}; - -list_deactivated(#{node := Node}, _Params) -> - {ok, #{code => ?SUCCESS, - data => emqx_mgmt:get_alarms(Node, deactivated)}}. - -deactivate(_Bindings, Params) -> - Node = get_node(Params), - Name = get_name(Params), - do_deactivate(Node, Name). - -delete_deactivated(Bindings, _Params) when map_size(Bindings) == 0 -> +delete() -> _ = emqx_mgmt:delete_all_deactivated_alarms(), - {ok, #{code => ?SUCCESS}}; + {200}. -delete_deactivated(#{node := Node}, _Params) -> - emqx_mgmt:delete_all_deactivated_alarms(Node), - {ok, #{code => ?SUCCESS}}. +%%%============================================================================================== +%% internal +do_list(Type) -> + {Table, Function} = + case Type of + activated -> + {?ACTIVATED_ALARM, query_activated}; + deactivated -> + {?DEACTIVATED_ALARM, query_deactivated} + end, + Response = emqx_mgmt_api:cluster_query([], {Table, []}, {?MODULE, Function}), + {200, Response}. -get_node(Params) -> - binary_to_atom(proplists:get_value(<<"node">>, Params, undefined), utf8). +query_activated(_, Start, Limit) -> + query(?ACTIVATED_ALARM, Start, Limit). -get_name(Params) -> - binary_to_atom(proplists:get_value(<<"name">>, Params, undefined), utf8). +query_deactivated(_, Start, Limit) -> + query(?DEACTIVATED_ALARM, Start, Limit). -do_deactivate(undefined, _) -> - minirest:return({error, missing_param}); -do_deactivate(_, undefined) -> - minirest:return({error, missing_param}); -do_deactivate(Node, Name) -> - case emqx_mgmt:deactivate(Node, Name) of - ok -> - minirest:return(); - {error, Reason} -> - minirest:return({error, Reason}) - end. +query(Table, Start, Limit) -> + Ms = [{'$1',[],['$1']}], + emqx_mgmt_api:select_table(Table, Ms, Start, Limit, fun format_alarm/1). + +format_alarm(Alarms) when is_list(Alarms) -> + [emqx_alarm:format(Alarm) || Alarm <- Alarms]; + +format_alarm(Alarm) -> + emqx_alarm:format(Alarm). diff --git a/apps/emqx_management/src/emqx_mgmt_api_apps.erl b/apps/emqx_management/src/emqx_mgmt_api_apps.erl index cca0b41f0..2a5f330c4 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_apps.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_apps.erl @@ -16,88 +16,219 @@ -module(emqx_mgmt_api_apps). --include("emqx_mgmt.hrl"). +-behaviour(minirest_api). --rest_api(#{name => add_app, - method => 'POST', - path => "/apps/", - func => add_app, - descr => "Add Application"}). +-export([api_spec/0]). --rest_api(#{name => del_app, - method => 'DELETE', - path => "/apps/:bin:appid", - func => del_app, - descr => "Delete Application"}). --rest_api(#{name => list_apps, - method => 'GET', - path => "/apps/", - func => list_apps, - descr => "List Applications"}). +-export([ apps/2 + , app/2]). --rest_api(#{name => lookup_app, - method => 'GET', - path => "/apps/:bin:appid", - func => lookup_app, - descr => "Lookup Application"}). +-define(BAD_APP_ID, 'BAD_APP_ID'). +-define(APP_ID_NOT_FOUND, <<"{\"code\": \"BAD_APP_ID\", \"reason\": \"App id not found\"}">>). --rest_api(#{name => update_app, - method => 'PUT', - path => "/apps/:bin:appid", - func => update_app, - descr => "Update Application"}). +api_spec() -> + { + [apps_api(), app_api()], + [app_schema(), app_secret_schema()] + }. --export([ add_app/2 - , del_app/2 - , list_apps/2 - , lookup_app/2 - , update_app/2 - ]). +app_schema() -> + #{app => #{ + type => object, + properties => app_properties()}}. -add_app(_Bindings, Params) -> - AppId = proplists:get_value(<<"app_id">>, Params), - Name = proplists:get_value(<<"name">>, Params), - Secret = proplists:get_value(<<"secret">>, Params), - Desc = proplists:get_value(<<"desc">>, Params), - Status = proplists:get_value(<<"status">>, Params), - Expired = proplists:get_value(<<"expired">>, Params), - case emqx_mgmt_auth:add_app(AppId, Name, Secret, Desc, Status, Expired) of - {ok, AppSecret} -> minirest:return({ok, #{secret => AppSecret}}); - {error, Reason} -> minirest:return({error, Reason}) +app_properties() -> + #{ + app_id => #{ + type => string, + description => <<"App ID">>}, + secret => #{ + type => string, + description => <<"App Secret">>}, + name => #{ + type => string, + description => <<"Dsiplay name">>}, + desc => #{ + type => string, + description => <<"App description">>}, + status => #{ + type => boolean, + description => <<"Enable or disable">>}, + expired => #{ + type => integer, + description => <<"Expired time">>} + }. + +app_secret_schema() -> + #{app_secret => #{ + type => object, + properties => #{ + secret => #{type => string}}}}. + +%% not export schema +app_without_secret_schema() -> + #{ + type => object, + properties => maps:without([secret], app_properties()) + }. + +apps_api() -> + Metadata = #{ + get => #{ + description => <<"List EMQ X apps">>, + responses => #{ + <<"200">> => + emqx_mgmt_util:response_array_schema(<<"All apps">>, + app_without_secret_schema())}}, + post => #{ + description => <<"EMQ X create apps">>, + 'requestBody' => emqx_mgmt_util:request_body_schema(<<"app">>), + responses => #{ + <<"200">> => + emqx_mgmt_util:response_schema(<<"Create apps">>, app_secret), + <<"400">> => + emqx_mgmt_util:response_error_schema(<<"App ID already exist">>, [?BAD_APP_ID])}}}, + {"/apps", Metadata, apps}. + +app_api() -> + Metadata = #{ + get => #{ + description => <<"EMQ X apps">>, + parameters => [#{ + name => app_id, + in => path, + required => true, + schema => #{type => string}}], + responses => #{ + <<"404">> => + emqx_mgmt_util:response_error_schema(<<"App id not found">>), + <<"200">> => + emqx_mgmt_util:response_schema(<<"Get App">>, app_without_secret_schema())}}, + delete => #{ + description => <<"EMQ X apps">>, + parameters => [#{ + name => app_id, + in => path, + required => true, + schema => #{type => string} + }], + responses => #{ + <<"200">> => emqx_mgmt_util:response_schema(<<"Remove app ok">>)}}, + put => #{ + description => <<"EMQ X update apps">>, + parameters => [#{ + name => app_id, + in => path, + required => true, + schema => #{type => string} + }], + 'requestBody' => emqx_mgmt_util:request_body_schema(app_without_secret_schema()), + responses => #{ + <<"404">> => + emqx_mgmt_util:response_error_schema(<<"App id not found">>, [?BAD_APP_ID]), + <<"200">> => + emqx_mgmt_util:response_schema(<<"Update ok">>, app_without_secret_schema())}}}, + {"/apps/:app_id", Metadata, app}. + +%%%============================================================================================== +%% parameters trans +apps(get, _Request) -> + list(#{}); + +apps(post, Request) -> + {ok, Body, _} = cowboy_req:read_body(Request), + Data = emqx_json:decode(Body, [return_maps]), + Parameters = #{ + app_id => maps:get(<<"app_id">>, Data), + name => maps:get(<<"name">>, Data), + secret => maps:get(<<"secret">>, Data), + desc => maps:get(<<"desc">>, Data), + status => maps:get(<<"status">>, Data), + expired => maps:get(<<"expired">>, Data, undefined) + }, + create(Parameters). + +app(get, Request) -> + AppID = cowboy_req:binding(app_id, Request), + lookup(#{app_id => AppID}); + +app(delete, Request) -> + AppID = cowboy_req:binding(app_id, Request), + delete(#{app_id => AppID}); + +app(put, Request) -> + AppID = cowboy_req:binding(app_id, Request), + {ok, Body, _} = cowboy_req:read_body(Request), + Data = emqx_json:decode(Body, [return_maps]), + Parameters = #{ + app_id => AppID, + name => maps:get(<<"name">>, Data), + desc => maps:get(<<"desc">>, Data), + status => maps:get(<<"status">>, Data), + expired => maps:get(<<"expired">>, Data, undefined) + }, + update(Parameters). + + +%%%============================================================================================== +%% api apply +list(_) -> + {200, [format_without_app_secret(Apps) || Apps <- emqx_mgmt_auth:list_apps()]}. + +create(#{app_id := AppID, name := Name, secret := Secret, + desc := Desc, status := Status, expired := Expired}) -> + case emqx_mgmt_auth:add_app(AppID, Name, Secret, Desc, Status, Expired) of + {ok, AppSecret} -> + {200, #{secret => AppSecret}}; + {error, alread_existed} -> + Message = list_to_binary(io_lib:format("appid ~p already existed", [AppID])), + {400, #{code => 'BAD_APP_ID', message => Message}}; + {error, Reason} -> + Response = #{code => 'UNKNOW_ERROR', + message => list_to_binary(io_lib:format("~p", [Reason]))}, + {500, Response} end. -del_app(#{appid := AppId}, _Params) -> - case emqx_mgmt_auth:del_app(AppId) of - ok -> minirest:return(); - {error, Reason} -> minirest:return({error, Reason}) - end. - -list_apps(_Bindings, _Params) -> - minirest:return({ok, [format(Apps)|| Apps <- emqx_mgmt_auth:list_apps()]}). - -lookup_app(#{appid := AppId}, _Params) -> - case emqx_mgmt_auth:lookup_app(AppId) of - {AppId, AppSecret, Name, Desc, Status, Expired} -> - minirest:return({ok, #{app_id => AppId, - secret => AppSecret, - name => Name, - desc => Desc, - status => Status, - expired => Expired}}); +lookup(#{app_id := AppID}) -> + case emqx_mgmt_auth:lookup_app(AppID) of undefined -> - minirest:return({ok, #{}}) + {404, ?APP_ID_NOT_FOUND}; + App -> + Response = format_with_app_secret(App), + {200, Response} end. -update_app(#{appid := AppId}, Params) -> - Name = proplists:get_value(<<"name">>, Params), - Desc = proplists:get_value(<<"desc">>, Params), - Status = proplists:get_value(<<"status">>, Params), - Expired = proplists:get_value(<<"expired">>, Params), - case emqx_mgmt_auth:update_app(AppId, Name, Desc, Status, Expired) of - ok -> minirest:return(); - {error, Reason} -> minirest:return({error, Reason}) +delete(#{app_id := AppID}) -> + _ = emqx_mgmt_auth:del_app(AppID), + {200}. + +update(App = #{app_id := AppID, name := Name, desc := Desc, status := Status, expired := Expired}) -> + case emqx_mgmt_auth:update_app(AppID, Name, Desc, Status, Expired) of + ok -> + {200, App}; + {error, not_found} -> + {404, ?APP_ID_NOT_FOUND}; + {error, Reason} -> + Response = #{code => 'UNKNOW_ERROR', message => list_to_binary(io_lib:format("~p", [Reason]))}, + {500, Response} end. -format({AppId, _AppSecret, Name, Desc, Status, Expired}) -> - [{app_id, AppId}, {name, Name}, {desc, Desc}, {status, Status}, {expired, Expired}]. +%%%============================================================================================== +%% format +format_without_app_secret(App) -> + format_without([secret], App). + +format_with_app_secret(App) -> + format_without([], App). + +format_without(List, {AppID, AppSecret, Name, Desc, Status, Expired}) -> + Data = #{ + app_id => AppID, + secret => AppSecret, + name => Name, + desc => Desc, + status => Status, + expired => Expired + }, + maps:without(List, Data). diff --git a/apps/emqx_management/src/emqx_mgmt_api_acl.erl b/apps/emqx_management/src/emqx_mgmt_api_authz.erl similarity index 63% rename from apps/emqx_management/src/emqx_mgmt_api_acl.erl rename to apps/emqx_management/src/emqx_mgmt_api_authz.erl index 039b4035a..6da16ff30 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_acl.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_authz.erl @@ -14,34 +14,34 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_mgmt_api_acl). +-module(emqx_mgmt_api_authz). -include("emqx_mgmt.hrl"). --rest_api(#{name => clean_acl_cache_all, +-rest_api(#{name => clean_authz_cache_all, method => 'DELETE', - path => "/acl-cache", + path => "/authz-cache", func => clean_all, - descr => "Clean acl cache on all nodes"}). + descr => "Clean authz cache on all nodes"}). --rest_api(#{name => clean_acl_cache_node, +-rest_api(#{name => clean_authz_cache_node, method => 'DELETE', - path => "nodes/:atom:node/acl-cache", + path => "nodes/:atom:node/authz-cache", func => clean_node, - descr => "Clean acl cache on specific node"}). + descr => "Clean authz cache on specific node"}). -export([ clean_all/2 , clean_node/2 ]). clean_all(_Bindings, _Params) -> - case emqx_mgmt:clean_acl_cache_all() of - ok -> minirest:return(); - {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) + case emqx_mgmt:clean_authz_cache_all() of + ok -> emqx_mgmt:return(); + {error, Reason} -> emqx_mgmt:return({error, ?ERROR1, Reason}) end. clean_node(#{node := Node}, _Params) -> - case emqx_mgmt:clean_acl_cache_all(Node) of - ok -> minirest:return(); - {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) + case emqx_mgmt:clean_authz_cache_all(Node) of + ok -> emqx_mgmt:return(); + {error, Reason} -> emqx_mgmt:return({error, ?ERROR1, Reason}) end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl index b92875d9e..bdd43b35c 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl @@ -44,7 +44,7 @@ ]). list(_Bindings, Params) -> - minirest:return({ok, emqx_mgmt_api:paginate(emqx_banned, Params, fun format/1)}). + emqx_mgmt:return({ok, emqx_mgmt_api:paginate(emqx_banned, Params, fun format/1)}). create(_Bindings, Params) -> case pipeline([fun ensure_required/1, @@ -52,9 +52,9 @@ create(_Bindings, Params) -> {ok, NParams} -> {ok, Banned} = pack_banned(NParams), ok = emqx_mgmt:create_banned(Banned), - minirest:return({ok, maps:from_list(Params)}); + emqx_mgmt:return({ok, maps:from_list(Params)}); {error, Code, Message} -> - minirest:return({error, Code, Message}) + emqx_mgmt:return({error, Code, Message}) end. delete(#{as := As, who := Who}, _) -> @@ -64,9 +64,9 @@ delete(#{as := As, who := Who}, _) -> fun validate_params/1], Params) of {ok, NParams} -> do_delete(proplists:get_value(<<"as">>, NParams), proplists:get_value(<<"who">>, NParams)), - minirest:return(); + emqx_mgmt:return(); {error, Code, Message} -> - minirest:return({error, Code, Message}) + emqx_mgmt:return({error, Code, Message}) end. pipeline([], Params) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_brokers.erl b/apps/emqx_management/src/emqx_mgmt_api_brokers.erl index bd901a3fe..836f097cb 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_brokers.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_brokers.erl @@ -35,13 +35,13 @@ ]). list(_Bindings, _Params) -> - minirest:return({ok, [Info || {_Node, Info} <- emqx_mgmt:list_brokers()]}). + emqx_mgmt:return({ok, [Info || {_Node, Info} <- emqx_mgmt:list_brokers()]}). get(#{node := Node}, _Params) -> case emqx_mgmt:lookup_broker(Node) of {error, Reason} -> - minirest:return({error, ?ERROR2, Reason}); + emqx_mgmt:return({error, ?ERROR2, Reason}); Info -> - minirest:return({ok, Info}) + emqx_mgmt:return({ok, Info}) end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 2fe6a5ccb..b99d6c6ef 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -16,308 +16,541 @@ -module(emqx_mgmt_api_clients). --include("emqx_mgmt.hrl"). +-behaviour(minirest_api). --include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx.hrl"). --define(CLIENT_QS_SCHEMA, {emqx_channel_info, - [{<<"clientid">>, binary}, - {<<"username">>, binary}, - {<<"zone">>, atom}, - {<<"ip_address">>, ip}, - {<<"conn_state">>, atom}, - {<<"clean_start">>, atom}, - {<<"proto_name">>, binary}, - {<<"proto_ver">>, integer}, - {<<"_like_clientid">>, binary}, - {<<"_like_username">>, binary}, - {<<"_gte_created_at">>, timestamp}, - {<<"_lte_created_at">>, timestamp}, - {<<"_gte_connected_at">>, timestamp}, - {<<"_lte_connected_at">>, timestamp}]}). +-include_lib("emqx/include/logger.hrl"). --rest_api(#{name => list_clients, - method => 'GET', - path => "/clients/", - func => list, - descr => "A list of clients on current node"}). +-include("emqx_mgmt.hrl"). --rest_api(#{name => list_node_clients, - method => 'GET', - path => "nodes/:atom:node/clients/", - func => list, - descr => "A list of clients on specified node"}). +%% API +-export([api_spec/0]). --rest_api(#{name => lookup_client, - method => 'GET', - path => "/clients/:bin:clientid", - func => lookup, - descr => "Lookup a client in the cluster"}). - --rest_api(#{name => lookup_node_client, - method => 'GET', - path => "nodes/:atom:node/clients/:bin:clientid", - func => lookup, - descr => "Lookup a client on the node"}). - --rest_api(#{name => lookup_client_via_username, - method => 'GET', - path => "/clients/username/:bin:username", - func => lookup, - descr => "Lookup a client via username in the cluster" - }). - --rest_api(#{name => lookup_node_client_via_username, - method => 'GET', - path => "/nodes/:atom:node/clients/username/:bin:username", - func => lookup, - descr => "Lookup a client via username on the node " - }). - --rest_api(#{name => kickout_client, - method => 'DELETE', - path => "/clients/:bin:clientid", - func => kickout, - descr => "Kick out the client in the cluster"}). - --rest_api(#{name => clean_acl_cache, - method => 'DELETE', - path => "/clients/:bin:clientid/acl_cache", - func => clean_acl_cache, - descr => "Clear the ACL cache of a specified client in the cluster"}). - --rest_api(#{name => list_acl_cache, - method => 'GET', - path => "/clients/:bin:clientid/acl_cache", - func => list_acl_cache, - descr => "List the ACL cache of a specified client in the cluster"}). - --rest_api(#{name => set_ratelimit_policy, - method => 'POST', - path => "/clients/:bin:clientid/ratelimit", - func => set_ratelimit_policy, - descr => "Set the client ratelimit policy"}). - --rest_api(#{name => clean_ratelimit, - method => 'DELETE', - path => "/clients/:bin:clientid/ratelimit", - func => clean_ratelimit, - descr => "Clear the ratelimit policy"}). - --rest_api(#{name => set_quota_policy, - method => 'POST', - path => "/clients/:bin:clientid/quota", - func => set_quota_policy, - descr => "Set the client quota policy"}). - --rest_api(#{name => clean_quota, - method => 'DELETE', - path => "/clients/:bin:clientid/quota", - func => clean_quota, - descr => "Clear the quota policy"}). - --import(emqx_mgmt_util, [ ntoa/1 - , strftime/1 - ]). - --export([ list/2 - , lookup/2 - , kickout/2 - , clean_acl_cache/2 - , list_acl_cache/2 - , set_ratelimit_policy/2 - , set_quota_policy/2 - , clean_ratelimit/2 - , clean_quota/2 - ]). +-export([ clients/2 + , client/2 + , subscriptions/2 + , authz_cache/2 + , subscribe/2 + , subscribe_batch/2]). -export([ query/3 - , format_channel_info/1 - ]). + , format_channel_info/1]). + +%% for batch operation +-export([do_subscribe/3]). + +-define(CLIENT_QS_SCHEMA, {emqx_channel_info, + [ {<<"clientid">>, binary} + , {<<"username">>, binary} + , {<<"zone">>, atom} + , {<<"ip_address">>, ip} + , {<<"conn_state">>, atom} + , {<<"clean_start">>, atom} + , {<<"proto_name">>, binary} + , {<<"proto_ver">>, integer} + , {<<"like_clientid">>, binary} + , {<<"like_username">>, binary} + , {<<"gte_created_at">>, timestamp} + , {<<"lte_created_at">>, timestamp} + , {<<"gte_connected_at">>, timestamp} + , {<<"lte_connected_at">>, timestamp}]}). -define(query_fun, {?MODULE, query}). -define(format_fun, {?MODULE, format_channel_info}). -list(Bindings, Params) when map_size(Bindings) == 0 -> - fence(fun() -> - emqx_mgmt_api:cluster_query(Params, ?CLIENT_QS_SCHEMA, ?query_fun) - end); +-define(CLIENT_ID_NOT_FOUND, + <<"{\"code\": \"RESOURCE_NOT_FOUND\", \"reason\": \"Client id not found\"}">>). -list(#{node := Node}, Params) when Node =:= node() -> - fence(fun() -> - emqx_mgmt_api:node_query(Node, Params, ?CLIENT_QS_SCHEMA, ?query_fun) - end); +api_spec() -> + {apis(), schemas()}. -list(Bindings = #{node := Node}, Params) -> - case rpc:call(Node, ?MODULE, list, [Bindings, Params]) of - {badrpc, Reason} -> minirest:return({error, ?ERROR1, Reason}); - Res -> Res +apis() -> + [ clients_api() + , client_api() + , clients_authz_cache_api() + , clients_subscriptions_api() + , subscribe_api()]. + +schemas() -> + Client = #{ + client => #{ + type => object, + properties => #{ + node => #{ + type => string, + description => <<"Name of the node to which the client is connected">>}, + clientid => #{ + type => string, + description => <<"Client identifier">>}, + username => #{ + type => string, + description => <<"User name of client when connecting">>}, + proto_name => #{ + type => string, + description => <<"Client protocol name">>}, + proto_ver => #{ + type => integer, + description => <<"Protocol version used by the client">>}, + ip_address => #{ + type => string, + description => <<"Client's IP address">>}, + is_bridge => #{ + type => boolean, + description => <<"Indicates whether the client is connectedvia bridge">>}, + connected_at => #{ + type => string, + description => <<"Client connection time">>}, + disconnected_at => #{ + type => string, + description => <<"Client offline time, This field is only valid and returned when connected is false">>}, + connected => #{ + type => boolean, + description => <<"Whether the client is connected">>}, + will_msg => #{ + type => string, + description => <<"Client will message">>}, + zone => #{ + type => string, + description => <<"Indicate the configuration group used by the client">>}, + keepalive => #{ + type => integer, + description => <<"keepalive time, with the unit of second">>}, + clean_start => #{ + type => boolean, + description => <<"Indicate whether the client is using a brand new session">>}, + expiry_interval => #{ + type => integer, + description => <<"Session expiration interval, with the unit of second">>}, + created_at => #{ + type => string, + description => <<"Session creation time">>}, + subscriptions_cnt => #{ + type => integer, + description => <<"Number of subscriptions established by this client.">>}, + subscriptions_max => #{ + type => integer, + description => <<"v4 api name [max_subscriptions] Maximum number of subscriptions allowed by this client">>}, + inflight_cnt => #{ + type => integer, + description => <<"Current length of inflight">>}, + inflight_max => #{ + type => integer, + description => <<"v4 api name [max_inflight]. Maximum length of inflight">>}, + mqueue_len => #{ + type => integer, + description => <<"Current length of message queue">>}, + mqueue_max => #{ + type => integer, + description => <<"v4 api name [max_mqueue]. Maximum length of message queue">>}, + mqueue_dropped => #{ + type => integer, + description => <<"Number of messages dropped by the message queue due to exceeding the length">>}, + awaiting_rel_cnt => #{ + type => integer, + description => <<"v4 api name [awaiting_rel] Number of awaiting PUBREC packet">>}, + awaiting_rel_max => #{ + type => integer, + description => <<"v4 api name [max_awaiting_rel]. Maximum allowed number of awaiting PUBREC packet">>}, + recv_oct => #{ + type => integer, + description => <<"Number of bytes received by EMQ X Broker (the same below)">>}, + recv_cnt => #{ + type => integer, + description => <<"Number of TCP packets received">>}, + recv_pkt => #{ + type => integer, + description => <<"Number of MQTT packets received">>}, + recv_msg => #{ + type => integer, + description => <<"Number of PUBLISH packets received">>}, + send_oct => #{ + type => integer, + description => <<"Number of bytes sent">>}, + send_cnt => #{ + type => integer, + description => <<"Number of TCP packets sent">>}, + send_pkt => #{ + type => integer, + description => <<"Number of MQTT packets sent">>}, + send_msg => #{ + type => integer, + description => <<"Number of PUBLISH packets sent">>}, + mailbox_len => #{ + type => integer, + description => <<"Process mailbox size">>}, + heap_size => #{ + type => integer, + description => <<"Process heap size with the unit of byte">> + }, + reductions => #{ + type => integer, + description => <<"Erlang reduction">>} + } + } + }, + AuthzCache = #{ + authz_cache => #{ + type => object, + properties => #{ + topic => #{ + type => string, + description => <<"Topic name">>}, + access => #{ + type => string, + enum => [<<"subscribe">>, <<"publish">>], + description => <<"Access type">>}, + result => #{ + type => string, + enum => [<<"allow">>, <<"deny">>], + default => <<"allow">>, + description => <<"Allow or deny">>}, + updated_time => #{ + type => integer, + description => <<"Update time">>} + } + } + }, + Subscription = #{ + subscription => #{ + type => object, + properties => #{ + topic => #{ + type => string}, + qos => #{ + type => integer, + enum => [0,1,2]}}} + }, + [Client, AuthzCache, Subscription]. + +clients_api() -> + Metadata = #{ + get => #{ + description => <<"List clients">>, + responses => #{ + <<"200">> => emqx_mgmt_util:response_array_schema(<<"List clients 200 OK">>, client)}}}, + {"/clients", Metadata, clients}. + +client_api() -> + Metadata = #{ + get => #{ + description => <<"Get clients info by client ID">>, + parameters => [#{ + name => clientid, + in => path, + schema => #{type => string}, + required => true + }], + responses => #{ + <<"404">> => emqx_mgmt_util:response_error_schema(<<"Client id not found">>), + <<"200">> => emqx_mgmt_util:response_schema(<<"List clients 200 OK">>, client)}}, + delete => #{ + description => <<"Kick out client by client ID">>, + parameters => [#{ + name => clientid, + in => path, + schema => #{type => string}, + required => true + }], + responses => #{ + <<"404">> => emqx_mgmt_util:response_error_schema(<<"Client id not found">>), + <<"200">> => emqx_mgmt_util:response_schema(<<"List clients 200 OK">>, client)}}}, + {"/clients/:clientid", Metadata, client}. + +clients_authz_cache_api() -> + Metadata = #{ + get => #{ + description => <<"Get client authz cache">>, + parameters => [#{ + name => clientid, + in => path, + schema => #{type => string}, + required => true + }], + responses => #{ + <<"404">> => emqx_mgmt_util:response_error_schema(<<"Client id not found">>), + <<"200">> => emqx_mgmt_util:response_schema(<<"Get client authz cache">>, <<"authz_cache">>)}}, + delete => #{ + description => <<"Clean client authz cache">>, + parameters => [#{ + name => clientid, + in => path, + schema => #{type => string}, + required => true + }], + responses => #{ + <<"404">> => emqx_mgmt_util:response_error_schema(<<"Client id not found">>), + <<"200">> => emqx_mgmt_util:response_schema(<<"Delete clients 200 OK">>)}}}, + {"/clients/:clientid/authz_cache", Metadata, authz_cache}. + +clients_subscriptions_api() -> + Metadata = #{ + get => #{ + description => <<"Get client subscriptions">>, + parameters => [#{ + name => clientid, + in => path, + schema => #{type => string}, + required => true + }], + responses => #{ + <<"200">> => emqx_mgmt_util:response_array_schema(<<"Get client subscriptions">>, subscription)}} + }, + {"/clients/:clientid/subscriptions", Metadata, subscriptions}. + +subscribe_api() -> + Metadata = #{ + post => #{ + description => <<"Subscribe">>, + parameters => [#{ + name => clientid, + in => path, + schema => #{type => string}, + required => true + }], + 'requestBody' => emqx_mgmt_util:request_body_schema(#{ + type => object, + properties => #{ + topic => #{ + type => string, + description => <<"Topic">>}, + qos => #{ + type => integer, + enum => [0, 1, 2], + example => 0, + description => <<"QoS">>}}}), + responses => #{ + <<"404">> => emqx_mgmt_util:response_error_schema(<<"Client id not found">>), + <<"200">> => emqx_mgmt_util:response_schema(<<"Subscribe ok">>)}}, + delete => #{ + description => <<"Unsubscribe">>, + parameters => [ + #{ + name => clientid, + in => path, + schema => #{type => string}, + required => true + }, + #{ + name => topic, + in => query, + schema => #{type => string}, + required => true + } + ], + responses => #{ + <<"404">> => emqx_mgmt_util:response_error_schema(<<"Client id not found">>), + <<"200">> => emqx_mgmt_util:response_schema(<<"Unsubscribe ok">>)}}}, + {"/clients/:clientid/subscribe", Metadata, subscribe}. + +%%%============================================================================================== +%% parameters trans +clients(get, _Request) -> + list(#{}). + +client(get, Request) -> + ClientID = cowboy_req:binding(clientid, Request), + lookup(#{clientid => ClientID}); + +client(delete, Request) -> + ClientID = cowboy_req:binding(clientid, Request), + kickout(#{clientid => ClientID}). + +authz_cache(get, Request) -> + ClientID = cowboy_req:binding(clientid, Request), + get_authz_cache(#{clientid => ClientID}); + +authz_cache(delete, Request) -> + ClientID = cowboy_req:binding(clientid, Request), + clean_authz_cache(#{clientid => ClientID}). + +subscribe(post, Request) -> + ClientID = cowboy_req:binding(clientid, Request), + {ok, Body, _} = cowboy_req:read_body(Request), + TopicInfo = emqx_json:decode(Body, [return_maps]), + Topic = maps:get(<<"topic">>, TopicInfo), + Qos = maps:get(<<"qos">>, TopicInfo, 0), + subscribe(#{clientid => ClientID, topic => Topic, qos => Qos}); + +subscribe(delete, Request) -> + ClientID = cowboy_req:binding(clientid, Request), + #{topic := Topic} = cowboy_req:match_qs([topic], Request), + unsubscribe(#{clientid => ClientID, topic => Topic}). + +%% TODO: batch +subscribe_batch(post, Request) -> + ClientID = cowboy_req:binding(clientid, Request), + {ok, Body, _} = cowboy_req:read_body(Request), + TopicInfos = emqx_json:decode(Body, [return_maps]), + Topics = + [begin + Topic = maps:get(<<"topic">>, TopicInfo), + Qos = maps:get(<<"qos">>, TopicInfo, 0), + #{topic => Topic, qos => Qos} + end || TopicInfo <- TopicInfos], + subscribe_batch(#{clientid => ClientID, topics => Topics}). + +subscriptions(get, Request) -> + ClientID = cowboy_req:binding(clientid, Request), + Subs0 = emqx_mgmt:list_client_subscriptions(ClientID), + Subs = lists:map(fun({Topic, SubOpts}) -> + #{topic => Topic, qos => maps:get(qos, SubOpts)} + end, Subs0), + {200, Subs}. + +%%%============================================================================================== +%% api apply + +list(Params) -> + Response = emqx_mgmt_api:cluster_query(maps:to_list(Params), ?CLIENT_QS_SCHEMA, ?query_fun), + {200, Response}. + +lookup(#{clientid := ClientID}) -> + case emqx_mgmt:lookup_client({clientid, ClientID}, ?format_fun) of + [] -> + {404, ?CLIENT_ID_NOT_FOUND}; + ClientInfo -> + {200, hd(ClientInfo)} end. -%% @private -fence(Func) -> - try - minirest:return({ok, Func()}) - catch - throw : {bad_value_type, {_Key, Type, Value}} -> - Reason = iolist_to_binary( - io_lib:format("Can't convert ~p to ~p type", - [Value, Type]) - ), - minirest:return({error, ?ERROR8, Reason}) +kickout(#{clientid := ClientID}) -> + emqx_mgmt:kickout_client(ClientID), + {200}. + +get_authz_cache(#{clientid := ClientID})-> + case emqx_mgmt:list_authz_cache(ClientID) of + {error, not_found} -> + {404, ?CLIENT_ID_NOT_FOUND}; + {error, Reason} -> + Message = list_to_binary(io_lib:format("~p", [Reason])), + {500, #{code => <<"UNKNOW_ERROR">>, message => Message}}; + Caches -> + Response = [format_authz_cache(Cache) || Cache <- Caches], + {200, Response} end. -lookup(#{node := Node, clientid := ClientId}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client(Node, {clientid, emqx_mgmt_util:urldecode(ClientId)}, ?format_fun)}); - -lookup(#{clientid := ClientId}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client({clientid, emqx_mgmt_util:urldecode(ClientId)}, ?format_fun)}); - -lookup(#{node := Node, username := Username}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client(Node, {username, emqx_mgmt_util:urldecode(Username)}, ?format_fun)}); - -lookup(#{username := Username}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client({username, emqx_mgmt_util:urldecode(Username)}, ?format_fun)}). - -kickout(#{clientid := ClientId}, _Params) -> - case emqx_mgmt:kickout_client(emqx_mgmt_util:urldecode(ClientId)) of - ok -> minirest:return(); - {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); - {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) +clean_authz_cache(#{clientid := ClientID}) -> + case emqx_mgmt:clean_authz_cache(ClientID) of + ok -> + {200}; + {error, not_found} -> + {404, ?CLIENT_ID_NOT_FOUND}; + {error, Reason} -> + Message = list_to_binary(io_lib:format("~p", [Reason])), + {500, #{code => <<"UNKNOW_ERROR">>, message => Message}} end. -clean_acl_cache(#{clientid := ClientId}, _Params) -> - case emqx_mgmt:clean_acl_cache(emqx_mgmt_util:urldecode(ClientId)) of - ok -> minirest:return(); - {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); - {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) +subscribe(#{clientid := ClientID, topic := Topic, qos := Qos}) -> + case do_subscribe(ClientID, Topic, Qos) of + {error, channel_not_found} -> + {404, ?CLIENT_ID_NOT_FOUND}; + {error, Reason} -> + Message = list_to_binary(io_lib:format("~p", [Reason])), + {500, #{code => <<"UNKNOW_ERROR">>, message => Message}}; + ok -> + {200} end. -list_acl_cache(#{clientid := ClientId}, _Params) -> - case emqx_mgmt:list_acl_cache(emqx_mgmt_util:urldecode(ClientId)) of - {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); - {error, Reason} -> minirest:return({error, ?ERROR1, Reason}); - Caches -> minirest:return({ok, [format_acl_cache(Cache) || Cache <- Caches]}) +unsubscribe(#{clientid := ClientID, topic := Topic}) -> + case do_unsubscribe(ClientID, Topic) of + {error, channel_not_found} -> + {404, ?CLIENT_ID_NOT_FOUND}; + {error, Reason} -> + Message = list_to_binary(io_lib:format("~p", [Reason])), + {500, #{code => <<"UNKNOW_ERROR">>, message => Message}}; + {unsubscribe, [{Topic, #{}}]} -> + {200} end. -set_ratelimit_policy(#{clientid := ClientId}, Params) -> - P = [{conn_bytes_in, proplists:get_value(<<"conn_bytes_in">>, Params)}, - {conn_messages_in, proplists:get_value(<<"conn_messages_in">>, Params)}], - case [{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined] of - [] -> minirest:return(); - Policy -> - case emqx_mgmt:set_ratelimit_policy(emqx_mgmt_util:urldecode(ClientId), Policy) of - ok -> minirest:return(); - {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); - {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) +subscribe_batch(#{clientid := ClientID, topics := Topics}) -> + ArgList = [[ClientID, Topic, Qos]|| #{topic := Topic, qos := Qos} <- Topics], + emqx_mgmt_util:batch_operation(?MODULE, do_subscribe, ArgList). + +%%%============================================================================================== +%% internal function +format_channel_info({_, ClientInfo, ClientStats}) -> + Fun = + fun + (_Key, Value, Current) when is_map(Value) -> + maps:merge(Current, Value); + (Key, Value, Current) -> + maps:put(Key, Value, Current) + end, + StatsMap = maps:without([memory, next_pkt_id, total_heap_size], + maps:from_list(ClientStats)), + ClientInfoMap0 = maps:fold(Fun, #{}, ClientInfo), + IpAddress = peer_to_binary(maps:get(peername, ClientInfoMap0)), + Connected = maps:get(conn_state, ClientInfoMap0) =:= connected, + ClientInfoMap1 = maps:merge(StatsMap, ClientInfoMap0), + ClientInfoMap2 = maps:put(node, node(), ClientInfoMap1), + ClientInfoMap3 = maps:put(ip_address, IpAddress, ClientInfoMap2), + ClientInfoMap = maps:put(connected, Connected, ClientInfoMap3), + RemoveList = [ + auth_result + , peername + , sockname + , peerhost + , conn_state + , send_pend + , conn_props + , peercert + , sockstate + , receive_maximum + , protocol + , is_superuser + , sockport + , anonymous + , mountpoint + , socktype + , active_n + , await_rel_timeout + , conn_mod + , sockname + , retry_interval + , upgrade_qos + ], + maps:without(RemoveList, ClientInfoMap). + +peer_to_binary({Addr, Port}) -> + AddrBinary = list_to_binary(inet:ntoa(Addr)), + PortBinary = integer_to_binary(Port), + <>; +peer_to_binary(Addr) -> + list_to_binary(inet:ntoa(Addr)). + +format_authz_cache({{PubSub, Topic}, {AuthzResult, Timestamp}}) -> + #{ + access => PubSub, + topic => Topic, + result => AuthzResult, + updated_time => Timestamp + }. + +do_subscribe(ClientID, Topic0, Qos) -> + {Topic, Opts} = emqx_topic:parse(Topic0), + TopicTable = [{Topic, Opts#{qos => Qos}}], + emqx_mgmt:subscribe(ClientID, TopicTable), + case emqx_mgmt:subscribe(ClientID, TopicTable) of + {error, Reason} -> + {error, Reason}; + {subscribe, Subscriptions} -> + case proplists:is_defined(Topic, Subscriptions) of + true -> + ok; + false -> + {error, unknow_error} end end. -clean_ratelimit(#{clientid := ClientId}, _Params) -> - case emqx_mgmt:set_ratelimit_policy(emqx_mgmt_util:urldecode(ClientId), []) of - ok -> minirest:return(); - {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); - {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) +do_unsubscribe(ClientID, Topic) -> + case emqx_mgmt:unsubscribe(ClientID, Topic) of + {error, Reason} -> + {error, Reason}; + Res -> + Res end. - -set_quota_policy(#{clientid := ClientId}, Params) -> - P = [{conn_messages_routing, proplists:get_value(<<"conn_messages_routing">>, Params)}], - case [{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined] of - [] -> minirest:return(); - Policy -> - case emqx_mgmt:set_quota_policy(emqx_mgmt_util:urldecode(ClientId), Policy) of - ok -> minirest:return(); - {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); - {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) - end - end. - -clean_quota(#{clientid := ClientId}, _Params) -> - case emqx_mgmt:set_quota_policy(emqx_mgmt_util:urldecode(ClientId), []) of - ok -> minirest:return(); - {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); - {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) - end. - -%% @private -%% S = 100,1s -%% | 100KB, 1m -parse_ratelimit_str(S) when is_binary(S) -> - parse_ratelimit_str(binary_to_list(S)); -parse_ratelimit_str(S) -> - [L, D] = string:tokens(S, ", "), - Limit = case cuttlefish_bytesize:parse(L) of - Sz when is_integer(Sz) -> Sz; - {error, Reason1} -> error(Reason1) - end, - Duration = case cuttlefish_duration:parse(D, s) of - Secs when is_integer(Secs) -> Secs; - {error, Reason} -> error(Reason) - end, - {Limit, Duration}. - -%%-------------------------------------------------------------------- -%% Format - -format_channel_info({_Key, Info, Stats0}) -> - Stats = maps:from_list(Stats0), - ClientInfo = maps:get(clientinfo, Info, #{}), - ConnInfo = maps:get(conninfo, Info, #{}), - Session = case maps:get(session, Info, #{}) of - undefined -> #{}; - _Sess -> _Sess - end, - SessCreated = maps:get(created_at, Session, maps:get(connected_at, ConnInfo)), - Connected = case maps:get(conn_state, Info, connected) of - connected -> true; - _ -> false - end, - NStats = Stats#{max_subscriptions => maps:get(subscriptions_max, Stats, 0), - max_inflight => maps:get(inflight_max, Stats, 0), - max_awaiting_rel => maps:get(awaiting_rel_max, Stats, 0), - max_mqueue => maps:get(mqueue_max, Stats, 0), - inflight => maps:get(inflight_cnt, Stats, 0), - awaiting_rel => maps:get(awaiting_rel_cnt, Stats, 0)}, - format( - lists:foldl(fun(Items, Acc) -> - maps:merge(Items, Acc) - end, #{connected => Connected}, - [maps:with([ subscriptions_cnt, max_subscriptions, - inflight, max_inflight, awaiting_rel, - max_awaiting_rel, mqueue_len, mqueue_dropped, - max_mqueue, heap_size, reductions, mailbox_len, - recv_cnt, recv_msg, recv_oct, recv_pkt, send_cnt, - send_msg, send_oct, send_pkt], NStats), - maps:with([clientid, username, mountpoint, is_bridge, zone], ClientInfo), - maps:with([clean_start, keepalive, expiry_interval, proto_name, - proto_ver, peername, connected_at, disconnected_at], ConnInfo), - #{created_at => SessCreated}])). - -format(Data) when is_map(Data)-> - {IpAddr, Port} = maps:get(peername, Data), - ConnectedAt = maps:get(connected_at, Data), - CreatedAt = maps:get(created_at, Data), - Data1 = maps:without([peername], Data), - maps:merge(Data1#{node => node(), - ip_address => iolist_to_binary(ntoa(IpAddr)), - port => Port, - connected_at => iolist_to_binary(strftime(ConnectedAt div 1000)), - created_at => iolist_to_binary(strftime(CreatedAt div 1000))}, - case maps:get(disconnected_at, Data, undefined) of - undefined -> #{}; - DisconnectedAt -> #{disconnected_at => iolist_to_binary(strftime(DisconnectedAt div 1000))} - end). - -format_acl_cache({{PubSub, Topic}, {AclResult, Timestamp}}) -> - #{access => PubSub, - topic => Topic, - result => AclResult, - updated_time => Timestamp}. - -%%-------------------------------------------------------------------- +%%%============================================================================================== %% Query Functions -%%-------------------------------------------------------------------- query({Qs, []}, Start, Limit) -> Ms = qs2ms(Qs), @@ -328,37 +561,8 @@ query({Qs, Fuzzy}, Start, Limit) -> MatchFun = match_fun(Ms, Fuzzy), emqx_mgmt_api:traverse_table(emqx_channel_info, MatchFun, Start, Limit, fun format_channel_info/1). -%%-------------------------------------------------------------------- -%% Match funcs - -match_fun(Ms, Fuzzy) -> - MsC = ets:match_spec_compile(Ms), - REFuzzy = lists:map(fun({K, like, S}) -> - {ok, RE} = re:compile(S), - {K, like, RE} - end, Fuzzy), - fun(Rows) -> - case ets:match_spec_run(Rows, MsC) of - [] -> []; - Ls -> - lists:filter(fun(E) -> - run_fuzzy_match(E, REFuzzy) - end, Ls) - end - end. - -run_fuzzy_match(_, []) -> - true; -run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE}|Fuzzy]) -> - Val = case maps:get(Key, ClientInfo, "") of - undefined -> ""; - V -> V - end, - re:run(Val, RE, [{capture, none}]) == match andalso run_fuzzy_match(E, Fuzzy). - -%%-------------------------------------------------------------------- +%%%============================================================================================== %% QueryString to Match Spec - -spec qs2ms(list()) -> ets:match_spec(). qs2ms(Qs) -> {MtchHead, Conds} = qs2ms(Qs, 2, {#{}, []}), @@ -380,7 +584,7 @@ put_conds({_, Op, V}, Holder, Conds) -> [{Op, Holder, V} | Conds]; put_conds({_, Op1, V1, Op2, V2}, Holder, Conds) -> [{Op2, Holder, V2}, - {Op1, Holder, V1} | Conds]. + {Op1, Holder, V1} | Conds]. ms(clientid, X) -> #{clientinfo => #{clientid => X}}; @@ -403,51 +607,29 @@ ms(connected_at, X) -> ms(created_at, X) -> #{session => #{created_at => X}}. -%%-------------------------------------------------------------------- -%% EUnits -%%-------------------------------------------------------------------- +%%%============================================================================================== +%% Match funcs +match_fun(Ms, Fuzzy) -> + MsC = ets:match_spec_compile(Ms), + REFuzzy = lists:map(fun({K, like, S}) -> + {ok, RE} = re:compile(S), + {K, like, RE} + end, Fuzzy), + fun(Rows) -> + case ets:match_spec_run(Rows, MsC) of + [] -> []; + Ls -> + lists:filter(fun(E) -> + run_fuzzy_match(E, REFuzzy) + end, Ls) + end + end. --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). - -params2qs_test() -> - QsSchema = element(2, ?CLIENT_QS_SCHEMA), - Params = [{<<"clientid">>, <<"abc">>}, - {<<"username">>, <<"def">>}, - {<<"zone">>, <<"external">>}, - {<<"ip_address">>, <<"127.0.0.1">>}, - {<<"conn_state">>, <<"connected">>}, - {<<"clean_start">>, true}, - {<<"proto_name">>, <<"MQTT">>}, - {<<"proto_ver">>, 4}, - {<<"_gte_created_at">>, 1}, - {<<"_lte_created_at">>, 5}, - {<<"_gte_connected_at">>, 1}, - {<<"_lte_connected_at">>, 5}, - {<<"_like_clientid">>, <<"a">>}, - {<<"_like_username">>, <<"e">>} - ], - ExpectedMtchHead = - #{clientinfo => #{clientid => <<"abc">>, - username => <<"def">>, - zone => external, - peerhost => {127,0,0,1} - }, - conn_state => connected, - conninfo => #{clean_start => true, - proto_name => <<"MQTT">>, - proto_ver => 4, - connected_at => '$3'}, - session => #{created_at => '$2'}}, - ExpectedCondi = [{'>=','$2', 1}, - {'=<','$2', 5}, - {'>=','$3', 1}, - {'=<','$3', 5}], - {10, {Qs1, []}} = emqx_mgmt_api:params2qs(Params, QsSchema), - [{{'$1', MtchHead, _}, Condi, _}] = qs2ms(Qs1), - ?assertEqual(ExpectedMtchHead, MtchHead), - ?assertEqual(ExpectedCondi, Condi), - - [{{'$1', #{}, '_'}, [], ['$_']}] = qs2ms([]). - --endif. +run_fuzzy_match(_, []) -> + true; +run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE}|Fuzzy]) -> + Val = case maps:get(Key, ClientInfo, "") of + undefined -> ""; + V -> V + end, + re:run(Val, RE, [{capture, none}]) == match andalso run_fuzzy_match(E, Fuzzy). diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index 7cccbd2ac..e845d2679 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -16,61 +16,319 @@ -module(emqx_mgmt_api_listeners). --rest_api(#{name => list_listeners, - method => 'GET', - path => "/listeners/", - func => list, - descr => "A list of listeners in the cluster"}). +-behaviour(minirest_api). --rest_api(#{name => list_node_listeners, - method => 'GET', - path => "/nodes/:atom:node/listeners", - func => list, - descr => "A list of listeners on the node"}). +-export([api_spec/0]). --rest_api(#{name => restart_listener, - method => 'PUT', - path => "/listeners/:bin:identifier/restart", - func => restart, - descr => "Restart a listener in the cluster"}). +-export([ listeners/2 + , listener/2 + , node_listener/2 + , node_listeners/2 + , manage_listeners/2 + , manage_nodes_listeners/2]). --rest_api(#{name => restart_node_listener, - method => 'PUT', - path => "/nodes/:atom:node/listeners/:bin:identifier/restart", - func => restart, - descr => "Restart a listener on a node"}). +-export([format/1]). --export([list/2, restart/2]). +-include_lib("emqx/include/emqx.hrl"). -%% List listeners on a node. -list(#{node := Node}, _Params) -> - minirest:return({ok, format(emqx_mgmt:list_listeners(Node))}); +api_spec() -> + { + [ + listeners_api(), + restart_listeners_api(), + nodes_listeners_api(), + nodes_listener_api(), + manage_listeners_api(), + manage_nodes_listeners_api() + ], + [listener_schema()] + }. + +listener_schema() -> + #{ + listener => #{ + type => object, + properties => #{ + node => #{ + type => string, + description => <<"Node">>, + example => node()}, + identifier => #{ + type => string, + description => <<"Identifier">>}, + acceptors => #{ + type => integer, + description => <<"Number of Acceptor proce">>}, + max_conn => #{ + type => integer, + description => <<"Maximum number of allowed connection">>}, + type => #{ + type => string, + description => <<"Plugin decription">>}, + listen_on => #{ + type => string, + description => <<"Litening port">>}, + running => #{ + type => boolean, + description => <<"Open or close">>}, + auth => #{ + type => boolean, + description => <<"Has auth">>}}}}. + +listeners_api() -> + Metadata = #{ + get => #{ + description => <<"List listeners in cluster">>, + responses => #{ + <<"200">> => + emqx_mgmt_util:response_array_schema(<<"List all listeners">>, listener)}}}, + {"/listeners", Metadata, listeners}. + +restart_listeners_api() -> + Metadata = #{ + get => #{ + description => <<"List listeners by listener ID">>, + parameters => [param_path_identifier()], + responses => #{ + <<"404">> => + emqx_mgmt_util:response_error_schema(<<"Listener id not found">>, ['BAD_LISTENER_ID']), + <<"200">> => + emqx_mgmt_util:response_array_schema(<<"List listener info ok">>, listener)}}}, + {"/listeners/:identifier", Metadata, listener}. + +manage_listeners_api() -> + Metadata = #{ + get => #{ + description => <<"Restart listeners in cluster">>, + parameters => [ + param_path_identifier(), + param_path_operation()], + responses => #{ + <<"500">> => + emqx_mgmt_util:response_error_schema(<<"Operation Failed">>, ['INTERNAL_ERROR']), + <<"404">> => + emqx_mgmt_util:response_error_schema(<<"Listener id not found">>, + ['BAD_LISTENER_ID']), + <<"400">> => + emqx_mgmt_util:response_error_schema(<<"Listener id not found">>, + ['BAD_REQUEST']), + <<"200">> => + emqx_mgmt_util:response_schema(<<"Operation success">>)}}}, + {"/listeners/:identifier/:operation", Metadata, manage_listeners}. + +manage_nodes_listeners_api() -> + Metadata = #{ + get => #{ + description => <<"Restart listeners in cluster">>, + parameters => [ + param_path_node(), + param_path_identifier(), + param_path_operation()], + responses => #{ + <<"500">> => + emqx_mgmt_util:response_error_schema(<<"Operation Failed">>, ['INTERNAL_ERROR']), + <<"404">> => + emqx_mgmt_util:response_error_schema(<<"Bad node or Listener id not found">>, + ['BAD_NODE_NAME','BAD_LISTENER_ID']), + <<"400">> => + emqx_mgmt_util:response_error_schema(<<"Listener id not found">>, + ['BAD_REQUEST']), + <<"200">> => + emqx_mgmt_util:response_schema(<<"Operation success">>)}}}, + {"/node/:node/listeners/:identifier/:operation", Metadata, manage_nodes_listeners}. + +nodes_listeners_api() -> + Metadata = #{ + get => #{ + description => <<"Get listener info in one node">>, + parameters => [param_path_node(), param_path_identifier()], + responses => #{ + <<"404">> => + emqx_mgmt_util:response_error_schema(<<"Node name or listener id not found">>, + ['BAD_NODE_NAME', 'BAD_LISTENER_ID']), + <<"200">> => + emqx_mgmt_util:response_schema(<<"Get listener info ok">>, listener)}}}, + {"/nodes/:node/listeners/:identifier", Metadata, node_listener}. + +nodes_listener_api() -> + Metadata = #{ + get => #{ + description => <<"List listeners in one node">>, + parameters => [param_path_node()], + responses => #{ + <<"404">> => + emqx_mgmt_util:response_error_schema(<<"Listener id not found">>), + <<"200">> => + emqx_mgmt_util:response_schema(<<"Get listener info ok">>, listener)}}}, + {"/nodes/:node/listeners", Metadata, node_listeners}. +%%%============================================================================================== +%% parameters +param_path_node() -> + #{ + name => node, + in => path, + schema => #{type => string}, + required => true, + example => node() + }. + +param_path_identifier() -> + {Example,_} = hd(emqx_mgmt:list_listeners(node())), + #{ + name => identifier, + in => path, + schema => #{type => string}, + required => true, + example => Example + }. + +param_path_operation()-> + #{ + name => operation, + in => path, + required => true, + schema => #{ + type => string, + enum => [start, stop, restart]}, + example => restart + }. + +%%%============================================================================================== +%% api +listeners(get, _Request) -> + list(). + +listener(get, Request) -> + ListenerID = binary_to_atom(cowboy_req:binding(identifier, Request)), + get_listeners(#{identifier => ListenerID}). + +node_listeners(get, Request) -> + Node = binary_to_atom(cowboy_req:binding(node, Request)), + get_listeners(#{node => Node}). + +node_listener(get, Request) -> + Node = binary_to_atom(cowboy_req:binding(node, Request)), + ListenerID = binary_to_atom(cowboy_req:binding(identifier, Request)), + get_listeners(#{node => Node, identifier => ListenerID}). + +manage_listeners(_, Request) -> + Identifier = binary_to_atom(cowboy_req:binding(identifier, Request)), + Operation = binary_to_atom(cowboy_req:binding(operation, Request)), + manage(Operation, #{identifier => Identifier}). + +manage_nodes_listeners(_, Request) -> + Node = binary_to_atom(cowboy_req:binding(node, Request)), + Identifier = binary_to_atom(cowboy_req:binding(identifier, Request)), + Operation = binary_to_atom(cowboy_req:binding(operation, Request)), + manage(Operation, #{identifier => Identifier, node => Node}). + +%%%============================================================================================== %% List listeners in the cluster. -list(_Binding, _Params) -> - minirest:return({ok, [#{node => Node, listeners => format(Listeners)} - || {Node, Listeners} <- emqx_mgmt:list_listeners()]}). +list() -> + {200, format(emqx_mgmt:list_listeners())}. -%% Restart listeners on a node. -restart(#{node := Node, identifier := Identifier}, _Params) -> - case emqx_mgmt:restart_listener(Node, Identifier) of - ok -> minirest:return({ok, "Listener restarted."}); - {error, Error} -> minirest:return({error, Error}) - end; - -%% Restart listeners in the cluster. -restart(#{identifier := <<"http", _/binary>>}, _Params) -> - {403, <<"http_listener_restart_unsupported">>}; -restart(#{identifier := Identifier}, _Params) -> - Results = [{Node, emqx_mgmt:restart_listener(Node, Identifier)} || {Node, _Info} <- emqx_mgmt:list_nodes()], - case lists:filter(fun({_, Result}) -> Result =/= ok end, Results) of - [] -> minirest:return(ok); - Errors -> minirest:return({error, {restart, Errors}}) +get_listeners(Param) -> + case list_listener(Param) of + {error, not_found} -> + Identifier = maps:get(identifier, Param), + Reason = list_to_binary(io_lib:format("Error listener identifier ~p", [Identifier])), + {404, #{code => 'BAD_LISTENER_ID', message => Reason}}; + {error, nodedown} -> + Node = maps:get(node, Param), + Reason = list_to_binary(io_lib:format("Node ~p rpc failed", [Node])), + Response = #{code => 'BAD_NODE_NAME', message => Reason}, + {404, Response}; + [] -> + Identifier = maps:get(identifier, Param), + Reason = list_to_binary(io_lib:format("Error listener identifier ~p", [Identifier])), + {404, #{code => 'BAD_LISTENER_ID', message => Reason}}; + Data -> + {200, Data} end. +manage(Operation0, Param) -> + OperationMap = #{start => start_listener, stop => stop_listener, restart => restart_listener}, + Operation = maps:get(Operation0, OperationMap), + case list_listener(Param) of + {error, not_found} -> + Identifier = maps:get(identifier, Param), + Reason = list_to_binary(io_lib:format("Error listener identifier ~p", [Identifier])), + {404, #{code => 'BAD_LISTENER_ID', message => Reason}}; + {error, nodedown} -> + Node = maps:get(node, Param), + Reason = list_to_binary(io_lib:format("Node ~p rpc failed", [Node])), + Response = #{code => 'BAD_NODE_NAME', message => Reason}, + {404, Response}; + [] -> + Identifier = maps:get(identifier, Param), + Reason = list_to_binary(io_lib:format("Error listener identifier ~p", [Identifier])), + {404, #{code => 'RESOURCE_NOT_FOUND', message => Reason}}; + ListenersOrSingleListener -> + manage_(Operation, ListenersOrSingleListener) + end. + +manage_(Operation, Listener) when is_map(Listener) -> + manage_(Operation, [Listener]); +manage_(Operation, Listeners) when is_list(Listeners) -> + Results = [emqx_mgmt:manage_listener(Operation, Listener) || Listener <- Listeners], + case lists:filter(fun(Result) -> Result =/= ok end, Results) of + [] -> + {200}; + Errors -> + case lists:filter(fun({error, {already_started, _}}) -> false; (_) -> true end, Results) of + [] -> + Identifier = maps:get(identifier, hd(Listeners)), + Message = list_to_binary(io_lib:format("Already Started: ~s", [Identifier])), + {400, #{code => 'BAD_REQUEST', message => Message}}; + _ -> + case lists:filter(fun({error,not_found}) -> false; (_) -> true end, Results) of + [] -> + Identifier = maps:get(identifier, hd(Listeners)), + Message = list_to_binary(io_lib:format("Already Stoped: ~s", [Identifier])), + {400, #{code => 'BAD_REQUEST', message => Message}}; + _ -> + Reason = list_to_binary(io_lib:format("~p", [Errors])), + {500, #{code => 'UNKNOW_ERROR', message => Reason}} + end + end + end. + +%%%============================================================================================== +%% util function +list_listener(Params) -> + format(list_listener_(Params)). + +list_listener_(#{node := Node, identifier := Identifier}) -> + emqx_mgmt:get_listener(Node, Identifier); +list_listener_(#{identifier := Identifier}) -> + emqx_mgmt:list_listeners_by_id(Identifier); +list_listener_(#{node := Node}) -> + emqx_mgmt:list_listeners(Node); +list_listener_(#{}) -> + emqx_mgmt:list_listeners(). + format(Listeners) when is_list(Listeners) -> - [ Info#{listen_on => list_to_binary(esockd:to_string(ListenOn))} - || Info = #{listen_on := ListenOn} <- Listeners ]; + [format(Listener) || Listener <- Listeners]; -format({error, Reason}) -> [{error, Reason}]. +format({error, Reason}) -> + {error, Reason}; +format({Identifier, Conf}) -> + #{ + identifier => Identifier, + node => maps:get(node, Conf), + acceptors => maps:get(acceptors, Conf), + max_conn => maps:get(max_connections, Conf), + type => maps:get(type, Conf), + listen_on => list_to_binary(esockd:to_string(maps:get(bind, Conf))), + running => trans_running(Conf), + auth => maps:get(enable, maps:get(auth, Conf)) + }. +trans_running(Conf) -> + case maps:get(running, Conf) of + {error, _} -> + false; + Running -> + Running + end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl index b59aa0ac5..6f7d7c5f0 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl @@ -16,27 +16,316 @@ -module(emqx_mgmt_api_metrics). --rest_api(#{name => list_all_metrics, - method => 'GET', - path => "/metrics", - func => list, - descr => "A list of metrics of all nodes in the cluster"}). +-behaviour(minirest_api). --rest_api(#{name => list_node_metrics, - method => 'GET', - path => "/nodes/:atom:node/metrics", - func => list, - descr => "A list of metrics of a node"}). +-export([api_spec/0]). -export([list/2]). -list(Bindings, _Params) when map_size(Bindings) == 0 -> - minirest:return({ok, [#{node => Node, metrics => maps:from_list(Metrics)} - || {Node, Metrics} <- emqx_mgmt:get_metrics()]}); +api_spec() -> + {[metrics_api()], [metrics_schema()]}. -list(#{node := Node}, _Params) -> - case emqx_mgmt:get_metrics(Node) of - {error, Reason} -> minirest:return({error, Reason}); - Metrics -> minirest:return({ok, maps:from_list(Metrics)}) +metrics_schema() -> + Metric = #{ + type => object, + properties => properties() + }, + Metrics = #{ + type => array, + items => #{ + type => object, + properties => properties() + } + }, + MetricsInfo = #{ + oneOf => [ minirest:ref(<<"metric">>) + , minirest:ref(<<"metrics">>) + ] + }, + #{metric => Metric, metrics => Metrics, metrics_info => MetricsInfo}. + +properties() -> + #{ + 'actions.failure' => #{ + type => integer, + description => <<"Number of failure executions of the rule engine action">>}, + 'actions.success' => #{ + type => integer, + description => <<"Number of successful executions of the rule engine action">>}, + 'bytes.received' => #{ + type => integer, + description => <<"Number of bytes received by EMQ X Broker">>}, + 'bytes.sent' => #{ + type => integer, + description => <<"Number of bytes sent by EMQ X Broker on this connection">>}, + 'client.authenticate' => #{ + type => integer, + description => <<"Number of client authentications">>}, + 'client.auth.anonymous' => #{ + type => integer, + description => <<"Number of clients who log in anonymously">>}, + 'client.connect' => #{ + type => integer, + description => <<"Number of client connections">>}, + 'client.connack' => #{ + type => integer, + description => <<"Number of CONNACK packet sent">>}, + 'client.connected' => #{ + type => integer, + description => <<"Number of successful client connections">>}, + 'client.disconnected' => #{ + type => integer, + description => <<"Number of client disconnects">>}, + 'client.check_authz' => #{ + type => integer, + description => <<"Number of Authorization rule checks">>}, + 'client.subscribe' => #{ + type => integer, + description => <<"Number of client subscriptions">>}, + 'client.unsubscribe' => #{ + type => integer, + description => <<"Number of client unsubscriptions">>}, + 'delivery.dropped.too_large' => #{ + type => integer, + description => <<"The number of messages that were dropped because the length exceeded the limit when sending">>}, + 'delivery.dropped.queue_full' => #{ + type => integer, + description => <<"Number of messages with a non-zero QoS that were dropped because the message queue was full when sending">>}, + 'delivery.dropped.qos0_msg' => #{ + type => integer, + description => <<"Number of messages with QoS 0 that were dropped because the message queue was full when sending">>}, + 'delivery.dropped.expired' => #{ + type => integer, + description => <<"Number of messages dropped due to message expiration on sending">>}, + 'delivery.dropped.no_local' => #{ + type => integer, + description => <<"Number of messages that were dropped due to the No Local subscription option when sending">>}, + 'delivery.dropped' => #{ + type => integer, + description => <<"Total number of discarded messages when sending">>}, + 'messages.delayed' => #{ + type => integer, + description => <<"Number of delay- published messages stored by EMQ X Broker">>}, + 'messages.delivered' => #{ + type => integer, + description => <<"Number of messages forwarded to the subscription process internally by EMQ X Broker">>}, + 'messages.dropped' => #{ + type => integer, + description => <<"Total number of messages dropped by EMQ X Broker before forwarding to the subscription process">>}, + 'messages.dropped.expired' => #{ + type => integer, + description => <<"Number of messages dropped due to message expiration when receiving">>}, + 'messages.dropped.no_subscribers' => #{ + type => integer, + description => <<"Number of messages dropped due to no subscribers">>}, + 'messages.forward' => #{ + type => integer, + description => <<"Number of messages forwarded to other nodes">>}, + 'messages.publish' => #{ + type => integer, + description => <<"Number of messages published in addition to system messages">>}, + 'messages.qos0.received' => #{ + type => integer, + description => <<"Number of QoS 0 messages received from clients">>}, + 'messages.qos1.received' => #{ + type => integer, + description => <<"Number of QoS 1 messages received from clients">>}, + 'messages.qos2.received' => #{ + type => integer, + description => <<"Number of QoS 2 messages received from clients">>}, + 'messages.qos0.sent' => #{ + type => integer, + description => <<"Number of QoS 0 messages sent to clients">>}, + 'messages.qos1.sent' => #{ + type => integer, + description => <<"Number of QoS 1 messages sent to clients">>}, + 'messages.qos2.sent' => #{ + type => integer, + description => <<"Number of QoS 2 messages sent to clients">>}, + 'messages.received' => #{ + type => integer, + description => <<"Number of messages received from the client, equal to the sum of messages.qos0.received，messages.qos1.received and messages.qos2.received">>}, + 'messages.sent' => #{ + type => integer, + description => <<"Number of messages sent to the client, equal to the sum of messages.qos0.sent，messages.qos1.sent and messages.qos2.sent">>}, + 'messages.retained' => #{ + type => integer, + description => <<"Number of retained messages stored by EMQ X Broker">>}, + 'messages.acked' => #{ + type => integer, + description => <<"Number of received PUBACK and PUBREC packet">>}, + 'packets.received' => #{ + type => integer, + description => <<"Number of received packet">>}, + 'packets.sent' => #{ + type => integer, + description => <<"Number of sent packet">>}, + 'packets.connect.received' => #{ + type => integer, + description => <<"Number of received CONNECT packet">>}, + 'packets.connack.auth_error' => #{ + type => integer, + description => <<"Number of received CONNECT packet with failed authentication">>}, + 'packets.connack.error' => #{ + type => integer, + description => <<"Number of received CONNECT packet with unsuccessful connections">>}, + 'packets.connack.sent' => #{ + type => integer, + description => <<"Number of sent CONNACK packet">>}, + 'packets.publish.received' => #{ + type => integer, + description => <<"Number of received PUBLISH packet">>}, + 'packets.publish.sent' => #{ + type => integer, + description => <<"Number of sent PUBLISH packet">>}, + 'packets.publish.inuse' => #{ + type => integer, + description => <<"Number of received PUBLISH packet with occupied identifiers">>}, + 'packets.publish.auth_error' => #{ + type => integer, + description => <<"Number of received PUBLISH packets with failed the Authorization check">>}, + 'packets.publish.error' => #{ + type => integer, + description => <<"Number of received PUBLISH packet that cannot be published">>}, + 'packets.publish.dropped' => #{ + type => integer, + description => <<"Number of messages discarded due to the receiving limit">>}, + 'packets.puback.received' => #{ + type => integer, + description => <<"Number of received PUBACK packet">>}, + 'packets.puback.sent' => #{ + type => integer, + description => <<"Number of sent PUBACK packet">>}, + 'packets.puback.inuse' => #{ + type => integer, + description => <<"Number of received PUBACK packet with occupied identifiers">>}, + 'packets.puback.missed' => #{ + type => integer, + description => <<"Number of received packet with identifiers.">>}, + 'packets.pubrec.received' => #{ + type => integer, + description => <<"Number of received PUBREC packet">>}, + 'packets.pubrec.sent' => #{ + type => integer, + description => <<"Number of sent PUBREC packet">>}, + 'packets.pubrec.inuse' => #{ + type => integer, + description => <<"Number of received PUBREC packet with occupied identifiers">>}, + 'packets.pubrec.missed' => #{ + type => integer, + description => <<"Number of received PUBREC packet with unknown identifiers">>}, + 'packets.pubrel.received' => #{ + type => integer, + description => <<"Number of received PUBREL packet">>}, + 'packets.pubrel.sent' => #{ + type => integer, + description => <<"Number of sent PUBREL packet">>}, + 'packets.pubrel.missed' => #{ + type => integer, + description => <<"Number of received PUBREC packet with unknown identifiers">>}, + 'packets.pubcomp.received' => #{ + type => integer, + description => <<"Number of received PUBCOMP packet">>}, + 'packets.pubcomp.sent' => #{ + type => integer, + description => <<"Number of sent PUBCOMP packet">>}, + 'packets.pubcomp.inuse' => #{ + type => integer, + description => <<"Number of received PUBCOMP packet with occupied identifiers">>}, + 'packets.pubcomp.missed' => #{ + type => integer, + description => <<"Number of missed PUBCOMP packet">>}, + 'packets.subscribe.received' => #{ + type => integer, + description => <<"Number of received SUBSCRIBE packet">>}, + 'packets.subscribe.error' => #{ + type => integer, + description => <<"Number of received SUBSCRIBE packet with failed subscriptions">>}, + 'packets.subscribe.auth_error' => #{ + type => integer, + description => <<"Number of received SUBACK packet with failed Authorization check">>}, + 'packets.suback.sent' => #{ + type => integer, + description => <<"Number of sent SUBACK packet">>}, + 'packets.unsubscribe.received' => #{ + type => integer, + description => <<"Number of received UNSUBSCRIBE packet">>}, + 'packets.unsubscribe.error' => #{ + type => integer, + description => <<"Number of received UNSUBSCRIBE packet with failed unsubscriptions">>}, + 'packets.unsuback.sent' => #{ + type => integer, + description => <<"Number of sent UNSUBACK packet">>}, + 'packets.pingreq.received' => #{ + type => integer, + description => <<"Number of received PINGREQ packet">>}, + 'packets.pingresp.sent' => #{ + type => integer, + description => <<"Number of sent PUBRESP packet">>}, + 'packets.disconnect.received' => #{ + type => integer, + description => <<"Number of received DISCONNECT packet">>}, + 'packets.disconnect.sent' => #{ + type => integer, + description => <<"Number of sent DISCONNECT packet">>}, + 'packets.auth.received' => #{ + type => integer, + description => <<"Number of received AUTH packet">>}, + 'packets.auth.sent' => #{ + type => integer, + description => <<"Number of sent AUTH packet">>}, + 'rules.matched' => #{ + type => integer, + description => <<"Number of rule matched">>}, + 'session.created' => #{ + type => integer, + description => <<"Number of sessions created">>}, + 'session.discarded' => #{ + type => integer, + description => <<"Number of sessions dropped because Clean Session or Clean Start is true">>}, + 'session.resumed' => #{ + type => integer, + description => <<"Number of sessions resumed because Clean Session or Clean Start is false">>}, + 'session.takeovered' => #{ + type => integer, + description => <<"Number of sessions takeovered because Clean Session or Clean Start is false">>}, + 'session.terminated' => #{ + type => integer, + description => <<"Number of terminated sessions">>} + }. + +metrics_api() -> + Metadata = #{ + get => #{ + description => <<"EMQ X metrics">>, + parameters => [#{ + name => aggregate, + in => query, + schema => #{type => boolean} + }], + responses => #{ + <<"200">> => #{ + description => <<"List all metrics">>, + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"metrics_info">>) + } + } + } + } + } + }, + {"/metrics", Metadata, list}. + +%%%============================================================================================== +%% api apply +list(get, Request) -> + Params = cowboy_req:parse_qs(Request), + case proplists:get_value(<<"aggregate">>, Params, undefined) of + <<"true">> -> + {200, emqx_mgmt:get_metrics()}; + _ -> + Data = [maps:from_list(emqx_mgmt:get_metrics(Node) ++ [{node, Node}]) || + Node <- ekka_mnesia:running_nodes()], + {200, Data} end. - diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index c24e46de9..bd00b173f 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -13,49 +13,219 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- - -module(emqx_mgmt_api_nodes). --rest_api(#{name => list_nodes, - method => 'GET', - path => "/nodes/", - func => list, - descr => "A list of nodes in the cluster"}). +-behaviour(minirest_api). --rest_api(#{name => get_node, - method => 'GET', - path => "/nodes/:atom:node", - func => get, - descr => "Lookup a node in the cluster"}). +-export([api_spec/0]). --export([ list/2 - , get/2 - ]). +-export([ nodes/2 + , node/2 + , node_metrics/2 + , node_stats/2]). -list(_Bindings, _Params) -> - minirest:return({ok, [format(Node, Info) || {Node, Info} <- emqx_mgmt:list_nodes()]}). +-include_lib("emqx/include/emqx.hrl"). -get(#{node := Node}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_node(Node)}). +api_spec() -> + {apis(), schemas()}. -format(Node, {error, Reason}) -> #{node => Node, error => Reason}; +apis() -> + [ nodes_api() + , node_api() + , node_metrics_api() + , node_stats_api()]. + +schemas() -> + %% notice: node api used schema metrics and stats + %% see these schema in emqx_mgmt_api_metrics emqx_mgmt_api_status + [node_schema()]. + +node_schema() -> + #{ + node => #{ + type => object, + properties => #{ + node => #{ + type => string, + description => <<"Node name">>}, + connections => #{ + type => integer, + description => <<"Number of clients currently connected to this node">>}, + load1 => #{ + type => string, + description => <<"CPU average load in 1 minute">>}, + load5 => #{ + type => string, + description => <<"CPU average load in 5 minute">>}, + load15 => #{ + type => string, + description => <<"CPU average load in 15 minute">>}, + max_fds => #{ + type => integer, + description => <<"Maximum file descriptor limit for the operating system">>}, + memory_total => #{ + type => string, + description => <<"VM allocated system memory">>}, + memory_used => #{ + type => string, + description => <<"VM occupied system memory">>}, + node_status => #{ + type => string, + description => <<"Node status">>}, + otp_release => #{ + type => string, + description => <<"Erlang/OTP version used by EMQ X Broker">>}, + process_available => #{ + type => integer, + description => <<"Number of available processes">>}, + process_used => #{ + type => integer, + description => <<"Number of used processes">>}, + uptime => #{ + type => string, + description => <<"EMQ X Broker runtime">>}, + version => #{ + type => string, + description => <<"EMQ X Broker version">>}, + sys_path => #{ + type => string, + description => <<"EMQ X system file location">>}, + log_path => #{ + type => string, + description => <<"EMQ X log file location">>}, + config_path => #{ + type => string, + description => <<"EMQ X config file location">>} + } + } + }. + +nodes_api() -> + Metadata = #{ + get => #{ + description => <<"List EMQ X nodes">>, + responses => #{ + <<"200">> => emqx_mgmt_util:response_array_schema(<<"List EMQ X Nodes">>, node)}}}, + {"/nodes", Metadata, nodes}. + +node_api() -> + Metadata = #{ + get => #{ + description => <<"Get node info">>, + parameters => [#{ + name => node_name, + in => path, + description => "node name", + schema => #{type => string}, + required => true, + example => node()}], + responses => #{ + <<"400">> => emqx_mgmt_util:response_error_schema(<<"Node error">>, ['SOURCE_ERROR']), + <<"200">> => emqx_mgmt_util:response_schema(<<"Get EMQ X Nodes info by name">>, node)}}}, + {"/nodes/:node_name", Metadata, node}. + +node_metrics_api() -> + Metadata = #{ + get => #{ + description => <<"Get node metrics">>, + parameters => [#{ + name => node_name, + in => path, + description => "node name", + schema => #{type => string}, + required => true, + example => node()}], + responses => #{ + <<"400">> => emqx_mgmt_util:response_error_schema(<<"Node error">>, ['SOURCE_ERROR']), + <<"200">> => emqx_mgmt_util:response_schema(<<"Get EMQ X Node Metrics">>, metrics)}}}, + {"/nodes/:node_name/metrics", Metadata, node_metrics}. + +node_stats_api() -> + Metadata = #{ + get => #{ + description => <<"Get node stats">>, + parameters => [#{ + name => node_name, + in => path, + description => "node name", + schema => #{type => string}, + required => true, + example => node()}], + responses => #{ + <<"400">> => emqx_mgmt_util:response_error_schema(<<"Node error">>, ['SOURCE_ERROR']), + <<"200">> => emqx_mgmt_util:response_schema(<<"Get EMQ X Node Stats">>, stats)}}}, + {"/nodes/:node_name/stats", Metadata, node_metrics}. + +%%%============================================================================================== +%% parameters trans +nodes(get, _Request) -> + list(#{}). + +node(get, Request) -> + Params = node_name_path_parameter(Request), + get_node(Params). + +node_metrics(get, Request) -> + Params = node_name_path_parameter(Request), + get_metrics(Params). + +node_stats(get, Request) -> + Params = node_name_path_parameter(Request), + get_stats(Params). + +%%%============================================================================================== +%% api apply +list(#{}) -> + NodesInfo = [format(Node, NodeInfo) || {Node, NodeInfo} <- emqx_mgmt:list_nodes()], + {200, NodesInfo}. + +get_node(#{node := Node}) -> + case emqx_mgmt:lookup_node(Node) of + #{node_status := 'ERROR'} -> + {400, #{code => 'SOURCE_ERROR', message => <<"rpc_failed">>}}; + NodeInfo -> + {200, format(Node, NodeInfo)} + end. + +get_metrics(#{node := Node}) -> + case emqx_mgmt:get_metrics(Node) of + {error, _} -> + {400, #{code => 'SOURCE_ERROR', message => <<"rpc_failed">>}}; + Metrics -> + {200, Metrics} + end. + +get_stats(#{node := Node}) -> + case emqx_mgmt:get_stats(Node) of + {error, _} -> + {400, #{code => 'SOURCE_ERROR', message => <<"rpc_failed">>}}; + Stats -> + {200, Stats} + end. + +%%============================================================================================================ +%% internal function +node_name_path_parameter(Request) -> + NodeName = cowboy_req:binding(node_name, Request), + Node = binary_to_atom(NodeName, utf8), + #{node => Node}. format(_Node, Info = #{memory_total := Total, memory_used := Used}) -> {ok, SysPathBinary} = file:get_cwd(), - SysPath = list_to_binary(SysPathBinary), - ConfigPath = <>, - LogPath = case log_path() of - undefined -> - <<"not found">>; - Path0 -> - Path = list_to_binary(Path0), - <> - end, - Info#{ memory_total := emqx_mgmt_util:kmg(Total) - , memory_used := emqx_mgmt_util:kmg(Used) - , sys_path => SysPath - , config_path => ConfigPath - , log_path => LogPath}. + SysPath = list_to_binary(SysPathBinary), + ConfigPath = <>, + LogPath = case log_path() of + undefined -> + <<"not found">>; + Path0 -> + Path = list_to_binary(Path0), + <> + end, + Info#{ memory_total := emqx_mgmt_util:kmg(Total) + , memory_used := emqx_mgmt_util:kmg(Used) + , sys_path => SysPath + , config_path => ConfigPath + , log_path => LogPath}. log_path() -> Configs = logger:get_handler_config(), diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index e4908aeff..fda7151d7 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -69,36 +69,36 @@ ]). list(#{node := Node}, _Params) -> - minirest:return({ok, [format(Plugin) || Plugin <- emqx_mgmt:list_plugins(Node)]}); + emqx_mgmt:return({ok, [format(Plugin) || Plugin <- emqx_mgmt:list_plugins(Node)]}); list(_Bindings, _Params) -> - minirest:return({ok, [format({Node, Plugins}) || {Node, Plugins} <- emqx_mgmt:list_plugins()]}). + emqx_mgmt:return({ok, [format({Node, Plugins}) || {Node, Plugins} <- emqx_mgmt:list_plugins()]}). load(#{node := Node, plugin := Plugin}, _Params) -> - minirest:return(emqx_mgmt:load_plugin(Node, Plugin)). + emqx_mgmt:return(emqx_mgmt:load_plugin(Node, Plugin)). unload(#{node := Node, plugin := Plugin}, _Params) -> - minirest:return(emqx_mgmt:unload_plugin(Node, Plugin)); + emqx_mgmt:return(emqx_mgmt:unload_plugin(Node, Plugin)); unload(#{plugin := Plugin}, _Params) -> Results = [emqx_mgmt:unload_plugin(Node, Plugin) || {Node, _Info} <- emqx_mgmt:list_nodes()], case lists:filter(fun(Item) -> Item =/= ok end, Results) of [] -> - minirest:return(ok); + emqx_mgmt:return(ok); Errors -> - minirest:return(lists:last(Errors)) + emqx_mgmt:return(lists:last(Errors)) end. reload(#{node := Node, plugin := Plugin}, _Params) -> - minirest:return(emqx_mgmt:reload_plugin(Node, Plugin)); + emqx_mgmt:return(emqx_mgmt:reload_plugin(Node, Plugin)); reload(#{plugin := Plugin}, _Params) -> Results = [emqx_mgmt:reload_plugin(Node, Plugin) || {Node, _Info} <- emqx_mgmt:list_nodes()], case lists:filter(fun(Item) -> Item =/= ok end, Results) of [] -> - minirest:return(ok); + emqx_mgmt:return(ok); Errors -> - minirest:return(lists:last(Errors)) + emqx_mgmt:return(lists:last(Errors)) end. format({Node, Plugins}) -> @@ -106,10 +106,8 @@ format({Node, Plugins}) -> format(#plugin{name = Name, descr = Descr, - active = Active, - type = Type}) -> + active = Active}) -> #{name => Name, description => iolist_to_binary(Descr), - active => Active, - type => Type}. + active => Active}. diff --git a/apps/emqx_management/src/emqx_mgmt_api_publish.erl b/apps/emqx_management/src/emqx_mgmt_api_publish.erl new file mode 100644 index 000000000..29e162b11 --- /dev/null +++ b/apps/emqx_management/src/emqx_mgmt_api_publish.erl @@ -0,0 +1,158 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_api_publish). +%% API +-include_lib("emqx/include/emqx.hrl"). + +-behaviour(minirest_api). + +-export([api_spec/0]). + +-export([ publish/2 + , publish_batch/2]). + +api_spec() -> + { + [publish_api(), publish_batch_api()], + [message_schema()] + }. + +publish_api() -> + MeteData = #{ + post => #{ + description => <<"Publish">>, + 'requestBody' => #{ + content => #{ + 'application/json' => #{ + schema => #{ + type => object, + properties => maps:with([id], message_properties())}}}}, + responses => #{ + <<"200">> => emqx_mgmt_util:response_schema(<<"publish ok">>, message)}}}, + {"/publish", MeteData, publish}. + +publish_batch_api() -> + MeteData = #{ + post => #{ + description => <<"publish">>, + 'requestBody' => #{ + content => #{ + 'application/json' => #{ + schema => #{ + type => array, + items => #{ + type => object, + properties => maps:with([id], message_properties())}}}}}, + responses => #{ + <<"200">> => emqx_mgmt_util:response_array_schema(<<"publish ok">>, message)}}}, + {"/publish/bulk", MeteData, publish_batch}. + +message_schema() -> + #{ + message => #{ + type => object, + properties => message_properties() + } + }. + +message_properties() -> + #{ + id => #{ + type => string, + description => <<"Message ID">>}, + topic => #{ + type => string, + description => <<"Topic">>}, + qos => #{ + type => integer, + enum => [0, 1, 2], + description => <<"Qos">>}, + payload => #{ + type => string, + description => <<"Topic">>}, + from => #{ + type => string, + description => <<"Message from">>}, + flag => #{ + type => <<"object">>, + description => <<"Message flag">>, + properties => #{ + sys => #{ + type => boolean, + default => false, + description => <<"System message flag, nullable, default false">>}, + dup => #{ + type => boolean, + default => false, + description => <<"Dup message flag, nullable, default false">>}, + retain => #{ + type => boolean, + default => false, + description => <<"Retain message flag, nullable, default false">>} + } + } + }. + +publish(post, Request) -> + {ok, Body, _} = cowboy_req:read_body(Request), + Message = message(emqx_json:decode(Body, [return_maps])), + _ = emqx_mgmt:publish(Message), + {200, format_message(Message)}. + +publish_batch(post, Request) -> + {ok, Body, _} = cowboy_req:read_body(Request), + Messages = messages(emqx_json:decode(Body, [return_maps])), + _ = [emqx_mgmt:publish(Message) || Message <- Messages], + {200, format_message(Messages)}. + +message(Map) -> + From = maps:get(<<"from">>, Map, http_api), + QoS = maps:get(<<"qos">>, Map, 0), + Topic = maps:get(<<"topic">>, Map), + Payload = maps:get(<<"payload">>, Map), + Flags = flags(Map), + emqx_message:make(From, QoS, Topic, Payload, Flags, #{}). + +flags(Map) -> + Flags = maps:get(<<"flags">>, Map, #{}), + Retain = maps:get(<<"retain">>, Flags, false), + Sys = maps:get(<<"sys">>, Flags, false), + Dup = maps:get(<<"dup">>, Flags, false), + #{ + retain => Retain, + sys => Sys, + dup => Dup + }. + +messages(List) -> + [message(MessageMap) || MessageMap <- List]. + +format_message(Messages) when is_list(Messages)-> + [format_message(Message) || Message <- Messages]; +format_message(#message{id = ID, qos = Qos, from = From, topic = Topic, payload = Payload, flags = Flags}) -> + #{ + id => emqx_guid:to_hexstr(ID), + qos => Qos, + topic => Topic, + payload => Payload, + flag => Flags, + from => to_binary(From) + }. + +to_binary(Data) when is_binary(Data) -> + Data; +to_binary(Data) -> + list_to_binary(io_lib:format("~p", [Data])). diff --git a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl index e5a3e9d77..28e67c9f1 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl @@ -67,7 +67,7 @@ subscribe(_Bindings, Params) -> logger:debug("API subscribe Params:~p", [Params]), {ClientId, Topic, QoS} = parse_subscribe_params(Params), - minirest:return(do_subscribe(ClientId, Topic, QoS)). + emqx_mgmt:return(do_subscribe(ClientId, Topic, QoS)). publish(_Bindings, Params) -> logger:debug("API publish Params:~p", [Params]), @@ -75,33 +75,33 @@ publish(_Bindings, Params) -> case do_publish(ClientId, Topic, Qos, Retain, Payload) of {ok, MsgIds} -> case proplists:get_value(<<"return">>, Params, undefined) of - undefined -> minirest:return(ok); + undefined -> emqx_mgmt:return(ok); _Val -> case proplists:get_value(<<"topics">>, Params, undefined) of - undefined -> minirest:return({ok, #{msgid => lists:last(MsgIds)}}); - _ -> minirest:return({ok, #{msgids => MsgIds}}) + undefined -> emqx_mgmt:return({ok, #{msgid => lists:last(MsgIds)}}); + _ -> emqx_mgmt:return({ok, #{msgids => MsgIds}}) end end; Result -> - minirest:return(Result) + emqx_mgmt:return(Result) end. unsubscribe(_Bindings, Params) -> logger:debug("API unsubscribe Params:~p", [Params]), {ClientId, Topic} = parse_unsubscribe_params(Params), - minirest:return(do_unsubscribe(ClientId, Topic)). + emqx_mgmt:return(do_unsubscribe(ClientId, Topic)). subscribe_batch(_Bindings, Params) -> logger:debug("API subscribe batch Params:~p", [Params]), - minirest:return({ok, loop_subscribe(Params)}). + emqx_mgmt:return({ok, loop_subscribe(Params)}). publish_batch(_Bindings, Params) -> logger:debug("API publish batch Params:~p", [Params]), - minirest:return({ok, loop_publish(Params)}). + emqx_mgmt:return({ok, loop_publish(Params)}). unsubscribe_batch(_Bindings, Params) -> logger:debug("API unsubscribe batch Params:~p", [Params]), - minirest:return({ok, loop_unsubscribe(Params)}). + emqx_mgmt:return({ok, loop_unsubscribe(Params)}). loop_subscribe(Params) -> loop_subscribe(Params, []). diff --git a/apps/emqx_management/src/emqx_mgmt_api_routes.erl b/apps/emqx_management/src/emqx_mgmt_api_routes.erl index 380c9f0f6..258680546 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_routes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_routes.erl @@ -18,30 +18,101 @@ -include_lib("emqx/include/emqx.hrl"). --rest_api(#{name => list_routes, - method => 'GET', - path => "/routes/", - func => list, - descr => "List routes"}). +%% API +-behaviour(minirest_api). --rest_api(#{name => lookup_routes, - method => 'GET', - path => "/routes/:bin:topic", - func => lookup, - descr => "Lookup routes to a topic"}). +-export([api_spec/0]). --export([ list/2 - , lookup/2 - ]). +-export([ routes/2 + , route/2]). -list(Bindings, Params) when map_size(Bindings) == 0 -> - minirest:return({ok, emqx_mgmt_api:paginate(emqx_route, Params, fun format/1)}). +-define(TOPIC_NOT_FOUND, 'TOPIC_NOT_FOUND'). -lookup(#{topic := Topic}, _Params) -> - Topic1 = emqx_mgmt_util:urldecode(Topic), - minirest:return({ok, [format(R) || R <- emqx_mgmt:lookup_routes(Topic1)]}). +api_spec() -> + { + [routes_api(), route_api()], + [route_schema()] + }. + +route_schema() -> + #{ + route => #{ + type => object, + properties => #{ + topic => #{ + type => string}, + node => #{ + type => string, + example => node()}}}}. + +routes_api() -> + Metadata = #{ + get => #{ + description => <<"EMQ X routes">>, + parameters => [ + #{ + name => page, + in => query, + description => <<"Page">>, + schema => #{type => integer, default => 1} + }, + #{ + name => limit, + in => query, + description => <<"Page size">>, + schema => #{type => integer, default => emqx_mgmt:max_row_limit()} + }], + responses => #{ + <<"200">> => + emqx_mgmt_util:response_array_schema("List route info", route)}}}, + {"/routes", Metadata, routes}. + +route_api() -> + Metadata = #{ + get => #{ + description => <<"EMQ X routes">>, + parameters => [#{ + name => topic, + in => path, + required => true, + description => <<"topic">>, + schema => #{type => string} + }], + responses => #{ + <<"200">> => + emqx_mgmt_util:response_schema(<<"Route info">>, route), + <<"404">> => + emqx_mgmt_util:response_error_schema(<<"Topic not found">>, [?TOPIC_NOT_FOUND]) + }}}, + {"/routes/:topic", Metadata, route}. + +%%%============================================================================================== +%% parameters trans +routes(get, Request) -> + Params = cowboy_req:parse_qs(Request), + list(Params). + +route(get, Request) -> + Topic = cowboy_req:binding(topic, Request), + lookup(#{topic => Topic}). + +%%%============================================================================================== +%% api apply +list(Params) -> + Response = emqx_mgmt_api:paginate(emqx_route, Params, fun format/1), + {200, Response}. + +lookup(#{topic := Topic}) -> + case emqx_mgmt:lookup_routes(Topic) of + [] -> + {404, #{code => ?TOPIC_NOT_FOUND, message => <<"Topic not found">>}}; + [Route] -> + {200, format(Route)} + end. + +%%%============================================================================================== +%% internal format(#route{topic = Topic, dest = {_, Node}}) -> #{topic => Topic, node => Node}; format(#route{topic = Topic, dest = Node}) -> #{topic => Topic, node => Node}. - diff --git a/apps/emqx_management/src/emqx_mgmt_api_stats.erl b/apps/emqx_management/src/emqx_mgmt_api_stats.erl index 95d54b775..706723f8b 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_stats.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_stats.erl @@ -13,33 +13,130 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- - -module(emqx_mgmt_api_stats). --rest_api(#{name => list_stats, - method => 'GET', - path => "/stats/", - func => list, - descr => "A list of stats of all nodes in the cluster"}). +-behaviour(minirest_api). --rest_api(#{name => lookup_node_stats, - method => 'GET', - path => "/nodes/:atom:node/stats/", - func => lookup, - descr => "A list of stats of a node"}). +-export([api_spec/0]). --export([ list/2 - , lookup/2 - ]). +-export([list/2]). -%% List stats of all nodes -list(Bindings, _Params) when map_size(Bindings) == 0 -> - minirest:return({ok, [#{node => Node, stats => maps:from_list(Stats)} - || {Node, Stats} <- emqx_mgmt:get_stats()]}). +api_spec() -> + {[stats_api()], stats_schema()}. -%% List stats of a node -lookup(#{node := Node}, _Params) -> - case emqx_mgmt:get_stats(Node) of - {error, Reason} -> minirest:return({error, Reason}); - Stats -> minirest:return({ok, maps:from_list(Stats)}) +stats_schema() -> + Stats = #{ + type => array, + items => #{ + type => object, + properties => maps:put('node', #{type => string, description => <<"Node">>}, properties()) + } + }, + Stat = #{ + type => object, + properties => properties() + }, + StatsInfo =#{ + oneOf => [ minirest:ref(<<"stats">>) + , minirest:ref(<<"stat">>) + ] + }, + [#{stats => Stats, stat => Stat, stats_info => StatsInfo}]. + +properties() -> + #{ + 'connections.count' => #{ + type => integer, + description => <<"Number of current connections">>}, + 'connections.max' => #{ + type => integer, + description => <<"Historical maximum number of connections">>}, + 'channels.count' => #{ + type => integer, + description => <<"sessions.count">>}, + 'channels.max' => #{ + type => integer, + description => <<"session.max">>}, + 'sessions.count' => #{ + type => integer, + description => <<"Number of current sessions">>}, + 'sessions.max' => #{ + type => integer, + description => <<"Historical maximum number of sessions">>}, + 'topics.count' => #{ + type => integer, + description => <<"Number of current topics">>}, + 'topics.max' => #{ + type => integer, + description => <<"Historical maximum number of topics">>}, + 'suboptions.count' => #{ + type => integer, + description => <<"subscriptions.count">>}, + 'suboptions.max' => #{ + type => integer, + description => <<"subscriptions.max">>}, + 'subscribers.count' => #{ + type => integer, + description => <<"Number of current subscribers">>}, + 'subscribers.max' => #{ + type => integer, + description => <<"Historical maximum number of subscribers">>}, + 'subscriptions.count' => #{ + type => integer, + description => <<"Number of current subscriptions, including shared subscriptions">>}, + 'subscriptions.max' => #{ + type => integer, + description => <<"Historical maximum number of subscriptions">>}, + 'subscriptions.shared.count' => #{ + type => integer, + description => <<"Number of current shared subscriptions">>}, + 'subscriptions.shared.max' => #{ + type => integer, + description => <<"Historical maximum number of shared subscriptions">>}, + 'routes.count' => #{ + type => integer, + description => <<"Number of current routes">>}, + 'routes.max' => #{ + type => integer, + description => <<"Historical maximum number of routes">>}, + 'retained.count' => #{ + type => integer, + description => <<"Number of currently retained messages">>}, + 'retained.max' => #{ + type => integer, + description => <<"Historical maximum number of retained messages">>} + }. + +stats_api() -> + Metadata = #{ + get => #{ + description => <<"EMQ X stats">>, + parameters => [#{ + name => aggregate, + in => query, + schema => #{type => boolean} + }], + responses => #{ + <<"200">> => #{ + description => <<"List stats ok">>, + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"stats_info">>) + } + } + } + }}}, + {"/stats", Metadata, list}. + +%%%============================================================================================== +%% api apply +list(get, Request) -> + Params = cowboy_req:parse_qs(Request), + case proplists:get_value(<<"aggregate">>, Params, undefined) of + <<"true">> -> + {200, emqx_mgmt:get_stats()}; + _ -> + Data = [maps:from_list(emqx_mgmt:get_stats(Node) ++ [{node, Node}]) || + Node <- ekka_mnesia:running_nodes()], + {200, Data} end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_status.erl b/apps/emqx_management/src/emqx_mgmt_api_status.erl new file mode 100644 index 000000000..fa46b1d25 --- /dev/null +++ b/apps/emqx_management/src/emqx_mgmt_api_status.erl @@ -0,0 +1,47 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_api_status). +%% API +-behaviour(minirest_api). + +-export([api_spec/0]). + +-export([running_status/2]). + +api_spec() -> + {[status_api()], []}. + +status_api() -> + Path = "/status", + Metadata = #{ + get => #{ + security => [], + responses => #{ + <<"200">> => #{description => <<"running">>}}}}, + {Path, Metadata, running_status}. + +running_status(get, _Request) -> + {InternalStatus, _ProvidedStatus} = init:get_status(), + AppStatus = + case lists:keysearch(emqx, 1, application:which_applications()) of + false -> not_running; + {value, _Val} -> running + end, + Status = io_lib:format("Node ~s is ~s~nemqx is ~s", [node(), InternalStatus, AppStatus]), + Body = list_to_binary(Status), + {200, #{<<"content-type">> => <<"text/plain">>}, Body}. + + diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index 4165ca51a..059327f4c 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -16,79 +16,107 @@ -module(emqx_mgmt_api_subscriptions). +-behaviour(minirest_api). + -include_lib("emqx/include/emqx.hrl"). --define(SUBS_QS_SCHEMA, {emqx_suboption, - [{<<"clientid">>, binary}, - {<<"topic">>, binary}, - {<<"share">>, binary}, - {<<"qos">>, integer}, - {<<"_match_topic">>, binary}]}). +-export([api_spec/0]). --rest_api(#{name => list_subscriptions, - method => 'GET', - path => "/subscriptions/", - func => list, - descr => "A list of subscriptions in the cluster"}). - --rest_api(#{name => list_node_subscriptions, - method => 'GET', - path => "/nodes/:atom:node/subscriptions/", - func => list, - descr => "A list of subscriptions on a node"}). - --rest_api(#{name => lookup_client_subscriptions, - method => 'GET', - path => "/subscriptions/:bin:clientid", - func => lookup, - descr => "A list of subscriptions of a client"}). - --rest_api(#{name => lookup_client_subscriptions_with_node, - method => 'GET', - path => "/nodes/:atom:node/subscriptions/:bin:clientid", - func => lookup, - descr => "A list of subscriptions of a client on the node"}). - --export([ list/2 - , lookup/2 - ]). +-export([subscriptions/2]). -export([ query/3 , format/1 ]). +-define(SUBS_QS_SCHEMA, {emqx_suboption, + [ {<<"clientid">>, binary} + , {<<"topic">>, binary} + , {<<"share">>, binary} + , {<<"qos">>, integer} + , {<<"match_topic">>, binary}]}). + -define(query_fun, {?MODULE, query}). -define(format_fun, {?MODULE, format}). -list(Bindings, Params) when map_size(Bindings) == 0 -> - case proplists:get_value(<<"topic">>, Params) of - undefined -> - minirest:return({ok, emqx_mgmt_api:cluster_query(Params, ?SUBS_QS_SCHEMA, ?query_fun)}); - Topic -> - minirest:return({ok, emqx_mgmt:list_subscriptions_via_topic(emqx_mgmt_util:urldecode(Topic), ?format_fun)}) - end; +api_spec() -> + { + [subscriptions_api()], + [subscription_schema()] + }. -list(#{node := Node} = Bindings, Params) -> - case proplists:get_value(<<"topic">>, Params) of - undefined -> - case Node =:= node() of - true -> - minirest:return({ok, emqx_mgmt_api:node_query(Node, Params, ?SUBS_QS_SCHEMA, ?query_fun)}); - false -> - case rpc:call(Node, ?MODULE, list, [Bindings, Params]) of - {badrpc, Reason} -> minirest:return({error, Reason}); - Res -> Res - end - end; - Topic -> - minirest:return({ok, emqx_mgmt:list_subscriptions_via_topic(Node, emqx_mgmt_util:urldecode(Topic), ?format_fun)}) - end. +subscriptions_api() -> + MetaData = #{ + get => #{ + description => <<"List subscriptions">>, + parameters => [ + #{ + name => page, + in => query, + description => <<"Page">>, + schema => #{type => integer} + }, + #{ + name => limit, + in => query, + description => <<"Page size">>, + schema => #{type => integer} + }, + #{ + name => clientid, + in => query, + description => <<"Client ID">>, + schema => #{type => string} + }, + #{ + name => qos, + in => query, + description => <<"QoS">>, + schema => #{type => integer} + }, + #{ + name => share, + in => query, + description => <<"Shared subscription">>, + schema => #{type => boolean} + }, + #{ + name => topic, + in => query, + description => <<"Topic">>, + schema => #{type => string} + } + #{ + name => match_topic, + in => query, + description => <<"Match topic string">>, + schema => #{type => string} + } + ], + responses => #{ + <<"200">> => emqx_mgmt_util:response_page_schema(subscription)}}}, + {"/subscriptions", MetaData, subscriptions}. -lookup(#{node := Node, clientid := ClientId}, _Params) -> - minirest:return({ok, format(emqx_mgmt:lookup_subscriptions(Node, emqx_mgmt_util:urldecode(ClientId)))}); +subscription_schema() -> + #{ + subscription => #{ + type => object, + properties => #{ + topic => #{ + type => string}, + clientid => #{ + type => string}, + qos => #{ + type => integer, + enum => [0,1,2]}}} + }. + +subscriptions(get, Request) -> + Params = cowboy_req:parse_qs(Request), + list(Params). + +list(Params) -> + {200, emqx_mgmt_api:cluster_query(Params, ?SUBS_QS_SCHEMA, ?query_fun)}. -lookup(#{clientid := ClientId}, _Params) -> - minirest:return({ok, format(emqx_mgmt:lookup_subscriptions(emqx_mgmt_util:urldecode(ClientId)))}). format(Items) when is_list(Items) -> [format(Item) || Item <- Items]; @@ -98,10 +126,10 @@ format({{Subscriber, Topic}, Options}) -> format({_Subscriber, Topic, Options = #{share := Group}}) -> QoS = maps:get(qos, Options), - #{node => node(), topic => filename:join([<<"$share">>, Group, Topic]), clientid => maps:get(subid, Options), qos => QoS}; + #{topic => filename:join([<<"$share">>, Group, Topic]), clientid => maps:get(subid, Options), qos => QoS}; format({_Subscriber, Topic, Options}) -> QoS = maps:get(qos, Options), - #{node => node(), topic => Topic, clientid => maps:get(subid, Options), qos => QoS}. + #{topic => Topic, clientid => maps:get(subid, Options), qos => QoS}. %%-------------------------------------------------------------------- %% Query Function diff --git a/apps/emqx_management/src/emqx_mgmt_app.erl b/apps/emqx_management/src/emqx_mgmt_app.erl index 824e218f2..ff85666b3 100644 --- a/apps/emqx_management/src/emqx_mgmt_app.erl +++ b/apps/emqx_management/src/emqx_mgmt_app.erl @@ -18,7 +18,7 @@ -behaviour(application). --emqx_plugin(?MODULE). +-define(APP, emqx_management). -export([ start/2 , stop/1 diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index 3998f6006..7fb120017 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -35,7 +35,7 @@ , list_apps/0 ]). -%% APP Auth/ACL API +%% APP Auth/Authorization API -export([is_authorized/2]). -define(APP, emqx_management). @@ -66,18 +66,20 @@ mnesia(copy) -> %%-------------------------------------------------------------------- %% Manage Apps %%-------------------------------------------------------------------- --spec(add_default_app() -> ok | {ok, appsecret()} | {error, term()}). +-spec(add_default_app() -> list()). add_default_app() -> - AppId = application:get_env(?APP, default_application_id, undefined), - AppSecret = application:get_env(?APP, default_application_secret, undefined), - case {AppId, AppSecret} of - {undefined, _} -> ok; - {_, undefined} -> ok; - {_, _} -> - AppId1 = erlang:list_to_binary(AppId), - AppSecret1 = erlang:list_to_binary(AppSecret), - add_app(AppId1, <<"Default">>, AppSecret1, <<"Application user">>, true, undefined) - end. + Apps = emqx_config:get([?APP, applications], []), + [ begin + case {AppId, AppSecret} of + {undefined, _} -> ok; + {_, undefined} -> ok; + {_, _} -> + AppId1 = to_binary(AppId), + AppSecret1 = to_binary(AppSecret), + add_app(AppId1, <<"Default">>, AppSecret1, <<"Application user">>, true, undefined) + end + end + || #{id := AppId, secret := AppSecret} <- Apps]. -spec(add_app(appid(), binary()) -> {ok, appsecret()} | {error, term()}). add_app(AppId, Name) when is_binary(AppId) -> @@ -129,11 +131,7 @@ force_add_app(AppId, Name, Secret, Desc, Status, Expired) -> generate_appsecret_if_need(InSecrt) when is_binary(InSecrt), byte_size(InSecrt) > 0 -> InSecrt; generate_appsecret_if_need(_) -> - AppConf = application:get_env(?APP, application, []), - case proplists:get_value(default_secret, AppConf) of - undefined -> emqx_guid:to_base62(emqx_guid:gen()); - Secret when is_binary(Secret) -> Secret - end. + emqx_guid:to_base62(emqx_guid:gen()). -spec(get_appsecret(appid()) -> {appsecret() | undefined}). get_appsecret(AppId) when is_binary(AppId) -> @@ -214,3 +212,6 @@ is_authorized(AppId, AppSecret) -> is_expired(undefined) -> true; is_expired(Expired) -> Expired >= erlang:system_time(second). + +to_binary(L) when is_list(L) -> list_to_binary(L); +to_binary(B) when is_binary(B) -> B. diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index e70def3ee..dcaac5052 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -38,7 +38,7 @@ , trace/1 , log/1 , mgmt/1 - , acl/1 + , authz/1 ]). -define(PROC_INFOKEYS, [status, @@ -463,112 +463,83 @@ trace_off(Who, Name) -> listeners([]) -> lists:foreach(fun({{Protocol, ListenOn}, _Pid}) -> - Info = [{listen_on, {string, emqx_listeners:format_listen_on(ListenOn)}}, + Info = [{listen_on, {string, format_listen_on(ListenOn)}}, {acceptors, esockd:get_acceptors({Protocol, ListenOn})}, {max_conns, esockd:get_max_connections({Protocol, ListenOn})}, {current_conn, esockd:get_current_connections({Protocol, ListenOn})}, {shutdown_count, esockd:get_shutdown_count({Protocol, ListenOn})} ], - emqx_ctl:print("~s~n", [listener_identifier(Protocol, ListenOn)]), + emqx_ctl:print("~s~n", [Protocol]), lists:foreach(fun indent_print/1, Info) end, esockd:listeners()), lists:foreach(fun({Protocol, Opts}) -> Port = proplists:get_value(port, Opts), - Info = [{listen_on, {string, emqx_listeners:format_listen_on(Port)}}, + Info = [{listen_on, {string, format_listen_on(Port)}}, {acceptors, maps:get(num_acceptors, proplists:get_value(transport_options, Opts, #{}), 0)}, {max_conns, proplists:get_value(max_connections, Opts)}, {current_conn, proplists:get_value(all_connections, Opts)}, {shutdown_count, []}], - emqx_ctl:print("~s~n", [listener_identifier(Protocol, Port)]), + emqx_ctl:print("~s~n", [Protocol]), lists:foreach(fun indent_print/1, Info) end, ranch:info()); -listeners(["stop", Name = "http" ++ _N | _MaybePort]) -> - %% _MaybePort is to be backward compatible, to stop http listener, there is no need for the port number - case minirest:stop_http(list_to_atom(Name)) of +listeners(["stop", ListenerId]) -> + case emqx_listeners:stop_listener(list_to_atom(ListenerId)) of ok -> - emqx_ctl:print("Stop ~s listener successfully.~n", [Name]); + emqx_ctl:print("Stop ~s listener successfully.~n", [ListenerId]); {error, Error} -> - emqx_ctl:print("Failed to stop ~s listener: ~0p~n", [Name, Error]) + emqx_ctl:print("Failed to stop ~s listener: ~0p~n", [ListenerId, Error]) end; -listeners(["stop", "mqtt:" ++ _ = Identifier]) -> - stop_listener(emqx_listeners:find_by_id(Identifier), Identifier); - -listeners(["stop", _Proto, ListenOn]) -> - %% this clause is kept to be backward compatible - ListenOn1 = case string:tokens(ListenOn, ":") of - [Port] -> list_to_integer(Port); - [IP, Port] -> {IP, list_to_integer(Port)} - end, - stop_listener(emqx_listeners:find_by_listen_on(ListenOn1), ListenOn1); - -listeners(["restart", "http:management"]) -> - restart_http_listener(http, emqx_management); - -listeners(["restart", "https:management"]) -> - restart_http_listener(https, emqx_management); - -listeners(["restart", "http:dashboard"]) -> - restart_http_listener(http, emqx_dashboard); - -listeners(["restart", "https:dashboard"]) -> - restart_http_listener(https, emqx_dashboard); - -listeners(["restart", Identifier]) -> - case emqx_listeners:restart_listener(Identifier) of +listeners(["start", ListenerId]) -> + case emqx_listeners:start_listener(list_to_atom(ListenerId)) of ok -> - emqx_ctl:print("Restarted ~s listener successfully.~n", [Identifier]); + emqx_ctl:print("Started ~s listener successfully.~n", [ListenerId]); {error, Error} -> - emqx_ctl:print("Failed to restart ~s listener: ~0p~n", [Identifier, Error]) + emqx_ctl:print("Failed to start ~s listener: ~0p~n", [ListenerId, Error]) + end; + +listeners(["restart", ListenerId]) -> + case emqx_listeners:restart_listener(list_to_atom(ListenerId)) of + ok -> + emqx_ctl:print("Restarted ~s listener successfully.~n", [ListenerId]); + {error, Error} -> + emqx_ctl:print("Failed to restart ~s listener: ~0p~n", [ListenerId, Error]) end; listeners(_) -> emqx_ctl:usage([{"listeners", "List listeners"}, {"listeners stop ", "Stop a listener"}, - {"listeners stop ", "Stop a listener"}, + {"listeners start ", "Start a listener"}, {"listeners restart ", "Restart a listener"} ]). -stop_listener(false, Input) -> - emqx_ctl:print("No such listener ~p~n", [Input]); -stop_listener(#{listen_on := ListenOn} = Listener, _Input) -> - ID = emqx_listeners:identifier(Listener), - ListenOnStr = emqx_listeners:format_listen_on(ListenOn), - case emqx_listeners:stop_listener(Listener) of - ok -> - emqx_ctl:print("Stop ~s listener on ~s successfully.~n", [ID, ListenOnStr]); - {error, Reason} -> - emqx_ctl:print("Failed to stop ~s listener on ~s: ~0p~n", - [ID, ListenOnStr, Reason]) - end. - %%-------------------------------------------------------------------- -%% @doc acl Command +%% @doc authz Command -acl(["cache-clean", "node", Node]) -> - case emqx_mgmt:clean_acl_cache_all(erlang:list_to_existing_atom(Node)) of +authz(["cache-clean", "node", Node]) -> + case emqx_mgmt:clean_authz_cache_all(erlang:list_to_existing_atom(Node)) of ok -> - emqx_ctl:print("ACL cache drain started on node ~s.~n", [Node]); + emqx_ctl:print("Authorization cache drain started on node ~s.~n", [Node]); {error, Reason} -> - emqx_ctl:print("ACL drain failed on node ~s: ~0p.~n", [Node, Reason]) + emqx_ctl:print("Authorization drain failed on node ~s: ~0p.~n", [Node, Reason]) end; -acl(["cache-clean", "all"]) -> - case emqx_mgmt:clean_acl_cache_all() of +authz(["cache-clean", "all"]) -> + case emqx_mgmt:clean_authz_cache_all() of ok -> - emqx_ctl:print("Started ACL cache drain in all nodes~n"); + emqx_ctl:print("Started Authorization cache drain in all nodes~n"); {error, Reason} -> - emqx_ctl:print("ACL cache-clean failed: ~p.~n", [Reason]) + emqx_ctl:print("Authorization cache-clean failed: ~p.~n", [Reason]) end; -acl(["cache-clean", ClientId]) -> - emqx_mgmt:clean_acl_cache(ClientId); +authz(["cache-clean", ClientId]) -> + emqx_mgmt:clean_authz_cache(ClientId); -acl(_) -> - emqx_ctl:usage([{"acl cache-clean all", "Clears acl cache on all nodes"}, - {"acl cache-clean node ", "Clears acl cache on given node"}, - {"acl cache-clean ", "Clears acl cache for given client"} +authz(_) -> + emqx_ctl:usage([{"authz cache-clean all", "Clears authorization cache on all nodes"}, + {"authz cache-clean node ", "Clears authorization cache on given node"}, + {"authz cache-clean ", "Clears authorization cache for given client"} ]). %%-------------------------------------------------------------------- @@ -619,18 +590,21 @@ print({client, {ClientId, ChanPid}}) -> InfoKeys = [clientid, username, peername, clean_start, keepalive, expiry_interval, subscriptions_cnt, inflight_cnt, awaiting_rel_cnt, send_msg, mqueue_len, mqueue_dropped, - connected, created_at, connected_at] ++ case maps:is_key(disconnected_at, Info) of - true -> [disconnected_at]; - false -> [] - end, + connected, created_at, connected_at] ++ + case maps:is_key(disconnected_at, Info) of + true -> [disconnected_at]; + false -> [] + end, + Info1 = Info#{expiry_interval => maps:get(expiry_interval, Info) div 1000}, emqx_ctl:print("Client(~s, username=~s, peername=~s, " "clean_start=~s, keepalive=~w, session_expiry_interval=~w, " "subscriptions=~w, inflight=~w, awaiting_rel=~w, delivered_msgs=~w, enqueued_msgs=~w, dropped_msgs=~w, " - "connected=~s, created_at=~w, connected_at=~w" ++ case maps:is_key(disconnected_at, Info) of - true -> ", disconnected_at=~w)~n"; - false -> ")~n" - end, - [format(K, maps:get(K, Info)) || K <- InfoKeys]); + "connected=~s, created_at=~w, connected_at=~w" ++ + case maps:is_key(disconnected_at, Info1) of + true -> ", disconnected_at=~w)~n"; + false -> ")~n" + end, + [format(K, maps:get(K, Info1)) || K <- InfoKeys]); print({emqx_route, #route{topic = Topic, dest = {_, Node}}}) -> emqx_ctl:print("~s -> ~s~n", [Topic, Node]); @@ -661,24 +635,9 @@ indent_print({Key, {string, Val}}) -> indent_print({Key, Val}) -> emqx_ctl:print(" ~-16s: ~w~n", [Key, Val]). -listener_identifier(Protocol, ListenOn) -> - case emqx_listeners:find_id_by_listen_on(ListenOn) of - false -> - atom_to_list(Protocol); - ID -> - ID - end. - -restart_http_listener(Scheme, AppName) -> - Listeners = application:get_env(AppName, listeners, []), - case lists:keyfind(Scheme, 1, Listeners) of - false -> - emqx_ctl:print("Listener ~s not exists!~n", [AppName]); - {Scheme, Port, Options} -> - ModName = http_mod_name(AppName), - ModName:stop_listener({Scheme, Port, Options}), - ModName:start_listener({Scheme, Port, Options}) - end. - -http_mod_name(emqx_management) -> emqx_mgmt_http; -http_mod_name(Name) -> Name. +format_listen_on(Port) when is_integer(Port) -> + io_lib:format("0.0.0.0:~w", [Port]); +format_listen_on({Addr, Port}) when is_list(Addr) -> + io_lib:format("~s:~w", [Addr, Port]); +format_listen_on({Addr, Port}) when is_tuple(Addr) -> + io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). diff --git a/apps/emqx_management/src/emqx_mgmt_http.erl b/apps/emqx_management/src/emqx_mgmt_http.erl index 4b927a982..85165412c 100644 --- a/apps/emqx_management/src/emqx_mgmt_http.erl +++ b/apps/emqx_management/src/emqx_mgmt_http.erl @@ -13,27 +13,21 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- - -module(emqx_mgmt_http). -export([ start_listeners/0 - , handle_request/2 , stop_listeners/0 , start_listener/1 - , stop_listener/1 - ]). + , stop_listener/1]). --export([init/2]). +%% Authorization +-export([authorize_appid/1]). -include_lib("emqx/include/emqx.hrl"). -define(APP, emqx_management). --define(EXCEPT_PLUGIN, [emqx_dashboard]). --ifdef(TEST). --define(EXCEPT, []). --else. --define(EXCEPT, [add_app, del_app, list_apps, lookup_app, update_app]). --endif. + +-define(BASE_PATH, "/api/v5"). %%-------------------------------------------------------------------- %% Start/Stop Listeners @@ -45,83 +39,66 @@ start_listeners() -> stop_listeners() -> lists:foreach(fun stop_listener/1, listeners()). -start_listener({Proto, Port, Options}) when Proto == http -> - Dispatch = [{"/status", emqx_mgmt_http, []}, - {"/api/v4/[...]", minirest, http_handlers()}], - minirest:start_http(listener_name(Proto), ranch_opts(Port, Options), Dispatch); - -start_listener({Proto, Port, Options}) when Proto == https -> - Dispatch = [{"/status", emqx_mgmt_http, []}, - {"/api/v4/[...]", minirest, http_handlers()}], - minirest:start_https(listener_name(Proto), ranch_opts(Port, Options), Dispatch). +start_listener({Proto, Port, Options}) -> + {ok, _} = application:ensure_all_started(minirest), + Authorization = {?MODULE, authorize_appid}, + RanchOptions = ranch_opts(Port, Options), + GlobalSpec = #{ + openapi => "3.0.0", + info => #{title => "EMQ X API", version => "5.0.0"}, + servers => [#{url => ?BASE_PATH}], + components => #{ + schemas => #{}, + securitySchemes => #{ + application => #{ + type => apiKey, + name => "authorization", + in => header}}}}, + Minirest = #{ + protocol => Proto, + base_path => ?BASE_PATH, + modules => api_modules(), + authorization => Authorization, + security => [#{application => []}], + swagger_global_spec => GlobalSpec}, + MinirestOptions = maps:merge(Minirest, RanchOptions), + {ok, _} = minirest:start(listener_name(Proto), MinirestOptions), + io:format("Start ~p listener on ~p successfully.~n", [listener_name(Proto), Port]). ranch_opts(Port, Options0) -> - NumAcceptors = proplists:get_value(num_acceptors, Options0, 4), - MaxConnections = proplists:get_value(max_connections, Options0, 512), - Options = lists:foldl(fun({K, _V}, Acc) when K =:= max_connections orelse K =:= num_acceptors -> - Acc; - ({inet6, true}, Acc) -> [inet6 | Acc]; - ({inet6, false}, Acc) -> Acc; - ({ipv6_v6only, true}, Acc) -> [{ipv6_v6only, true} | Acc]; - ({ipv6_v6only, false}, Acc) -> Acc; - ({K, V}, Acc)-> - [{K, V} | Acc] - end, [], Options0), - - Res = #{num_acceptors => NumAcceptors, - max_connections => MaxConnections, - socket_opts => [{port, Port} | Options]}, - Res. + Options = lists:foldl( + fun + ({K, _V}, Acc) when K =:= max_connections orelse K =:= num_acceptors -> Acc; + ({inet6, true}, Acc) -> [inet6 | Acc]; + ({inet6, false}, Acc) -> Acc; + ({ipv6_v6only, true}, Acc) -> [{ipv6_v6only, true} | Acc]; + ({ipv6_v6only, false}, Acc) -> Acc; + ({K, V}, Acc)-> + [{K, V} | Acc] + end, [], Options0), + maps:from_list([{port, Port} | Options]). stop_listener({Proto, Port, _}) -> io:format("Stop http:management listener on ~s successfully.~n",[format(Port)]), - minirest:stop_http(listener_name(Proto)). + minirest:stop(listener_name(Proto)). listeners() -> - application:get_env(?APP, listeners, []). + [{Protocol, Port, maps:to_list(maps:without([protocol, port], Map))} + || Map = #{protocol := Protocol,port := Port} + <- emqx_config:get([emqx_management, listeners], [])]. listener_name(Proto) -> list_to_atom(atom_to_list(Proto) ++ ":management"). -http_handlers() -> - Apps = [ App || {App, _, _} <- application:loaded_applications(), - case re:run(atom_to_list(App), "^emqx") of - {match,[{0,4}]} -> true; - _ -> false - end], - Plugins = lists:map(fun(Plugin) -> Plugin#plugin.name end, emqx_plugins:list()), - [{"/api/v4", minirest:handler(#{apps => Plugins ++ Apps -- ?EXCEPT_PLUGIN, - except => ?EXCEPT, - filter => fun(_) -> true end}), - [{authorization, fun authorize_appid/1}]}]. - -%%-------------------------------------------------------------------- -%% Handle 'status' request -%%-------------------------------------------------------------------- -init(Req, Opts) -> - Req1 = handle_request(cowboy_req:path(Req), Req), - {ok, Req1, Opts}. - -handle_request(Path, Req) -> - handle_request(cowboy_req:method(Req), Path, Req). - -handle_request(<<"GET">>, <<"/status">>, Req) -> - {InternalStatus, _ProvidedStatus} = init:get_status(), - AppStatus = case lists:keysearch(emqx, 1, application:which_applications()) of - false -> not_running; - {value, _Val} -> running - end, - Status = io_lib:format("Node ~s is ~s~nemqx is ~s", - [node(), InternalStatus, AppStatus]), - cowboy_req:reply(200, #{<<"content-type">> => <<"text/plain">>}, Status, Req); - -handle_request(_Method, _Path, Req) -> - cowboy_req:reply(400, #{<<"content-type">> => <<"text/plain">>}, <<"Not found.">>, Req). - authorize_appid(Req) -> case cowboy_req:parse_header(<<"authorization">>, Req) of - {basic, AppId, AppSecret} -> emqx_mgmt_auth:is_authorized(AppId, AppSecret); - _ -> false + {basic, AppId, AppSecret} -> + case emqx_mgmt_auth:is_authorized(AppId, AppSecret) of + true -> ok; + false -> {401, #{<<"WWW-Authenticate">> => <<"Basic Realm=\"minirest-server\"">>}, <<"UNAUTHORIZED">>} + end; + _ -> + {401, #{<<"WWW-Authenticate">> => <<"Basic Realm=\"minirest-server\"">>}, <<"UNAUTHORIZED">>} end. format(Port) when is_integer(Port) -> @@ -130,3 +107,20 @@ format({Addr, Port}) when is_list(Addr) -> io_lib:format("~s:~w", [Addr, Port]); format({Addr, Port}) when is_tuple(Addr) -> io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). + +apps() -> + Apps = [App || {App, _, _} <- application:loaded_applications(), App =/= emqx_dashboard], + lists:filter(fun(App) -> + case re:run(atom_to_list(App), "^emqx") of + {match,[{0,4}]} -> true; + _ -> false + end + end, Apps). + +-ifdef(TEST). +api_modules() -> + minirest_api:find_api_modules(apps()). +-else. +api_modules() -> + minirest_api:find_api_modules(apps()) -- [emqx_mgmt_api_apps]. +-endif. diff --git a/apps/emqx_management/src/emqx_mgmt_util.erl b/apps/emqx_management/src/emqx_mgmt_util.erl index 132bbc83f..7d5f85cf2 100644 --- a/apps/emqx_management/src/emqx_mgmt_util.erl +++ b/apps/emqx_management/src/emqx_mgmt_util.erl @@ -21,8 +21,19 @@ , kmg/1 , ntoa/1 , merge_maps/2 + , batch_operation/3 ]). +-export([ request_body_schema/1 + , request_body_array_schema/1 + , response_schema/1 + , response_schema/2 + , response_array_schema/2 + , response_error_schema/1 + , response_error_schema/2 + , response_page_schema/1 + , response_batch_schema/1]). + -export([urldecode/1]). -define(KB, 1024). @@ -77,3 +88,128 @@ merge_maps(Default, New) -> urldecode(S) -> emqx_http_lib:uri_decode(S). +%%%============================================================================================== +%% schema util + +request_body_array_schema(Schema) when is_map(Schema) -> + json_content_schema("", #{type => array, items => Schema}); +request_body_array_schema(Ref) when is_atom(Ref) -> + request_body_array_schema(atom_to_binary(Ref, utf8)); +request_body_array_schema(Ref) when is_binary(Ref) -> + json_content_schema("", #{type => array, items => minirest:ref(Ref)}). + +request_body_schema(Schema) when is_map(Schema) -> + json_content_schema("", Schema); +request_body_schema(Ref) when is_atom(Ref) -> + request_body_schema(atom_to_binary(Ref)); +request_body_schema(Ref) when is_binary(Ref) -> + json_content_schema("", minirest:ref(Ref)). + +response_array_schema(Description, Schema) when is_map(Schema) -> + json_content_schema(Description, #{type => array, items => Schema}); +response_array_schema(Description, Ref) when is_atom(Ref) -> + response_array_schema(Description, atom_to_binary(Ref, utf8)); +response_array_schema(Description, Ref) when is_binary(Ref) -> + json_content_schema(Description, #{type => array, items => minirest:ref(Ref)}). + +response_schema(Description) -> + json_content_schema(Description). + +response_schema(Description, Schema) when is_map(Schema) -> + json_content_schema(Description, Schema); +response_schema(Description, Ref) when is_atom(Ref) -> + response_schema(Description, atom_to_binary(Ref, utf8)); +response_schema(Description, Ref) when is_binary(Ref) -> + json_content_schema(Description, minirest:ref(Ref)). + +%% @doc default code is RESOURCE_NOT_FOUND +response_error_schema(Description) -> + response_error_schema(Description, ['RESOURCE_NOT_FOUND']). + +response_error_schema(Description, Enum) -> + Schema = #{ + type => object, + properties => #{ + code => #{ + type => string, + enum => Enum}, + message => #{ + type => string}}}, + json_content_schema(Description, Schema). + +response_page_schema(Def) when is_atom(Def) -> + response_page_schema(atom_to_binary(Def, utf8)); +response_page_schema(Def) when is_binary(Def) -> + Schema = #{ + type => object, + properties => #{ + meta => #{ + type => object, + properties => #{ + page => #{ + type => integer}, + limit => #{ + type => integer}, + count => #{ + type => integer}}}, + data => #{ + type => array, + items => minirest:ref(Def)}}}, + json_content_schema("", Schema). + +response_batch_schema(DefName) when is_atom(DefName) -> + response_batch_schema(atom_to_binary(DefName, utf8)); +response_batch_schema(DefName) when is_binary(DefName) -> + Schema = #{ + type => object, + properties => #{ + success => #{ + type => integer, + description => <<"Success count">>}, + failed => #{ + type => integer, + description => <<"Failed count">>}, + detail => #{ + type => array, + description => <<"Failed object & reason">>, + items => #{ + type => object, + properties => + #{ + data => minirest:ref(DefName), + reason => #{ + type => <<"string">>}}}}}}, + json_content_schema("", Schema). + +json_content_schema(Description, Schema) -> + Content = + #{content => #{ + 'application/json' => #{ + schema => Schema}}}, + case Description of + "" -> + Content; + _ -> + maps:merge(#{description => Description}, Content) + end. + +json_content_schema(Description) -> + #{description => Description}. + +%%%============================================================================================== +batch_operation(Module, Function, ArgsList) -> + Failed = batch_operation(Module, Function, ArgsList, []), + Len = erlang:length(Failed), + Success = erlang:length(ArgsList) - Len, + Fun = fun({Args, Reason}, Detail) -> [#{data => Args, reason => io_lib:format("~p", [Reason])} | Detail] end, + #{success => Success, failed => Len, detail => lists:foldl(Fun, [], Failed)}. + +batch_operation(_Module, _Function, [], Failed) -> + lists:reverse(Failed); +batch_operation(Module, Function, [Args | ArgsList], Failed) -> + case erlang:apply(Module, Function, Args) of + ok -> + batch_operation(Module, Function, ArgsList, Failed); + {error ,Reason} -> + batch_operation(Module, Function, ArgsList, [{Args, Reason} | Failed]) + end. diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl deleted file mode 100644 index 84d5f2ebb..000000000 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ /dev/null @@ -1,369 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_mgmt_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("eunit/include/eunit.hrl"). - --include_lib("common_test/include/ct.hrl"). - --define(LOG_LEVELS, ["debug", "error", "info"]). --define(LOG_HANDLER_ID, [file, default]). - -all() -> - [{group, manage_apps}, - {group, check_cli}]. - -groups() -> - [{manage_apps, [sequence], - [t_app - ]}, - {check_cli, [sequence], - [t_cli, - t_log_cmd, - t_mgmt_cmd, - t_status_cmd, - t_clients_cmd, - t_vm_cmd, - t_plugins_cmd, - t_trace_cmd, - t_broker_cmd, - t_router_cmd, - t_subscriptions_cmd, - t_listeners_cmd_old, - t_listeners_cmd_new - ]}]. - -apps() -> - [emqx_management, emqx_retainer, emqx_modules]. - -init_per_suite(Config) -> - application:set_env(ekka, strict_mode, true), - ekka_mnesia:start(), - emqx_mgmt_auth:mnesia(boot), - emqx_ct_helpers:start_apps(apps()), - Config. - -end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps(apps()). - -t_app(_Config) -> - {ok, AppSecret} = emqx_mgmt_auth:add_app(<<"app_id">>, <<"app_name">>), - ?assert(emqx_mgmt_auth:is_authorized(<<"app_id">>, AppSecret)), - ?assertEqual(AppSecret, emqx_mgmt_auth:get_appsecret(<<"app_id">>)), - ?assertEqual({<<"app_id">>, AppSecret, - <<"app_name">>, <<"Application user">>, - true, undefined}, - lists:keyfind(<<"app_id">>, 1, emqx_mgmt_auth:list_apps())), - emqx_mgmt_auth:del_app(<<"app_id">>), - %% Use the default application secret - application:set_env(emqx_management, application, [{default_secret, <<"public">>}]), - {ok, AppSecret1} = emqx_mgmt_auth:add_app( - <<"app_id">>, <<"app_name">>, <<"app_desc">>, true, undefined), - ?assert(emqx_mgmt_auth:is_authorized(<<"app_id">>, AppSecret1)), - ?assertEqual(AppSecret1, emqx_mgmt_auth:get_appsecret(<<"app_id">>)), - ?assertEqual(AppSecret1, <<"public">>), - ?assertEqual({<<"app_id">>, AppSecret1, <<"app_name">>, <<"app_desc">>, true, undefined}, - lists:keyfind(<<"app_id">>, 1, emqx_mgmt_auth:list_apps())), - emqx_mgmt_auth:del_app(<<"app_id">>), - application:set_env(emqx_management, application, []), - %% Specify the application secret - {ok, AppSecret2} = emqx_mgmt_auth:add_app( - <<"app_id">>, <<"app_name">>, <<"secret">>, - <<"app_desc">>, true, undefined), - ?assert(emqx_mgmt_auth:is_authorized(<<"app_id">>, AppSecret2)), - ?assertEqual(AppSecret2, emqx_mgmt_auth:get_appsecret(<<"app_id">>)), - ?assertEqual({<<"app_id">>, AppSecret2, <<"app_name">>, <<"app_desc">>, true, undefined}, - lists:keyfind(<<"app_id">>, 1, emqx_mgmt_auth:list_apps())), - emqx_mgmt_auth:del_app(<<"app_id">>), - ok. - -t_log_cmd(_) -> - mock_print(), - lists:foreach(fun(Level) -> - emqx_mgmt_cli:log(["primary-level", Level]), - ?assertEqual(Level ++ "\n", emqx_mgmt_cli:log(["primary-level"])) - end, ?LOG_LEVELS), - lists:foreach(fun(Level) -> - emqx_mgmt_cli:log(["set-level", Level]), - ?assertEqual(Level ++ "\n", emqx_mgmt_cli:log(["primary-level"])) - end, ?LOG_LEVELS), - [lists:foreach(fun(Level) -> - ?assertEqual(Level ++ "\n", emqx_mgmt_cli:log(["handlers", "set-level", - atom_to_list(Id), Level])) - end, ?LOG_LEVELS) - || #{id := Id} <- emqx_logger:get_log_handlers()], - meck:unload(). - -t_mgmt_cmd(_) -> - % ct:pal("start testing the mgmt command"), - mock_print(), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( - ["lookup", "emqx_appid"]), "Not Found.")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( - ["insert", "emqx_appid", "emqx_name"]), "AppSecret:")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( - ["insert", "emqx_appid", "emqx_name"]), "Error:")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( - ["lookup", "emqx_appid"]), "app_id:")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( - ["update", "emqx_appid", "ts"]), "update successfully")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( - ["delete", "emqx_appid"]), "ok")), - ok = emqx_mgmt_cli:mgmt(["list"]), - meck:unload(). - -t_status_cmd(_) -> - % ct:pal("start testing status command"), - mock_print(), - %% init internal status seem to be always 'starting' when running ct tests - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:status([]), "Node\s.*@.*\sis\sstart(ed|ing)")), - meck:unload(). - -t_broker_cmd(_) -> - % ct:pal("start testing the broker command"), - mock_print(), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker([]), "sysdescr")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker(["stats"]), "subscriptions.shared")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker(["metrics"]), "bytes.sent")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker([undefined]), "broker")), - meck:unload(). - -t_clients_cmd(_) -> - % ct:pal("start testing the client command"), - mock_print(), - process_flag(trap_exit, true), - {ok, T} = emqtt:start_link([{clientid, <<"client12">>}, - {username, <<"testuser1">>}, - {password, <<"pass1">>} - ]), - {ok, _} = emqtt:connect(T), - timer:sleep(300), - emqx_mgmt_cli:clients(["list"]), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:clients(["show", "client12"]), "client12")), - ?assertEqual((emqx_mgmt_cli:clients(["kick", "client12"])), "ok~n"), - timer:sleep(500), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:clients(["show", "client12"]), "Not Found")), - receive - {'EXIT', T, _} -> - ok - % ct:pal("Connection closed: ~p~n", [Reason]) - after - 500 -> - erlang:error("Client is not kick") - end, - WS = rfc6455_client:new("ws://127.0.0.1:8083" ++ "/mqtt", self()), - {ok, _} = rfc6455_client:open(WS), - Packet = raw_send_serialize(?CONNECT_PACKET(#mqtt_packet_connect{ - clientid = <<"client13">>})), - ok = rfc6455_client:send_binary(WS, Packet), - Connack = ?CONNACK_PACKET(?CONNACK_ACCEPT), - {binary, Bin} = rfc6455_client:recv(WS), - {ok, Connack, <<>>, _} = raw_recv_pase(Bin), - timer:sleep(300), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:clients(["show", "client13"]), "client13")), - meck:unload(). - % emqx_mgmt_cli:clients(["kick", "client13"]), - % timer:sleep(500), - % ?assertMatch({match, _}, re:run(emqx_mgmt_cli:clients(["show", "client13"]), "Not Found")). - -raw_recv_pase(Packet) -> - emqx_frame:parse(Packet). - -raw_send_serialize(Packet) -> - emqx_frame:serialize(Packet). - -t_vm_cmd(_) -> - % ct:pal("start testing the vm command"), - mock_print(), - [[?assertMatch({match, _}, re:run(Result, Name)) - || Result <- emqx_mgmt_cli:vm([Name])] - || Name <- ["load", "memory", "process", "io", "ports"]], - [?assertMatch({match, _}, re:run(Result, "load")) - || Result <- emqx_mgmt_cli:vm(["load"])], - [?assertMatch({match, _}, re:run(Result, "memory")) - || Result <- emqx_mgmt_cli:vm(["memory"])], - [?assertMatch({match, _}, re:run(Result, "process")) - || Result <- emqx_mgmt_cli:vm(["process"])], - [?assertMatch({match, _}, re:run(Result, "io")) - || Result <- emqx_mgmt_cli:vm(["io"])], - [?assertMatch({match, _}, re:run(Result, "ports")) - || Result <- emqx_mgmt_cli:vm(["ports"])], - unmock_print(). - -t_trace_cmd(_) -> - % ct:pal("start testing the trace command"), - mock_print(), - logger:set_primary_config(level, debug), - {ok, T} = emqtt:start_link([{clientid, <<"client">>}, - {username, <<"testuser">>}, - {password, <<"pass">>} - ]), - emqtt:connect(T), - emqtt:subscribe(T, <<"a/b/c">>), - Trace1 = emqx_mgmt_cli:trace(["start", "client", "client", - "log/clientid_trace.log"]), - ?assertMatch({match, _}, re:run(Trace1, "successfully")), - Trace2 = emqx_mgmt_cli:trace(["stop", "client", "client"]), - ?assertMatch({match, _}, re:run(Trace2, "successfully")), - Trace3 = emqx_mgmt_cli:trace(["start", "client", "client", - "log/clientid_trace.log", - "error"]), - ?assertMatch({match, _}, re:run(Trace3, "successfully")), - Trace4 = emqx_mgmt_cli:trace(["stop", "client", "client"]), - ?assertMatch({match, _}, re:run(Trace4, "successfully")), - Trace5 = emqx_mgmt_cli:trace(["start", "topic", "a/b/c", - "log/clientid_trace.log"]), - ?assertMatch({match, _}, re:run(Trace5, "successfully")), - Trace6 = emqx_mgmt_cli:trace(["stop", "topic", "a/b/c"]), - ?assertMatch({match, _}, re:run(Trace6, "successfully")), - Trace7 = emqx_mgmt_cli:trace(["start", "topic", "a/b/c", - "log/clientid_trace.log", "error"]), - ?assertMatch({match, _}, re:run(Trace7, "successfully")), - logger:set_primary_config(level, error), - unmock_print(). - -t_router_cmd(_) -> - % ct:pal("start testing the router command"), - mock_print(), - {ok, T} = emqtt:start_link([{clientid, <<"client1">>}, - {username, <<"testuser1">>}, - {password, <<"pass1">>} - ]), - emqtt:connect(T), - emqtt:subscribe(T, <<"a/b/c">>), - {ok, T1} = emqtt:start_link([{clientid, <<"client2">>}, - {username, <<"testuser2">>}, - {password, <<"pass2">>} - ]), - - emqtt:connect(T1), - emqtt:subscribe(T1, <<"a/b/c/d">>), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:routes(["list"]), "a/b/c | a/b/c")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:routes(["show", "a/b/c"]), "a/b/c")), - unmock_print(). - -t_subscriptions_cmd(_) -> - % ct:pal("Start testing the subscriptions command"), - mock_print(), - {ok, T3} = emqtt:start_link([{clientid, <<"client">>}, - {username, <<"testuser">>}, - {password, <<"pass">>} - ]), - {ok, _} = emqtt:connect(T3), - {ok, _, _} = emqtt:subscribe(T3, <<"b/b/c">>), - timer:sleep(300), - [?assertMatch({match, _} , re:run(Result, "b/b/c")) - || Result <- emqx_mgmt_cli:subscriptions(["show", <<"client">>])], - ?assertEqual(emqx_mgmt_cli:subscriptions(["add", "client", "b/b/c", "0"]), "ok~n"), - ?assertEqual(emqx_mgmt_cli:subscriptions(["del", "client", "b/b/c"]), "ok~n"), - unmock_print(). - -t_listeners_cmd_old(_) -> - ok = emqx_listeners:ensure_all_started(), - mock_print(), - ?assertEqual(emqx_mgmt_cli:listeners([]), ok), - ?assertEqual( - "Stop mqtt:wss:external listener on 0.0.0.0:8084 successfully.\n", - emqx_mgmt_cli:listeners(["stop", "wss", "8084"]) - ), - unmock_print(). - -t_listeners_cmd_new(_) -> - ok = emqx_listeners:ensure_all_started(), - mock_print(), - ?assertEqual(emqx_mgmt_cli:listeners([]), ok), - ?assertEqual( - "Stop mqtt:wss:external listener on 0.0.0.0:8084 successfully.\n", - emqx_mgmt_cli:listeners(["stop", "mqtt:wss:external"]) - ), - ?assertEqual( - emqx_mgmt_cli:listeners(["restart", "mqtt:tcp:external"]), - "Restarted mqtt:tcp:external listener successfully.\n" - ), - ?assertEqual( - emqx_mgmt_cli:listeners(["restart", "mqtt:ssl:external"]), - "Restarted mqtt:ssl:external listener successfully.\n" - ), - ?assertEqual( - emqx_mgmt_cli:listeners(["restart", "bad:listener:identifier"]), - "Failed to restart bad:listener:identifier listener: {no_such_listener,\"bad:listener:identifier\"}\n" - ), - unmock_print(). - -t_plugins_cmd(_) -> - mock_print(), - meck:new(emqx_plugins, [non_strict, passthrough]), - meck:expect(emqx_plugins, load, fun(_) -> ok end), - meck:expect(emqx_plugins, unload, fun(_) -> ok end), - meck:expect(emqx_plugins, reload, fun(_) -> ok end), - ?assertEqual(emqx_mgmt_cli:plugins(["list"]), ok), - ?assertEqual( - emqx_mgmt_cli:plugins(["unload", "emqx_retainer"]), - "Plugin emqx_retainer unloaded successfully.\n" - ), - ?assertEqual( - emqx_mgmt_cli:plugins(["load", "emqx_retainer"]), - "Plugin emqx_retainer loaded successfully.\n" - ), - ?assertEqual( - emqx_mgmt_cli:plugins(["unload", "emqx_management"]), - "Plugin emqx_management can not be unloaded.~n" - ), - unmock_print(). - -t_cli(_) -> - mock_print(), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:status([""]), "status")), - [?assertMatch({match, _}, re:run(Value, "broker")) - || Value <- emqx_mgmt_cli:broker([""])], - [?assertMatch({match, _}, re:run(Value, "cluster")) - || Value <- emqx_mgmt_cli:cluster([""])], - [?assertMatch({match, _}, re:run(Value, "clients")) - || Value <- emqx_mgmt_cli:clients([""])], - [?assertMatch({match, _}, re:run(Value, "routes")) - || Value <- emqx_mgmt_cli:routes([""])], - [?assertMatch({match, _}, re:run(Value, "subscriptions")) - || Value <- emqx_mgmt_cli:subscriptions([""])], - [?assertMatch({match, _}, re:run(Value, "plugins")) - || Value <- emqx_mgmt_cli:plugins([""])], - [?assertMatch({match, _}, re:run(Value, "listeners")) - || Value <- emqx_mgmt_cli:listeners([""])], - [?assertMatch({match, _}, re:run(Value, "vm")) - || Value <- emqx_mgmt_cli:vm([""])], - [?assertMatch({match, _}, re:run(Value, "mnesia")) - || Value <- emqx_mgmt_cli:mnesia([""])], - [?assertMatch({match, _}, re:run(Value, "trace")) - || Value <- emqx_mgmt_cli:trace([""])], - [?assertMatch({match, _}, re:run(Value, "mgmt")) - || Value <- emqx_mgmt_cli:mgmt([""])], - unmock_print(). - -mock_print() -> - catch meck:unload(emqx_ctl), - meck:new(emqx_ctl, [non_strict, passthrough]), - meck:expect(emqx_ctl, print, fun(Arg) -> emqx_ctl:format(Arg) end), - meck:expect(emqx_ctl, print, fun(Msg, Arg) -> emqx_ctl:format(Msg, Arg) end), - meck:expect(emqx_ctl, usage, fun(Usages) -> emqx_ctl:format_usage(Usages) end), - meck:expect(emqx_ctl, usage, fun(Cmd, Descr) -> emqx_ctl:format_usage(Cmd, Descr) end). - -unmock_print() -> - meck:unload(emqx_ctl). diff --git a/apps/emqx_management/test/emqx_mgmt_alarms_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_alarms_api_SUITE.erl new file mode 100644 index 000000000..2929e961d --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_alarms_api_SUITE.erl @@ -0,0 +1,73 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_alarms_api_SUITE). + + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-define(ACT_ALARM, test_act_alarm). +-define(DE_ACT_ALARM, test_de_act_alarm). + +all() -> + [t_alarms_api, t_delete_alarms_api]. + +init_per_suite(Config) -> + ekka_mnesia:start(), + emqx_mgmt_auth:mnesia(boot), + emqx_ct_helpers:start_apps([emqx_management], fun set_special_configs/1), + Config. + +end_per_suite(_) -> + emqx_ct_helpers:stop_apps([emqx_management]). + +set_special_configs(emqx_management) -> + emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + applications =>[#{id => "admin", secret => "public"}]}), + ok; +set_special_configs(_App) -> + ok. + +t_alarms_api(_) -> + ok = emqx_alarm:activate(?ACT_ALARM), + ok = emqx_alarm:activate(?DE_ACT_ALARM), + ok = emqx_alarm:deactivate(?DE_ACT_ALARM), + get_alarms(1, true), + get_alarms(1, false). + +t_delete_alarms_api(_) -> + Path = emqx_mgmt_api_test_util:api_path(["alarms"]), + {ok, _} = emqx_mgmt_api_test_util:request_api(delete, Path), + get_alarms(1, true), + get_alarms(0, false). + +get_alarms(AssertCount, Activated) when is_atom(Activated) -> + get_alarms(AssertCount, atom_to_list(Activated)); +get_alarms(AssertCount, Activated) -> + Path = emqx_mgmt_api_test_util:api_path(["alarms"]), + Qs = "activated=" ++ Activated, + Headers = emqx_mgmt_api_test_util:auth_header_(), + {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, Qs, Headers), + Data = emqx_json:decode(Response, [return_maps]), + Meta = maps:get(<<"meta">>, Data), + Page = maps:get(<<"page">>, Meta), + Limit = maps:get(<<"limit">>, Meta), + Count = maps:get(<<"count">>, Meta), + ?assertEqual(Page, 1), + ?assertEqual(Limit, emqx_mgmt:max_row_limit()), + ?assert(Count >= AssertCount). diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl deleted file mode 100644 index d372596ed..000000000 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ /dev/null @@ -1,586 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_mgmt_api_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). - --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("emqx_management/include/emqx_mgmt.hrl"). - --define(CONTENT_TYPE, "application/x-www-form-urlencoded"). - --define(HOST, "http://127.0.0.1:8081/"). - --define(API_VERSION, "v4"). - --define(BASE_PATH, "api"). - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - application:load(emqx_modules), - emqx_ct_helpers:start_apps([emqx_management]), - Config. - -end_per_suite(Config) -> - emqx_ct_helpers:stop_apps([emqx_management]), - Config. - -init_per_testcase(_, Config) -> - Config. - -end_per_testcase(_, Config) -> - Config. - -get(Key, ResponseBody) -> - maps:get(Key, jiffy:decode(list_to_binary(ResponseBody), [return_maps])). - -lookup_alarm(Name, [#{<<"name">> := Name} | _More]) -> - true; -lookup_alarm(Name, [_Alarm | More]) -> - lookup_alarm(Name, More); -lookup_alarm(_Name, []) -> - false. - -is_existing(Name, [#{name := Name} | _More]) -> - true; -is_existing(Name, [_Alarm | More]) -> - is_existing(Name, More); -is_existing(_Name, []) -> - false. - -t_alarms(_) -> - emqx_alarm:activate(alarm1), - emqx_alarm:activate(alarm2), - - ?assert(is_existing(alarm1, emqx_alarm:get_alarms(activated))), - ?assert(is_existing(alarm2, emqx_alarm:get_alarms(activated))), - - {ok, Return1} = request_api(get, api_path(["alarms/activated"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return1))))), - ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return1))))), - - emqx_alarm:deactivate(alarm1), - - {ok, Return2} = request_api(get, api_path(["alarms"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return2))))), - ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return2))))), - - {ok, Return3} = request_api(get, api_path(["alarms/deactivated"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return3))))), - ?assertNot(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return3))))), - - emqx_alarm:deactivate(alarm2), - - {ok, Return4} = request_api(get, api_path(["alarms/deactivated"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return4))))), - ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return4))))), - - {ok, _} = request_api(delete, api_path(["alarms/deactivated"]), auth_header_()), - - {ok, Return5} = request_api(get, api_path(["alarms/deactivated"]), auth_header_()), - ?assertNot(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return5))))), - ?assertNot(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return5))))). - -t_apps(_) -> - AppId = <<"123456">>, - meck:new(emqx_mgmt_auth, [passthrough, no_history]), - meck:expect(emqx_mgmt_auth, add_app, 6, fun(_, _, _, _, _, _) -> {error, undefined} end), - {ok, Error1} = request_api(post, api_path(["apps"]), [], - auth_header_(), #{<<"app_id">> => AppId, - <<"name">> => <<"test">>, - <<"status">> => true}), - ?assertMatch(<<"undefined">>, get(<<"message">>, Error1)), - - meck:expect(emqx_mgmt_auth, del_app, 1, fun(_) -> {error, undefined} end), - {ok, Error2} = request_api(delete, api_path(["apps", binary_to_list(AppId)]), auth_header_()), - ?assertMatch(<<"undefined">>, get(<<"message">>, Error2)), - meck:unload(emqx_mgmt_auth), - - {ok, NoApp} = request_api(get, api_path(["apps", binary_to_list(AppId)]), auth_header_()), - ?assertEqual(0, maps:size(get(<<"data">>, NoApp))), - {ok, NotFound} = request_api(put, api_path(["apps", binary_to_list(AppId)]), [], - auth_header_(), #{<<"name">> => <<"test 2">>, - <<"status">> => true}), - ?assertEqual(<<"not_found">>, get(<<"message">>, NotFound)), - - {ok, _} = request_api(post, api_path(["apps"]), [], - auth_header_(), #{<<"app_id">> => AppId, - <<"name">> => <<"test">>, - <<"status">> => true}), - {ok, _} = request_api(get, api_path(["apps"]), auth_header_()), - {ok, _} = request_api(get, api_path(["apps", binary_to_list(AppId)]), auth_header_()), - {ok, _} = request_api(put, api_path(["apps", binary_to_list(AppId)]), [], - auth_header_(), #{<<"name">> => <<"test 2">>, - <<"status">> => true}), - {ok, AppInfo} = request_api(get, api_path(["apps", binary_to_list(AppId)]), auth_header_()), - ?assertEqual(<<"test 2">>, maps:get(<<"name">>, get(<<"data">>, AppInfo))), - {ok, _} = request_api(delete, api_path(["apps", binary_to_list(AppId)]), auth_header_()), - {ok, Result} = request_api(get, api_path(["apps"]), auth_header_()), - [App] = get(<<"data">>, Result), - ?assertEqual(<<"admin">>, maps:get(<<"app_id">>, App)). - -t_banned(_) -> - Who = <<"myclient">>, - {ok, _} = request_api(post, api_path(["banned"]), [], - auth_header_(), #{<<"who">> => Who, - <<"as">> => <<"clientid">>, - <<"reason">> => <<"test">>, - <<"by">> => <<"dashboard">>, - <<"at">> => erlang:system_time(second), - <<"until">> => erlang:system_time(second) + 10}), - - {ok, Result} = request_api(get, api_path(["banned"]), auth_header_()), - [Banned] = get(<<"data">>, Result), - ?assertEqual(Who, maps:get(<<"who">>, Banned)), - - {ok, _} = request_api(delete, api_path(["banned", "clientid", binary_to_list(Who)]), auth_header_()), - {ok, Result2} = request_api(get, api_path(["banned"]), auth_header_()), - ?assertEqual([], get(<<"data">>, Result2)). - -t_brokers(_) -> - {ok, _} = request_api(get, api_path(["brokers"]), auth_header_()), - {ok, _} = request_api(get, api_path(["brokers", atom_to_list(node())]), auth_header_()), - meck:new(emqx_mgmt, [passthrough, no_history]), - meck:expect(emqx_mgmt, lookup_broker, 1, fun(_) -> {error, undefined} end), - {ok, Error} = request_api(get, api_path(["brokers", atom_to_list(node())]), auth_header_()), - ?assertEqual(<<"undefined">>, get(<<"message">>, Error)), - meck:unload(emqx_mgmt). - -t_clients(_) -> - process_flag(trap_exit, true), - Username1 = <<"user1">>, - Username2 = <<"user2">>, - ClientId1 = <<"client1">>, - ClientId2 = <<"client2">>, - {ok, C1} = emqtt:start_link(#{username => Username1, clientid => ClientId1}), - {ok, _} = emqtt:connect(C1), - {ok, C2} = emqtt:start_link(#{username => Username2, clientid => ClientId2}), - {ok, _} = emqtt:connect(C2), - - timer:sleep(300), - - {ok, Clients1} = request_api(get, api_path(["clients", binary_to_list(ClientId1)]) - , auth_header_()), - ?assertEqual(<<"client1">>, maps:get(<<"clientid">>, lists:nth(1, get(<<"data">>, Clients1)))), - - {ok, Clients2} = request_api(get, api_path(["nodes", atom_to_list(node()), - "clients", binary_to_list(ClientId2)]) - , auth_header_()), - ?assertEqual(<<"client2">>, maps:get(<<"clientid">>, lists:nth(1, get(<<"data">>, Clients2)))), - - {ok, Clients3} = request_api(get, api_path(["clients", - "username", binary_to_list(Username1)]), - auth_header_()), - ?assertEqual(<<"client1">>, maps:get(<<"clientid">>, lists:nth(1, get(<<"data">>, Clients3)))), - - {ok, Clients4} = request_api(get, api_path(["nodes", atom_to_list(node()), - "clients", - "username", binary_to_list(Username2)]) - , auth_header_()), - ?assertEqual(<<"client2">>, maps:get(<<"clientid">>, lists:nth(1, get(<<"data">>, Clients4)))), - - {ok, Clients5} = request_api(get, api_path(["clients"]), "_limit=100&_page=1", auth_header_()), - ?assertEqual(2, maps:get(<<"count">>, get(<<"meta">>, Clients5))), - - meck:new(emqx_mgmt, [passthrough, no_history]), - meck:expect(emqx_mgmt, kickout_client, 1, fun(_) -> {error, undefined} end), - - {ok, MeckRet1} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), - ?assertEqual(?ERROR1, get(<<"code">>, MeckRet1)), - - meck:expect(emqx_mgmt, clean_acl_cache, 1, fun(_) -> {error, undefined} end), - {ok, MeckRet2} = request_api(delete, api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), - ?assertEqual(?ERROR1, get(<<"code">>, MeckRet2)), - - meck:expect(emqx_mgmt, list_acl_cache, 1, fun(_) -> {error, undefined} end), - {ok, MeckRet3} = request_api(get, api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), - ?assertEqual(?ERROR1, get(<<"code">>, MeckRet3)), - - meck:unload(emqx_mgmt), - - {ok, Ok} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), - ?assertEqual(?SUCCESS, get(<<"code">>, Ok)), - - timer:sleep(300), - - {ok, NotFound0} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), - ?assertEqual(?ERROR12, get(<<"code">>, NotFound0)), - - {ok, Clients6} = request_api(get, api_path(["clients"]), "_limit=100&_page=1", auth_header_()), - ?assertEqual(1, maps:get(<<"count">>, get(<<"meta">>, Clients6))), - - {ok, NotFound1} = request_api(get, api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), - ?assertEqual(?ERROR12, get(<<"code">>, NotFound1)), - - {ok, NotFound2} = request_api(delete, api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), - ?assertEqual(?ERROR12, get(<<"code">>, NotFound2)), - - {ok, EmptyAclCache} = request_api(get, api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), - ?assertEqual(0, length(get(<<"data">>, EmptyAclCache))), - - {ok, Ok1} = request_api(delete, api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), - ?assertEqual(?SUCCESS, get(<<"code">>, Ok1)). - -receive_exit(0) -> - ok; -receive_exit(Count) -> - receive - {'EXIT', Client, {shutdown, tcp_closed}} -> - ct:log("receive exit signal, Client: ~p", [Client]), - receive_exit(Count - 1); - {'EXIT', Client, _Reason} -> - ct:log("receive exit signal, Client: ~p", [Client]), - receive_exit(Count - 1) - after 1000 -> - ct:log("timeout") - end. - -t_listeners(_) -> - {ok, _} = request_api(get, api_path(["listeners"]), auth_header_()), - {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node()), "listeners"]), auth_header_()), - meck:new(emqx_mgmt, [passthrough, no_history]), - meck:expect(emqx_mgmt, list_listeners, 0, fun() -> [{node(), {error, undefined}}] end), - {ok, Return} = request_api(get, api_path(["listeners"]), auth_header_()), - [Error] = get(<<"data">>, Return), - ?assertEqual(<<"undefined">>, - maps:get(<<"error">>, maps:get(<<"listeners">>, Error))), - meck:unload(emqx_mgmt). - -t_metrics(_) -> - {ok, _} = request_api(get, api_path(["metrics"]), auth_header_()), - {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node()), "metrics"]), auth_header_()), - meck:new(emqx_mgmt, [passthrough, no_history]), - meck:expect(emqx_mgmt, get_metrics, 1, fun(_) -> {error, undefined} end), - {ok, "{\"message\":\"undefined\"}"} = request_api(get, api_path(["nodes", atom_to_list(node()), "metrics"]), auth_header_()), - meck:unload(emqx_mgmt). - -t_nodes(_) -> - {ok, _} = request_api(get, api_path(["nodes"]), auth_header_()), - {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node())]), auth_header_()), - meck:new(emqx_mgmt, [passthrough, no_history]), - meck:expect(emqx_mgmt, list_nodes, 0, fun() -> [{node(), {error, undefined}}] end), - {ok, Return} = request_api(get, api_path(["nodes"]), auth_header_()), - [Error] = get(<<"data">>, Return), - ?assertEqual(<<"undefined">>, maps:get(<<"error">>, Error)), - meck:unload(emqx_mgmt). - -t_plugins(_) -> - application:ensure_all_started(emqx_retainer), - {ok, Plugins1} = request_api(get, api_path(["plugins"]), auth_header_()), - [Plugins11] = filter(get(<<"data">>, Plugins1), <<"node">>, atom_to_binary(node(), utf8)), - [Plugin1] = filter(maps:get(<<"plugins">>, Plugins11), <<"name">>, <<"emqx_retainer">>), - ?assertEqual(<<"emqx_retainer">>, maps:get(<<"name">>, Plugin1)), - ?assertEqual(true, maps:get(<<"active">>, Plugin1)), - - {ok, _} = request_api(put, - api_path(["plugins", - atom_to_list(emqx_retainer), - "unload"]), - auth_header_()), - {ok, Error1} = request_api(put, - api_path(["plugins", - atom_to_list(emqx_retainer), - "unload"]), - auth_header_()), - ?assertEqual(<<"not_started">>, get(<<"message">>, Error1)), - {ok, Plugins2} = request_api(get, - api_path(["nodes", atom_to_list(node()), "plugins"]), - auth_header_()), - [Plugin2] = filter(get(<<"data">>, Plugins2), <<"name">>, <<"emqx_retainer">>), - ?assertEqual(<<"emqx_retainer">>, maps:get(<<"name">>, Plugin2)), - ?assertEqual(false, maps:get(<<"active">>, Plugin2)), - - {ok, _} = request_api(put, - api_path(["nodes", - atom_to_list(node()), - "plugins", - atom_to_list(emqx_retainer), - "load"]), - auth_header_()), - {ok, Plugins3} = request_api(get, - api_path(["nodes", atom_to_list(node()), "plugins"]), - auth_header_()), - [Plugin3] = filter(get(<<"data">>, Plugins3), <<"name">>, <<"emqx_retainer">>), - ?assertEqual(<<"emqx_retainer">>, maps:get(<<"name">>, Plugin3)), - ?assertEqual(true, maps:get(<<"active">>, Plugin3)), - - {ok, _} = request_api(put, - api_path(["nodes", - atom_to_list(node()), - "plugins", - atom_to_list(emqx_retainer), - "unload"]), - auth_header_()), - {ok, Error2} = request_api(put, - api_path(["nodes", - atom_to_list(node()), - "plugins", - atom_to_list(emqx_retainer), - "unload"]), - auth_header_()), - ?assertEqual(<<"not_started">>, get(<<"message">>, Error2)), - application:stop(emqx_retainer). - -t_acl_cache(_) -> - ClientId = <<"client1">>, - Topic = <<"mytopic">>, - {ok, C1} = emqtt:start_link(#{clientid => ClientId}), - {ok, _} = emqtt:connect(C1), - {ok, _, _} = emqtt:subscribe(C1, Topic, 2), - %% get acl cache, should not be empty - {ok, Result} = request_api(get, api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), - #{<<"code">> := 0, <<"data">> := Caches} = jiffy:decode(list_to_binary(Result), [return_maps]), - ?assert(length(Caches) > 0), - ?assertMatch(#{<<"access">> := <<"subscribe">>, - <<"topic">> := Topic, - <<"result">> := <<"allow">>, - <<"updated_time">> := _}, hd(Caches)), - %% clear acl cache - {ok, Result2} = request_api(delete, api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), - ?assertMatch(#{<<"code">> := 0}, jiffy:decode(list_to_binary(Result2), [return_maps])), - %% get acl cache again, after the acl cache is cleared - {ok, Result3} = request_api(get, api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), - #{<<"code">> := 0, <<"data">> := Caches3} = jiffy:decode(list_to_binary(Result3), [return_maps]), - ?assertEqual(0, length(Caches3)), - ok = emqtt:disconnect(C1). - -t_pubsub(_) -> - Qos1Received = emqx_metrics:val('messages.qos1.received'), - Qos2Received = emqx_metrics:val('messages.qos2.received'), - Received = emqx_metrics:val('messages.received'), - - ClientId = <<"client1">>, - Options = #{clientid => ClientId, - proto_ver => 5}, - Topic = <<"mytopic">>, - {ok, C1} = emqtt:start_link(Options), - {ok, _} = emqtt:connect(C1), - - meck:new(emqx_mgmt, [passthrough, no_history]), - meck:expect(emqx_mgmt, subscribe, 2, fun(_, _) -> {error, undefined} end), - {ok, NotFound1} = request_api(post, api_path(["mqtt/subscribe"]), [], auth_header_(), - #{<<"clientid">> => ClientId, - <<"topic">> => Topic, - <<"qos">> => 2}), - ?assertEqual(?ERROR12, get(<<"code">>, NotFound1)), - meck:unload(emqx_mgmt), - - {ok, BadTopic1} = request_api(post, api_path(["mqtt/subscribe"]), [], auth_header_(), - #{<<"clientid">> => ClientId, - <<"topics">> => <<"">>, - <<"qos">> => 2}), - ?assertEqual(?ERROR15, get(<<"code">>, BadTopic1)), - - {ok, BadTopic2} = request_api(post, api_path(["mqtt/publish"]), [], auth_header_(), - #{<<"clientid">> => ClientId, - <<"topics">> => <<"">>, - <<"qos">> => 1, - <<"payload">> => <<"hello">>}), - ?assertEqual(?ERROR15, get(<<"code">>, BadTopic2)), - - {ok, BadTopic3} = request_api(post, api_path(["mqtt/unsubscribe"]), [], auth_header_(), - #{<<"clientid">> => ClientId, - <<"topic">> => <<"">>}), - ?assertEqual(?ERROR15, get(<<"code">>, BadTopic3)), - - meck:new(emqx_mgmt, [passthrough, no_history]), - meck:expect(emqx_mgmt, unsubscribe, 2, fun(_, _) -> {error, undefined} end), - {ok, NotFound2} = request_api(post, api_path(["mqtt/unsubscribe"]), [], auth_header_(), - #{<<"clientid">> => ClientId, - <<"topic">> => Topic}), - ?assertEqual(?ERROR12, get(<<"code">>, NotFound2)), - meck:unload(emqx_mgmt), - - {ok, Code} = request_api(post, api_path(["mqtt/subscribe"]), [], auth_header_(), - #{<<"clientid">> => ClientId, - <<"topic">> => Topic, - <<"qos">> => 2}), - ?assertEqual(?SUCCESS, get(<<"code">>, Code)), - {ok, Code} = request_api(post, api_path(["mqtt/publish"]), [], auth_header_(), - #{<<"clientid">> => ClientId, - <<"topic">> => <<"mytopic">>, - <<"qos">> => 1, - <<"payload">> => <<"hello">>}), - ?assert(receive - {publish, #{payload := <<"hello">>}} -> - true - after 100 -> - false - end), - %% json payload - {ok, Code} = request_api(post, api_path(["mqtt/publish"]), [], auth_header_(), - #{<<"clientid">> => ClientId, - <<"topic">> => <<"mytopic">>, - <<"qos">> => 1, - <<"payload">> => #{body => "hello world"}}), - Payload = emqx_json:encode(#{body => "hello world"}), - ?assert(receive - {publish, #{payload := Payload}} -> - true - after 100 -> - false - end), - - {ok, Code} = request_api(post, api_path(["mqtt/unsubscribe"]), [], auth_header_(), - #{<<"clientid">> => ClientId, - <<"topic">> => Topic}), - - %% tests subscribe_batch - Topic_list = [<<"mytopic1">>, <<"mytopic2">>], - [ {ok, _, [2]} = emqtt:subscribe(C1, Topics, 2) || Topics <- Topic_list], - - Body1 = [ #{<<"clientid">> => ClientId, <<"topic">> => Topics, <<"qos">> => 2} || Topics <- Topic_list], - {ok, Data1} = request_api(post, api_path(["mqtt/subscribe_batch"]), [], auth_header_(), Body1), - loop(maps:get(<<"data">>, jiffy:decode(list_to_binary(Data1), [return_maps]))), - - %% tests publish_batch - Body2 = [ #{<<"clientid">> => ClientId, <<"topic">> => Topics, <<"qos">> => 2, <<"retain">> => <<"false">>, <<"payload">> => #{body => "hello world"}} || Topics <- Topic_list ], - {ok, Data2} = request_api(post, api_path(["mqtt/publish_batch"]), [], auth_header_(), Body2), - loop(maps:get(<<"data">>, jiffy:decode(list_to_binary(Data2), [return_maps]))), - [ ?assert(receive - {publish, #{topic := Topics}} -> - true - after 100 -> - false - end) || Topics <- Topic_list ], - - %% tests unsubscribe_batch - Body3 = [#{<<"clientid">> => ClientId, <<"topic">> => Topics} || Topics <- Topic_list], - {ok, Data3} = request_api(post, api_path(["mqtt/unsubscribe_batch"]), [], auth_header_(), Body3), - loop(maps:get(<<"data">>, jiffy:decode(list_to_binary(Data3), [return_maps]))), - - ok = emqtt:disconnect(C1), - - ?assertEqual(2, emqx_metrics:val('messages.qos1.received') - Qos1Received), - ?assertEqual(2, emqx_metrics:val('messages.qos2.received') - Qos2Received), - ?assertEqual(4, emqx_metrics:val('messages.received') - Received). - -loop([]) -> []; - -loop(Data) -> - [H | T] = Data, - ct:pal("H: ~p~n", [H]), - ?assertEqual(0, maps:get(<<"code">>, H)), - loop(T). - -t_routes_and_subscriptions(_) -> - ClientId = <<"myclient">>, - Topic = <<"mytopic">>, - {ok, NonRoute} = request_api(get, api_path(["routes"]), auth_header_()), - ?assertEqual([], get(<<"data">>, NonRoute)), - {ok, NonSubscription} = request_api(get, api_path(["subscriptions"]), auth_header_()), - ?assertEqual([], get(<<"data">>, NonSubscription)), - {ok, NonSubscription1} = request_api(get, api_path(["nodes", atom_to_list(node()), "subscriptions"]), auth_header_()), - ?assertEqual([], get(<<"data">>, NonSubscription1)), - {ok, NonSubscription2} = request_api(get, - api_path(["subscriptions", binary_to_list(ClientId)]), - auth_header_()), - ?assertEqual([], get(<<"data">>, NonSubscription2)), - {ok, NonSubscription3} = request_api(get, api_path(["nodes", - atom_to_list(node()), - "subscriptions", - binary_to_list(ClientId)]) - , auth_header_()), - ?assertEqual([], get(<<"data">>, NonSubscription3)), - {ok, C1} = emqtt:start_link(#{clean_start => true, - clientid => ClientId, - proto_ver => ?MQTT_PROTO_V5}), - {ok, _} = emqtt:connect(C1), - {ok, _, [2]} = emqtt:subscribe(C1, Topic, qos2), - {ok, Result} = request_api(get, api_path(["routes"]), auth_header_()), - [Route] = get(<<"data">>, Result), - ?assertEqual(Topic, maps:get(<<"topic">>, Route)), - - {ok, Result2} = request_api(get, api_path(["routes", binary_to_list(Topic)]), auth_header_()), - [Route] = get(<<"data">>, Result2), - - {ok, Result3} = request_api(get, api_path(["subscriptions"]), auth_header_()), - [Subscription] = get(<<"data">>, Result3), - ?assertEqual(Topic, maps:get(<<"topic">>, Subscription)), - ?assertEqual(ClientId, maps:get(<<"clientid">>, Subscription)), - - {ok, Result3} = request_api(get, api_path(["nodes", atom_to_list(node()), "subscriptions"]), auth_header_()), - - {ok, Result4} = request_api(get, api_path(["subscriptions", binary_to_list(ClientId)]), auth_header_()), - [Subscription] = get(<<"data">>, Result4), - {ok, Result4} = request_api(get, api_path(["nodes", atom_to_list(node()), "subscriptions", binary_to_list(ClientId)]) - , auth_header_()), - - ok = emqtt:disconnect(C1). - -t_stats(_) -> - {ok, _} = request_api(get, api_path(["stats"]), auth_header_()), - {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node()), "stats"]), auth_header_()), - meck:new(emqx_mgmt, [passthrough, no_history]), - meck:expect(emqx_mgmt, get_stats, 1, fun(_) -> {error, undefined} end), - {ok, Return} = request_api(get, api_path(["nodes", atom_to_list(node()), "stats"]), auth_header_()), - ?assertEqual(<<"undefined">>, get(<<"message">>, Return)), - meck:unload(emqx_mgmt). - -request_api(Method, Url, Auth) -> - request_api(Method, Url, [], Auth, []). - -request_api(Method, Url, QueryParams, Auth) -> - request_api(Method, Url, QueryParams, Auth, []). - -request_api(Method, Url, QueryParams, Auth, []) -> - NewUrl = case QueryParams of - "" -> Url; - _ -> Url ++ "?" ++ QueryParams - end, - do_request_api(Method, {NewUrl, [Auth]}); -request_api(Method, Url, QueryParams, Auth, Body) -> - NewUrl = case QueryParams of - "" -> Url; - _ -> Url ++ "?" ++ QueryParams - end, - do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). - -do_request_api(Method, Request)-> - ct:pal("Method: ~p, Request: ~p", [Method, Request]), - case httpc:request(Method, Request, [], []) of - {error, socket_closed_remotely} -> - {error, socket_closed_remotely}; - {ok, {{"HTTP/1.1", Code, _}, _, Return} } - when Code =:= 200 orelse Code =:= 201 -> - {ok, Return}; - {ok, {Reason, _, _}} -> - {error, Reason} - end. - -auth_header_() -> - AppId = <<"admin">>, - AppSecret = <<"public">>, - auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)). - -auth_header_(User, Pass) -> - Encoded = base64:encode_to_string(lists:append([User,":",Pass])), - {"Authorization","Basic " ++ Encoded}. - -api_path(Parts)-> - ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts). - -filter(List, Key, Value) -> - lists:filter(fun(Item) -> - maps:get(Key, Item) == Value - end, List). diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl new file mode 100644 index 000000000..568ed46a6 --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -0,0 +1,67 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_api_test_util). +-compile(export_all). +-compile(nowarn_export_all). + +-define(SERVER, "http://127.0.0.1:8081"). +-define(BASE_PATH, "/api/v5"). + +request_api(Method, Url) -> + request_api(Method, Url, [], auth_header_(), []). + +request_api(Method, Url, Auth) -> + request_api(Method, Url, [], Auth, []). + +request_api(Method, Url, QueryParams, Auth) -> + request_api(Method, Url, QueryParams, Auth, []). + +request_api(Method, Url, QueryParams, Auth, []) -> + NewUrl = case QueryParams of + "" -> Url; + _ -> Url ++ "?" ++ QueryParams + end, + do_request_api(Method, {NewUrl, [Auth]}); +request_api(Method, Url, QueryParams, Auth, Body) -> + NewUrl = case QueryParams of + "" -> Url; + _ -> Url ++ "?" ++ QueryParams + end, + do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). + +do_request_api(Method, Request)-> + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], []) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {ok, {{"HTTP/1.1", Code, _}, _, Return} } + when Code =:= 200 orelse Code =:= 201 -> + {ok, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +auth_header_() -> + AppId = <<"admin">>, + AppSecret = <<"public">>, + auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)). + +auth_header_(User, Pass) -> + Encoded = base64:encode_to_string(lists:append([User,":",Pass])), + {"Authorization","Basic " ++ Encoded}. + +api_path(Parts)-> + ?SERVER ++ filename:join([?BASE_PATH | Parts]). diff --git a/apps/emqx_management/test/emqx_mgmt_apps_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_apps_api_SUITE.erl new file mode 100644 index 000000000..a139208a5 --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_apps_api_SUITE.erl @@ -0,0 +1,110 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_apps_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + ekka_mnesia:start(), + emqx_mgmt_auth:mnesia(boot), + emqx_ct_helpers:start_apps([emqx_management], fun set_special_configs/1), + Config. + +end_per_suite(_) -> + emqx_ct_helpers:stop_apps([emqx_management]). + +set_special_configs(emqx_management) -> + emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + applications =>[#{id => "admin", secret => "public"}]}), + ok; +set_special_configs(_App) -> + ok. + +t_list_app(_) -> + Path = emqx_mgmt_api_test_util:api_path(["apps"]), + {ok, Body} = emqx_mgmt_api_test_util:request_api(get, Path), + Data = emqx_json:decode(Body, [return_maps]), + AdminApp = hd(Data), + Admin = maps:get(<<"app_id">>, AdminApp), + ?assertEqual(<<"admin">>, Admin). + +t_get_app(_) -> + Path = emqx_mgmt_api_test_util:api_path(["apps/admin"]), + {ok, Body} = emqx_mgmt_api_test_util:request_api(get, Path), + AdminApp = emqx_json:decode(Body, [return_maps]), + ?assertEqual(<<"admin">>, maps:get(<<"app_id">>, AdminApp)), + ?assertEqual(<<"public">>, maps:get(<<"secret">>, AdminApp)). + +t_add_app(_) -> + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + AppId = <<"test_app_id">>, + TestAppPath = emqx_mgmt_api_test_util:api_path(["apps", AppId]), + AppSecret = <<"test_app_secret">>, + + %% new test app + Path = emqx_mgmt_api_test_util:api_path(["apps"]), + RequestBody = #{ + app_id => AppId, + secret => AppSecret, + desc => <<"test desc">>, + name => <<"test_app_name">>, + expired => erlang:system_time(second) + 3000, + status => true + }, + {ok, Body} = emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, RequestBody), + TestAppSecret = emqx_json:decode(Body, [return_maps]), + ?assertEqual(AppSecret, maps:get(<<"secret">>, TestAppSecret)), + + %% get new test app + {ok, GetApp} = emqx_mgmt_api_test_util:request_api(get, TestAppPath), + TestApp = emqx_json:decode(GetApp, [return_maps]), + ?assertEqual(AppId, maps:get(<<"app_id">>, TestApp)), + ?assertEqual(AppSecret, maps:get(<<"secret">>, TestApp)), + + %% update app + Desc2 = <<"test desc 2">>, + Name2 = <<"test_app_name_2">>, + PutBody = #{ + desc => Desc2, + name => Name2, + expired => erlang:system_time(second) + 3000, + status => false + }, + {ok, PutApp} = emqx_mgmt_api_test_util:request_api(put, TestAppPath, "", AuthHeader, PutBody), + TestApp1 = emqx_json:decode(PutApp, [return_maps]), + ?assertEqual(Desc2, maps:get(<<"desc">>, TestApp1)), + ?assertEqual(Name2, maps:get(<<"name">>, TestApp1)), + ?assertEqual(false, maps:get(<<"status">>, TestApp1)), + + %% after update + {ok, GetApp2} = emqx_mgmt_api_test_util:request_api(get, TestAppPath), + TestApp2 = emqx_json:decode(GetApp2, [return_maps]), + ?assertEqual(Desc2, maps:get(<<"desc">>, TestApp2)), + ?assertEqual(Name2, maps:get(<<"name">>, TestApp2)), + ?assertEqual(false, maps:get(<<"status">>, TestApp2)), + + %% delete new app + {ok, _} = emqx_mgmt_api_test_util:request_api(delete, TestAppPath), + + %% after delete + ?assertEqual({error,{"HTTP/1.1",404,"Not Found"}}, + emqx_mgmt_api_test_util:request_api(get, TestAppPath)). diff --git a/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl new file mode 100644 index 000000000..dbe2d83fa --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl @@ -0,0 +1,108 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_clients_api_SUITE). +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-define(APP, emqx_management). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + ekka_mnesia:start(), + emqx_mgmt_auth:mnesia(boot), + emqx_ct_helpers:start_apps([emqx_management], fun set_special_configs/1), + Config. + +end_per_suite(_) -> + emqx_ct_helpers:stop_apps([emqx_management]). + +set_special_configs(emqx_management) -> + emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + applications =>[#{id => "admin", secret => "public"}]}), + ok; +set_special_configs(_App) -> + ok. + +t_clients(_) -> + process_flag(trap_exit, true), + + Username1 = <<"user1">>, + ClientId1 = <<"client1">>, + + Username2 = <<"user2">>, + ClientId2 = <<"client2">>, + + Topic = <<"topic_1">>, + Qos = 0, + + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + + {ok, C1} = emqtt:start_link(#{username => Username1, clientid => ClientId1}), + {ok, _} = emqtt:connect(C1), + {ok, C2} = emqtt:start_link(#{username => Username2, clientid => ClientId2}), + {ok, _} = emqtt:connect(C2), + + timer:sleep(300), + + %% get /clients + ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]), + {ok, Clients} = emqx_mgmt_api_test_util:request_api(get, ClientsPath), + ClientsResponse = emqx_json:decode(Clients, [return_maps]), + ClientsMeta = maps:get(<<"meta">>, ClientsResponse), + ClientsPage = maps:get(<<"page">>, ClientsMeta), + ClientsLimit = maps:get(<<"limit">>, ClientsMeta), + ClientsCount = maps:get(<<"count">>, ClientsMeta), + ?assertEqual(ClientsPage, 1), + ?assertEqual(ClientsLimit, emqx_mgmt:max_row_limit()), + ?assertEqual(ClientsCount, 2), + + %% get /clients/:clientid + Client1Path = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId1)]), + {ok, Client1} = emqx_mgmt_api_test_util:request_api(get, Client1Path), + Client1Response = emqx_json:decode(Client1, [return_maps]), + ?assertEqual(Username1, maps:get(<<"username">>, Client1Response)), + ?assertEqual(ClientId1, maps:get(<<"clientid">>, Client1Response)), + + %% delete /clients/:clientid kickout + Client2Path = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId2)]), + {ok, _} = emqx_mgmt_api_test_util:request_api(delete, Client2Path), + timer:sleep(300), + AfterKickoutResponse = emqx_mgmt_api_test_util:request_api(get, Client2Path), + ?assertEqual({error, {"HTTP/1.1", 404, "Not Found"}}, AfterKickoutResponse), + + %% get /clients/:clientid/authz_cache should has no authz cache + Client1AuthzCachePath = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId1), "authz_cache"]), + {ok, Client1AuthzCache} = emqx_mgmt_api_test_util:request_api(get, Client1AuthzCachePath), + ?assertEqual("[]", Client1AuthzCache), + + %% post /clients/:clientid/subscribe + SubscribeBody = #{topic => Topic, qos => Qos}, + SubscribePath = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId1), "subscribe"]), + {ok, _} = emqx_mgmt_api_test_util:request_api(post, SubscribePath, "", AuthHeader, SubscribeBody), + timer:sleep(100), + [{{_, AfterSubTopic}, #{qos := AfterSubQos}}] = emqx_mgmt:lookup_subscriptions(ClientId1), + ?assertEqual(AfterSubTopic, Topic), + ?assertEqual(AfterSubQos, Qos), + + %% delete /clients/:clientid/subscribe + UnSubscribeQuery = "topic=" ++ binary_to_list(Topic), + {ok, _} = emqx_mgmt_api_test_util:request_api(delete, SubscribePath, UnSubscribeQuery, AuthHeader), + timer:sleep(100), + ?assertEqual([], emqx_mgmt:lookup_subscriptions(Client1)). diff --git a/apps/emqx_management/test/emqx_mgmt_listeners_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_listeners_api_SUITE.erl new file mode 100644 index 000000000..e3d50f57c --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_listeners_api_SUITE.erl @@ -0,0 +1,120 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_listeners_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + ekka_mnesia:start(), + emqx_mgmt_auth:mnesia(boot), + emqx_ct_helpers:start_apps([emqx_management], fun set_special_configs/1), + Config. + +end_per_suite(_) -> + emqx_ct_helpers:stop_apps([emqx_management]). + +set_special_configs(emqx_management) -> + emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + applications =>[#{id => "admin", secret => "public"}]}), + ok; +set_special_configs(_App) -> + ok. + +t_list_listeners(_) -> + Path = emqx_mgmt_api_test_util:api_path(["listeners"]), + get_api(Path). + +t_list_node_listeners(_) -> + Path = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_binary(node(), utf8), "listeners"]), + get_api(Path). + +t_get_listeners(_) -> + LocalListener = emqx_mgmt_api_listeners:format(hd(emqx_mgmt:list_listeners())), + Identifier = maps:get(identifier, LocalListener), + Path = emqx_mgmt_api_test_util:api_path(["listeners", atom_to_list(Identifier)]), + get_api(Path). + +t_get_node_listeners(_) -> + LocalListener = emqx_mgmt_api_listeners:format(hd(emqx_mgmt:list_listeners())), + Identifier = maps:get(identifier, LocalListener), + Path = emqx_mgmt_api_test_util:api_path( + ["nodes", atom_to_binary(node(), utf8), "listeners", atom_to_list(Identifier)]), + get_api(Path). + +t_stop_listener(_) -> + LocalListener = emqx_mgmt_api_listeners:format(hd(emqx_mgmt:list_listeners())), + Identifier = maps:get(identifier, LocalListener), + Path = emqx_mgmt_api_test_util:api_path(["listeners", atom_to_list(Identifier), "stop"]), + {ok, _} = emqx_mgmt_api_test_util:request_api(get, Path), + GetPath = emqx_mgmt_api_test_util:api_path(["listeners", atom_to_list(Identifier)]), + {ok, ListenersResponse} = emqx_mgmt_api_test_util:request_api(get, GetPath), + Listeners = emqx_json:decode(ListenersResponse, [return_maps]), + [listener_stats(Listener, false) || Listener <- Listeners]. + +get_api(Path) -> + {ok, ListenersData} = emqx_mgmt_api_test_util:request_api(get, Path), + LocalListeners = emqx_mgmt_api_listeners:format(emqx_mgmt:list_listeners()), + case emqx_json:decode(ListenersData, [return_maps]) of + [Listener] -> + Identifier = binary_to_atom(maps:get(<<"identifier">>, Listener), utf8), + Filter = + fun(Local) -> + maps:get(identifier, Local) =:= Identifier + end, + LocalListener = hd(lists:filter(Filter, LocalListeners)), + comparison_listener(LocalListener, Listener); + Listeners when is_list(Listeners) -> + ?assertEqual(erlang:length(LocalListeners), erlang:length(Listeners)), + Fun = + fun(LocalListener) -> + Identifier = maps:get(identifier, LocalListener), + IdentifierBinary = atom_to_binary(Identifier, utf8), + Filter = + fun(Listener) -> + maps:get(<<"identifier">>, Listener) =:= IdentifierBinary + end, + Listener = hd(lists:filter(Filter, Listeners)), + comparison_listener(LocalListener, Listener) + end, + lists:foreach(Fun, LocalListeners); + Listener when is_map(Listener) -> + Identifier = binary_to_atom(maps:get(<<"identifier">>, Listener), utf8), + Filter = + fun(Local) -> + maps:get(identifier, Local) =:= Identifier + end, + LocalListener = hd(lists:filter(Filter, LocalListeners)), + comparison_listener(LocalListener, Listener) + end. + +comparison_listener(Local, Response) -> + ?assertEqual(maps:get(identifier, Local), binary_to_atom(maps:get(<<"identifier">>, Response))), + ?assertEqual(maps:get(node, Local), binary_to_atom(maps:get(<<"node">>, Response))), + ?assertEqual(maps:get(acceptors, Local), maps:get(<<"acceptors">>, Response)), + ?assertEqual(maps:get(max_conn, Local), maps:get(<<"max_conn">>, Response)), + ?assertEqual(maps:get(listen_on, Local), maps:get(<<"listen_on">>, Response)), + ?assertEqual(maps:get(running, Local), maps:get(<<"running">>, Response)), + ?assertEqual(maps:get(auth, Local), maps:get(<<"auth">>, Response)). + + +listener_stats(Listener, Stats) -> + ?assertEqual(maps:get(<<"running">>, Listener), Stats). diff --git a/apps/emqx_management/test/emqx_mgmt_metrics_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_metrics_api_SUITE.erl new file mode 100644 index 000000000..b54489e37 --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_metrics_api_SUITE.erl @@ -0,0 +1,52 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_metrics_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + ekka_mnesia:start(), + emqx_mgmt_auth:mnesia(boot), + emqx_ct_helpers:start_apps([emqx_management], fun set_special_configs/1), + Config. + +end_per_suite(_) -> + emqx_ct_helpers:stop_apps([emqx_management]). + +set_special_configs(emqx_management) -> + emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + applications =>[#{id => "admin", secret => "public"}]}), + ok; +set_special_configs(_App) -> + ok. + +t_metrics_api(_) -> + MetricsPath = emqx_mgmt_api_test_util:api_path(["metrics?aggregate=true"]), + SystemMetrics = emqx_mgmt:get_metrics(), + {ok, MetricsResponse} = emqx_mgmt_api_test_util:request_api(get, MetricsPath), + Metrics = emqx_json:decode(MetricsResponse, [return_maps]), + ?assertEqual(erlang:length(maps:keys(SystemMetrics)), erlang:length(maps:keys(Metrics))), + Fun = + fun(Key) -> + ?assertEqual(maps:get(Key, SystemMetrics), maps:get(atom_to_binary(Key, utf8), Metrics)) + end, + lists:foreach(Fun, maps:keys(SystemMetrics)). diff --git a/apps/emqx_management/test/emqx_mgmt_nodes_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_nodes_api_SUITE.erl new file mode 100644 index 000000000..f0829b7fb --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_nodes_api_SUITE.erl @@ -0,0 +1,77 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_nodes_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + ekka_mnesia:start(), + emqx_mgmt_auth:mnesia(boot), + emqx_ct_helpers:start_apps([emqx_management], fun set_special_configs/1), + Config. + +end_per_suite(_) -> + emqx_ct_helpers:stop_apps([emqx_management]). + +set_special_configs(emqx_management) -> + emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + applications =>[#{id => "admin", secret => "public"}]}), + ok; +set_special_configs(_App) -> + ok. + +t_nodes_api(_) -> + NodesPath = emqx_mgmt_api_test_util:api_path(["nodes"]), + {ok, Nodes} = emqx_mgmt_api_test_util:request_api(get, NodesPath), + NodesResponse = emqx_json:decode(Nodes, [return_maps]), + LocalNodeInfo = hd(NodesResponse), + Node = binary_to_atom(maps:get(<<"node">>, LocalNodeInfo), utf8), + ?assertEqual(Node, node()), + + NodePath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_list(node())]), + {ok, NodeInfo} = emqx_mgmt_api_test_util:request_api(get, NodePath), + NodeNameResponse = + binary_to_atom(maps:get(<<"node">>, emqx_json:decode(NodeInfo, [return_maps])), utf8), + ?assertEqual(node(), NodeNameResponse). + +t_node_stats_api() -> + StatsPath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_binary(node(), utf8), "stats"]), + SystemStats= emqx_mgmt:get_stats(), + {ok, StatsResponse} = emqx_mgmt_api_test_util:request_api(get, StatsPath), + Stats = emqx_json:decode(StatsResponse, [return_maps]), + Fun = + fun(Key) -> + ?assertEqual(maps:get(Key, SystemStats), maps:get(atom_to_binary(Key, utf8), Stats)) + end, + lists:foreach(Fun, maps:keys(SystemStats)). + +t_node_metrics_api() -> + MetricsPath = + emqx_mgmt_api_test_util:api_path(["nodes", atom_to_binary(node(), utf8), "metrics"]), + SystemMetrics= emqx_mgmt:get_metrics(), + {ok, MetricsResponse} = emqx_mgmt_api_test_util:request_api(get, MetricsPath), + Metrics = emqx_json:decode(MetricsResponse, [return_maps]), + Fun = + fun(Key) -> + ?assertEqual(maps:get(Key, SystemMetrics), maps:get(atom_to_binary(Key, utf8), Metrics)) + end, + lists:foreach(Fun, maps:keys(SystemMetrics)). diff --git a/apps/emqx_management/test/emqx_mgmt_publish_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_publish_api_SUITE.erl new file mode 100644 index 000000000..2f566e5f4 --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_publish_api_SUITE.erl @@ -0,0 +1,92 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_publish_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-define(CLIENTID, <<"api_clientid">>). +-define(USERNAME, <<"api_username">>). + +-define(TOPIC1, <<"api_topic1">>). +-define(TOPIC2, <<"api_topic2">>). + + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + ekka_mnesia:start(), + emqx_mgmt_auth:mnesia(boot), + emqx_ct_helpers:start_apps([emqx_management], fun set_special_configs/1), + Config. + + +end_per_suite(_) -> + emqx_ct_helpers:stop_apps([emqx_management]). + +set_special_configs(emqx_management) -> + emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + applications =>[#{id => "admin", secret => "public"}]}), + ok; +set_special_configs(_App) -> + ok. + +t_publish_api(_) -> + {ok, Client} = emqtt:start_link(#{username => <<"api_username">>, clientid => <<"api_clientid">>}), + {ok, _} = emqtt:connect(Client), + {ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC1), + {ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC2), + Payload = <<"hello">>, + Path = emqx_mgmt_api_test_util:api_path(["publish"]), + Auth = emqx_mgmt_api_test_util:auth_header_(), + Body = #{topic => ?TOPIC1, payload => Payload}, + {ok, _} = emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, Body), + ?assertEqual(receive_assert(?TOPIC1, 0, Payload), ok), + emqtt:disconnect(Client). + +t_publish_bulk_api(_) -> + {ok, Client} = emqtt:start_link(#{username => <<"api_username">>, clientid => <<"api_clientid">>}), + {ok, _} = emqtt:connect(Client), + {ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC1), + {ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC2), + Payload = <<"hello">>, + Path = emqx_mgmt_api_test_util:api_path(["publish", "bulk"]), + Auth = emqx_mgmt_api_test_util:auth_header_(), + Body =[#{topic => ?TOPIC1, payload => Payload}, #{topic => ?TOPIC2, payload => Payload}], + {ok, Response} = emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, Body), + ResponseMap = emqx_json:decode(Response, [return_maps]), + ?assertEqual(2, erlang:length(ResponseMap)), + ?assertEqual(receive_assert(?TOPIC1, 0, Payload), ok), + ?assertEqual(receive_assert(?TOPIC2, 0, Payload), ok), + emqtt:disconnect(Client). + +receive_assert(Topic, Qos, Payload) -> + receive + {publish, Message} -> + ReceiveTopic = maps:get(topic, Message), + ReceiveQos = maps:get(qos, Message), + ReceivePayload = maps:get(payload, Message), + ?assertEqual(ReceiveTopic , Topic), + ?assertEqual(ReceiveQos , Qos), + ?assertEqual(ReceivePayload , Payload), + ok + after 5000 -> + timeout + end. + diff --git a/apps/emqx_management/test/emqx_mgmt_routes_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_routes_api_SUITE.erl new file mode 100644 index 000000000..1756f6ff5 --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_routes_api_SUITE.erl @@ -0,0 +1,65 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_routes_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + ekka_mnesia:start(), + emqx_mgmt_auth:mnesia(boot), + emqx_ct_helpers:start_apps([emqx_management], fun set_special_configs/1), + Config. + +end_per_suite(_) -> + emqx_ct_helpers:stop_apps([emqx_management]). + +set_special_configs(emqx_management) -> + emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + applications =>[#{id => "admin", secret => "public"}]}), + ok; +set_special_configs(_App) -> + ok. + +t_nodes_api(_) -> + Topic = <<"test_topic">>, + {ok, Client} = emqtt:start_link(#{username => <<"routes_username">>, clientid => <<"routes_cid">>}), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, Topic), + + Path = emqx_mgmt_api_test_util:api_path(["routes"]), + {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path), + RoutesData = emqx_json:decode(Response, [return_maps]), + Meta = maps:get(<<"meta">>, RoutesData), + ?assertEqual(1, maps:get(<<"page">>, Meta)), + ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, Meta)), + ?assertEqual(1, maps:get(<<"count">>, Meta)), + Data = maps:get(<<"data">>, RoutesData), + Route = erlang:hd(Data), + ?assertEqual(Topic, maps:get(<<"topic">>, Route)), + ?assertEqual(atom_to_binary(node(), utf8), maps:get(<<"node">>, Route)), + + %% get routes/:topic + RoutePath = emqx_mgmt_api_test_util:api_path(["routes", Topic]), + {ok, RouteResponse} = emqx_mgmt_api_test_util:request_api(get, RoutePath), + RouteData = emqx_json:decode(RouteResponse, [return_maps]), + ?assertEqual(Topic, maps:get(<<"topic">>, RouteData)), + ?assertEqual(atom_to_binary(node(), utf8), maps:get(<<"node">>, RouteData)). \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_mgmt_stats_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_stats_api_SUITE.erl new file mode 100644 index 000000000..395a0851e --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_stats_api_SUITE.erl @@ -0,0 +1,52 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_stats_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + ekka_mnesia:start(), + emqx_mgmt_auth:mnesia(boot), + emqx_ct_helpers:start_apps([emqx_management], fun set_special_configs/1), + Config. + +end_per_suite(_) -> + emqx_ct_helpers:stop_apps([emqx_management]). + +set_special_configs(emqx_management) -> + emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + applications =>[#{id => "admin", secret => "public"}]}), + ok; +set_special_configs(_App) -> + ok. + +t_stats_api(_) -> + StatsPath = emqx_mgmt_api_test_util:api_path(["stats?aggregate=true"]), + SystemStats = emqx_mgmt:get_stats(), + {ok, StatsResponse} = emqx_mgmt_api_test_util:request_api(get, StatsPath), + Stats = emqx_json:decode(StatsResponse, [return_maps]), + ?assertEqual(erlang:length(maps:keys(SystemStats)), erlang:length(maps:keys(Stats))), + Fun = + fun(Key) -> + ?assertEqual(maps:get(Key, SystemStats), maps:get(atom_to_binary(Key, utf8), Stats)) + end, + lists:foreach(Fun, maps:keys(SystemStats)). diff --git a/apps/emqx_management/test/emqx_mgmt_subscription_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_subscription_api_SUITE.erl new file mode 100644 index 000000000..6d56f21c4 --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_subscription_api_SUITE.erl @@ -0,0 +1,74 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_subscription_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-define(CLIENTID, <<"api_clientid">>). +-define(USERNAME, <<"api_username">>). + +%% notice: integer topic for sort response +-define(TOPIC1, <<"0000">>). +-define(TOPIC2, <<"0001">>). + + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + ekka_mnesia:start(), + emqx_mgmt_auth:mnesia(boot), + emqx_ct_helpers:start_apps([emqx_management], fun set_special_configs/1), + Config. + + +end_per_suite(_) -> + emqx_ct_helpers:stop_apps([emqx_management]). + +set_special_configs(emqx_management) -> + emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + applications =>[#{id => "admin", secret => "public"}]}), + ok; +set_special_configs(_App) -> + ok. + +t_subscription_api(_) -> + {ok, Client} = emqtt:start_link(#{username => ?USERNAME, clientid => ?CLIENTID}), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, ?TOPIC1), + {ok, _, _} = emqtt:subscribe(Client, ?TOPIC2), + Path = emqx_mgmt_api_test_util:api_path(["subscriptions"]), + {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path), + Data = emqx_json:decode(Response, [return_maps]), + Meta = maps:get(<<"meta">>, Data), + ?assertEqual(1, maps:get(<<"page">>, Meta)), + ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, Meta)), + ?assertEqual(2, maps:get(<<"count">>, Meta)), + Subscriptions = maps:get(<<"data">>, Data), + ?assertEqual(length(Subscriptions), 2), + Sort = + fun(#{<<"topic">> := T1}, #{<<"topic">> := T2}) -> + binary_to_integer(T1) =< binary_to_integer(T2) + end, + [Subscriptions1, Subscriptions2] = lists:sort(Sort, Subscriptions), + ?assertEqual(maps:get(<<"topic">>, Subscriptions1), ?TOPIC1), + ?assertEqual(maps:get(<<"topic">>, Subscriptions2), ?TOPIC2), + ?assertEqual(maps:get(<<"clientid">>, Subscriptions1), ?CLIENTID), + ?assertEqual(maps:get(<<"clientid">>, Subscriptions2), ?CLIENTID), + emqtt:disconnect(Client). diff --git a/apps/emqx_management/test/etc/emqx_management.conf b/apps/emqx_management/test/etc/emqx_management.conf deleted file mode 100644 index cec70cc8e..000000000 --- a/apps/emqx_management/test/etc/emqx_management.conf +++ /dev/null @@ -1,35 +0,0 @@ -##-------------------------------------------------------------------- -## EMQ X Management Plugin -##-------------------------------------------------------------------- - -## Max Row Limit -management.max_row_limit = 10000 - -## Application default secret -# -# management.application.default_secret = public - -##-------------------------------------------------------------------- -## HTTP Listener - -management.listener.http = 8080 -management.listener.http.acceptors = 2 -management.listener.http.max_clients = 512 -management.listener.http.backlog = 512 -management.listener.http.send_timeout = 15s -management.listener.http.send_timeout_close = on - -##-------------------------------------------------------------------- -## HTTPS Listener - -## management.listener.https = 8081 -## management.listener.https.acceptors = 2 -## management.listener.https.max_clients = 512 -## management.listener.https.backlog = 512 -## management.listener.https.send_timeout = 15s -## management.listener.https.send_timeout_close = on -## management.listener.https.certfile = etc/certs/cert.pem -## management.listener.https.keyfile = etc/certs/key.pem -## management.listener.https.cacertfile = etc/certs/cacert.pem -## management.listener.https.verify = verify_peer -## management.listener.https.fail_if_no_peer_cert = true diff --git a/apps/emqx_management/test/etc/emqx_reloader.conf b/apps/emqx_management/test/etc/emqx_reloader.conf deleted file mode 100644 index 0919c8411..000000000 --- a/apps/emqx_management/test/etc/emqx_reloader.conf +++ /dev/null @@ -1,24 +0,0 @@ -##-------------------------------------------------------------------- -## Reloader Plugin -##-------------------------------------------------------------------- - -## Interval of hot code reloading. -## -## Value: Duration -## - h: hour -## - m: minute -## - s: second -## -## Examples: -## - 2h: 2 hours -## - 30m: 30 minutes -## - 20s: 20 seconds -## -## Defaut: 60s -reloader.interval = 60s - -## Logfile of reloader. -## -## Value: File -reloader.logfile = reloader.log - diff --git a/apps/emqx_management/test/rfc6455_client.erl b/apps/emqx_management/test/rfc6455_client.erl deleted file mode 100644 index 987b72407..000000000 --- a/apps/emqx_management/test/rfc6455_client.erl +++ /dev/null @@ -1,252 +0,0 @@ -%% The contents of this file are subject to the Mozilla Public License -%% Version 1.1 (the "License"); you may not use this file except in -%% compliance with the License. You may obtain a copy of the License at -%% http://www.mozilla.org/MPL/ -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the -%% License for the specific language governing rights and limitations -%% under the License. -%% -%% The Original Code is RabbitMQ Management Console. -%% -%% The Initial Developer of the Original Code is GoPivotal, Inc. -%% Copyright (c) 2012-2016 Pivotal Software, Inc. All rights reserved. -%% - --module(rfc6455_client). - --export([new/2, open/1, recv/1, send/2, send_binary/2, close/1, close/2]). - --record(state, {host, port, addr, path, ppid, socket, data, phase}). - -%% -------------------------------------------------------------------------- - -new(WsUrl, PPid) -> - crypto:start(), - "ws://" ++ Rest = WsUrl, - [Addr, Path] = split("/", Rest, 1), - [Host, MaybePort] = split(":", Addr, 1, empty), - Port = case MaybePort of - empty -> 80; - V -> {I, ""} = string:to_integer(V), I - end, - State = #state{host = Host, - port = Port, - addr = Addr, - path = "/" ++ Path, - ppid = PPid}, - spawn(fun() -> - start_conn(State) - end). - -open(WS) -> - receive - {rfc6455, open, WS, Opts} -> - {ok, Opts}; - {rfc6455, close, WS, R} -> - {close, R} - end. - -recv(WS) -> - receive - {rfc6455, recv, WS, Payload} -> - {ok, Payload}; - {rfc6455, recv_binary, WS, Payload} -> - {binary, Payload}; - {rfc6455, close, WS, R} -> - {close, R} - end. - -send(WS, IoData) -> - WS ! {send, IoData}, - ok. - -send_binary(WS, IoData) -> - WS ! {send_binary, IoData}, - ok. - -close(WS) -> - close(WS, {1000, ""}). - -close(WS, WsReason) -> - WS ! {close, WsReason}, - receive - {rfc6455, close, WS, R} -> - {close, R} - end. - - -%% -------------------------------------------------------------------------- - -start_conn(State) -> - {ok, Socket} = gen_tcp:connect(State#state.host, State#state.port, - [binary, - {packet, 0}]), - Key = base64:encode_to_string(crypto:strong_rand_bytes(16)), - gen_tcp:send(Socket, - "GET " ++ State#state.path ++ " HTTP/1.1\r\n" ++ - "Host: " ++ State#state.addr ++ "\r\n" ++ - "Upgrade: websocket\r\n" ++ - "Connection: Upgrade\r\n" ++ - "Sec-WebSocket-Key: " ++ Key ++ "\r\n" ++ - "Origin: null\r\n" ++ - "Sec-WebSocket-Protocol: mqtt\r\n" ++ - "Sec-WebSocket-Version: 13\r\n\r\n"), - - loop(State#state{socket = Socket, - data = <<>>, - phase = opening}). - -do_recv(State = #state{phase = opening, ppid = PPid, data = Data}) -> - case split("\r\n\r\n", binary_to_list(Data), 1, empty) of - [_Http, empty] -> State; - [Http, Data1] -> - %% TODO: don't ignore http response data, verify key - PPid ! {rfc6455, open, self(), [{http_response, Http}]}, - State#state{phase = open, - data = Data1} - end; -do_recv(State = #state{phase = Phase, data = Data, socket = Socket, ppid = PPid}) - when Phase =:= open orelse Phase =:= closing -> - R = case Data of - <> - when L < 126 -> - {F, O, Payload, Rest}; - - <> -> - {F, O, Payload, Rest}; - - <> -> - {F, O, Payload, Rest}; - - <<_:1, _:3, _:4, 1:1, _/binary>> -> - %% According o rfc6455 5.1 the server must not mask any frames. - die(Socket, PPid, {1006, "Protocol error"}, normal); - _ -> - moredata - end, - case R of - moredata -> - State; - _ -> do_recv2(State, R) - end. - -do_recv2(State = #state{phase = Phase, socket = Socket, ppid = PPid}, R) -> - case R of - {1, 1, Payload, Rest} -> - PPid ! {rfc6455, recv, self(), Payload}, - State#state{data = Rest}; - {1, 2, Payload, Rest} -> - PPid ! {rfc6455, recv_binary, self(), Payload}, - State#state{data = Rest}; - {1, 8, Payload, _Rest} -> - WsReason = case Payload of - <> -> {WC, WR}; - <<>> -> {1005, "No status received"} - end, - case Phase of - open -> %% echo - do_close(State, WsReason), - gen_tcp:close(Socket); - closing -> - ok - end, - die(Socket, PPid, WsReason, normal); - {_, _, _, _Rest2} -> - io:format("Unknown frame type~n"), - die(Socket, PPid, {1006, "Unknown frame type"}, normal) - end. - -encode_frame(F, O, Payload) -> - Mask = crypto:strong_rand_bytes(4), - MaskedPayload = apply_mask(Mask, iolist_to_binary(Payload)), - - L = byte_size(MaskedPayload), - IoData = case L of - _ when L < 126 -> - [<>, Mask, MaskedPayload]; - _ when L < 65536 -> - [<>, Mask, MaskedPayload]; - _ -> - [<>, Mask, MaskedPayload] - end, - iolist_to_binary(IoData). - -do_send(State = #state{socket = Socket}, Payload) -> - gen_tcp:send(Socket, encode_frame(1, 1, Payload)), - State. - -do_send_binary(State = #state{socket = Socket}, Payload) -> - gen_tcp:send(Socket, encode_frame(1, 2, Payload)), - State. - -do_close(State = #state{socket = Socket}, {Code, Reason}) -> - Payload = iolist_to_binary([<>, Reason]), - gen_tcp:send(Socket, encode_frame(1, 8, Payload)), - State#state{phase = closing}. - - -loop(State = #state{socket = Socket, ppid = PPid, data = Data, - phase = Phase}) -> - receive - {tcp, Socket, Bin} -> - State1 = State#state{data = iolist_to_binary([Data, Bin])}, - loop(do_recv(State1)); - {send, Payload} when Phase == open -> - loop(do_send(State, Payload)); - {send_binary, Payload} when Phase == open -> - loop(do_send_binary(State, Payload)); - {tcp_closed, Socket} -> - die(Socket, PPid, {1006, "Connection closed abnormally"}, normal); - {close, WsReason} when Phase == open -> - loop(do_close(State, WsReason)) - end. - - -die(Socket, PPid, WsReason, Reason) -> - gen_tcp:shutdown(Socket, read_write), - PPid ! {rfc6455, close, self(), WsReason}, - exit(Reason). - - -%% -------------------------------------------------------------------------- - -split(SubStr, Str, Limit) -> - split(SubStr, Str, Limit, ""). - -split(SubStr, Str, Limit, Default) -> - Acc = split(SubStr, Str, Limit, [], Default), - lists:reverse(Acc). -split(_SubStr, Str, 0, Acc, _Default) -> [Str | Acc]; -split(SubStr, Str, Limit, Acc, Default) -> - {L, R} = case string:str(Str, SubStr) of - 0 -> {Str, Default}; - I -> {string:substr(Str, 1, I-1), - string:substr(Str, I+length(SubStr))} - end, - split(SubStr, R, Limit-1, [L | Acc], Default). - - -apply_mask(Mask, Data) when is_number(Mask) -> - apply_mask(<>, Data); - -apply_mask(<<0:32>>, Data) -> - Data; -apply_mask(Mask, Data) -> - iolist_to_binary(lists:reverse(apply_mask2(Mask, Data, []))). - -apply_mask2(M = <>, <>, Acc) -> - T = Data bxor Mask, - apply_mask2(M, Rest, [<> | Acc]); -apply_mask2(<>, <>, Acc) -> - T = Data bxor Mask, - [<> | Acc]; -apply_mask2(<>, <>, Acc) -> - T = Data bxor Mask, - [<> | Acc]; -apply_mask2(<>, <>, Acc) -> - T = Data bxor Mask, - [<> | Acc]; -apply_mask2(_, <<>>, Acc) -> - Acc. diff --git a/apps/emqx_management/test/test_utils.erl b/apps/emqx_management/test/test_utils.erl deleted file mode 100644 index 337a9499b..000000000 --- a/apps/emqx_management/test/test_utils.erl +++ /dev/null @@ -1,19 +0,0 @@ -%% @author: -%% @description: --module(test_utils). -%% ==================================================================== -%% API functions -%% ==================================================================== --include_lib("eunit/include/eunit.hrl"). --include_lib("emqx_rule_engine/include/rule_engine.hrl"). - --compile([export_all, nowarn_export_all]). - -%% ==================================================================== -%% Internal functions -%% ==================================================================== -resource_is_alive(Id) -> - {ok, #resource_params{status = #{is_alive := Alive}} = Params} = emqx_rule_registry:find_resource_params(Id), - ct:pal("Id: ~p, Alive: ~p, Resource ===> :~p~n", [Id, Alive, Params]), - ?assertEqual(true, Alive), - Alive. diff --git a/apps/emqx_modules/etc/emqx_modules.conf b/apps/emqx_modules/etc/emqx_modules.conf index 1bb8bf6d7..92f563342 100644 --- a/apps/emqx_modules/etc/emqx_modules.conf +++ b/apps/emqx_modules/etc/emqx_modules.conf @@ -1 +1,41 @@ -# empty +delayed: { + enable: true + max_delayed_messages: 0 +} + +recon: { + enable: true +} + +telemetry: { + enable: true +} + + +event_message: { + topics: [ + "$event/client_connected", + "$event/client_disconnected", + "$event/session_subscribed", + "$event/session_unsubscribed", + "$event/message_delivered", + "$event/message_acked", + "$event/message_dropped" + ] +} + +topic_metrics:{ + topics: ["topic/#"] +} + +rewrite:{ + rules: [ + { + action: publish + source_topic: "x/#" + re: "^x/y/(.+)$" + dest_topic: "z/y/$1" + } + ] +} + diff --git a/apps/emqx_modules/include/emqx_modules.hrl b/apps/emqx_modules/include/emqx_modules.hrl new file mode 100644 index 000000000..b7cdb154e --- /dev/null +++ b/apps/emqx_modules/include/emqx_modules.hrl @@ -0,0 +1,13 @@ +%% The destination URL for the telemetry data report +-define(TELEMETRY_URL, "https://telemetry.emqx.io/api/telemetry"). + +%% Interval for reporting telemetry data, Default: 7d +-define(REPORT_INTERVAR, 604800). + +-define(BASE_TOPICS, [<<"$event/client_connected">>, + <<"$event/client_disconnected">>, + <<"$event/session_subscribed">>, + <<"$event/session_unsubscribed">>, + <<"$event/message_delivered">>, + <<"$event/message_acked">>, + <<"$event/message_dropped">>]). diff --git a/apps/emqx_modules/priv/emqx_modules.schema b/apps/emqx_modules/priv/emqx_modules.schema deleted file mode 100644 index d7c52c644..000000000 --- a/apps/emqx_modules/priv/emqx_modules.schema +++ /dev/null @@ -1 +0,0 @@ -% empty diff --git a/apps/emqx_modules/src/emqx_mod_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl similarity index 77% rename from apps/emqx_modules/src/emqx_mod_delayed.erl rename to apps/emqx_modules/src/emqx_delayed.erl index ac5be58b2..2e0706cbd 100644 --- a/apps/emqx_modules/src/emqx_mod_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -14,10 +14,9 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_mod_delayed). +-module(emqx_delayed). -behaviour(gen_server). --behaviour(emqx_gen_mod). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -30,12 +29,6 @@ -boot_mnesia({mnesia, [boot]}). -copy_mnesia({mnesia, [copy]}). -%% emqx_gen_mod callbacks --export([ load/1 - , unload/1 - , description/0 - ]). - -export([ start_link/0 , on_message_publish/1 ]). @@ -49,19 +42,22 @@ , code_change/3 ]). --record(delayed_message, - { key - , msg - }). +%% gen_server callbacks +-export([ enable/0 + , disable/0 + ]). + +-record(delayed_message, {key, msg}). -define(TAB, ?MODULE). -define(SERVER, ?MODULE). -define(MAX_INTERVAL, 4294967). +-rlog_shard({?MOD_DELAYED_SHARD, ?TAB}). + %%-------------------------------------------------------------------- %% Mnesia bootstrap %%-------------------------------------------------------------------- - mnesia(boot) -> ok = ekka_mnesia:create_table(?TAB, [ {type, ordered_set}, @@ -73,25 +69,8 @@ mnesia(copy) -> ok = ekka_mnesia:copy_table(?TAB, disc_copies). %%-------------------------------------------------------------------- -%% Load/Unload -%%-------------------------------------------------------------------- - --spec(load(list()) -> ok). -load(_Env) -> - emqx_mod_sup:start_child(?MODULE, worker), - emqx:hook('message.publish', {?MODULE, on_message_publish, []}). - --spec(unload(list()) -> ok). -unload(_Env) -> - emqx:unhook('message.publish', {?MODULE, on_message_publish}), - emqx_mod_sup:stop_child(?MODULE). - -description() -> - "EMQ X Delayed Publish Module". -%%-------------------------------------------------------------------- %% Hooks %%-------------------------------------------------------------------- - on_message_publish(Msg = #message{ id = Id, topic = <<"$delayed/", Topic/binary>>, @@ -110,7 +89,11 @@ on_message_publish(Msg = #message{ end, PubMsg = Msg#message{topic = Topic1}, Headers = PubMsg#message.headers, - ok = store(#delayed_message{key = {PubAt, Id}, msg = PubMsg}), + case store(#delayed_message{key = {PubAt, Id}, msg = PubMsg}) of + ok -> ok; + {error, Error} -> + ?LOG(error, "Store delayed message fail: ~p", [Error]) + end, {stop, PubMsg#message{headers = Headers#{allow_publish => false}}}; on_message_publish(Msg) -> @@ -122,25 +105,56 @@ on_message_publish(Msg) -> -spec(start_link() -> emqx_types:startlink_ret()). start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + Opts = emqx_config:get([delayed], #{}), + gen_server:start_link({local, ?SERVER}, ?MODULE, [Opts], []). --spec(store(#delayed_message{}) -> ok). +-spec(store(#delayed_message{}) -> ok | {error, atom()}). store(DelayedMsg) -> gen_server:call(?SERVER, {store, DelayedMsg}, infinity). +enable() -> + gen_server:call(?SERVER, enable). + +disable() -> + gen_server:call(?SERVER, disable). + %%-------------------------------------------------------------------- %% gen_server callback %%-------------------------------------------------------------------- -init([]) -> +init([Opts]) -> + MaxDelayedMessages = maps:get(max_delayed_messages, Opts, 0), {ok, ensure_stats_event( - ensure_publish_timer(#{timer => undefined, publish_at => 0}))}. + ensure_publish_timer(#{timer => undefined, + publish_at => 0, + max_delayed_messages => MaxDelayedMessages}))}. -handle_call({store, DelayedMsg = #delayed_message{key = Key}}, _From, State) -> - ok = mnesia:dirty_write(?TAB, DelayedMsg), +handle_call({store, DelayedMsg = #delayed_message{key = Key}}, + _From, State = #{max_delayed_messages := 0}) -> + ok = ekka_mnesia:dirty_write(?TAB, DelayedMsg), emqx_metrics:inc('messages.delayed'), {reply, ok, ensure_publish_timer(Key, State)}; +handle_call({store, DelayedMsg = #delayed_message{key = Key}}, + _From, State = #{max_delayed_messages := Val}) -> + Size = mnesia:table_info(?TAB, size), + case Size > Val of + true -> + {reply, {error, max_delayed_messages_full}, State}; + false -> + ok = ekka_mnesia:dirty_write(?TAB, DelayedMsg), + emqx_metrics:inc('messages.delayed'), + {reply, ok, ensure_publish_timer(Key, State)} + end; + +handle_call(enable, _From, State) -> + emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}), + {reply, ok, State}; + +handle_call(disable, _From, State) -> + emqx_hooks:del('message.publish', {?MODULE, on_message_publish}), + {reply, ok, State}; + handle_call(Req, _From, State) -> ?LOG(error, "Unexpected call: ~p", [Req]), {reply, ignored, State}. @@ -152,7 +166,7 @@ handle_cast(Msg, State) -> %% Do Publish... handle_info({timeout, TRef, do_publish}, State = #{timer := TRef}) -> DeletedKeys = do_publish(mnesia:dirty_first(?TAB), os:system_time(seconds)), - lists:foreach(fun(Key) -> mnesia:dirty_delete(?TAB, Key) end, DeletedKeys), + lists:foreach(fun(Key) -> ekka_mnesia:dirty_delete(?TAB, Key) end, DeletedKeys), {noreply, ensure_publish_timer(State#{timer := undefined, publish_at := 0})}; handle_info(stats, State = #{stats_fun := StatsFun}) -> @@ -222,4 +236,3 @@ do_publish(Key = {Ts, _Id}, Now, Acc) when Ts =< Now -> -spec(delayed_count() -> non_neg_integer()). delayed_count() -> mnesia:table_info(?TAB, size). - diff --git a/apps/emqx_modules/src/emqx_event_message.erl b/apps/emqx_modules/src/emqx_event_message.erl new file mode 100644 index 000000000..3bdc54c2b --- /dev/null +++ b/apps/emqx_modules/src/emqx_event_message.erl @@ -0,0 +1,266 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_event_message). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include("emqx_modules.hrl"). + +-export([ enable/0 + , disable/0 + ]). + +-export([ on_client_connected/2 + , on_client_disconnected/3 + , on_session_subscribed/3 + , on_session_unsubscribed/3 + , on_message_dropped/3 + , on_message_delivered/2 + , on_message_acked/2 + ]). + +-ifdef(TEST). +-export([reason/1]). +-endif. + +enable() -> + Topics = emqx_config:get([event_message, topics], []), + lists:foreach(fun(Topic) -> + case Topic of + <<"$event/client_connected">> -> + emqx_hooks:put('client.connected', {?MODULE, on_client_connected, []}); + <<"$event/client_disconnected">> -> + emqx_hooks:put('client.disconnected', {?MODULE, on_client_disconnected, []}); + <<"$event/session_subscribed">> -> + emqx_hooks:put('session.subscribed', {?MODULE, on_session_subscribed, []}); + <<"$event/session_unsubscribed">> -> + emqx_hooks:put('session.unsubscribed', {?MODULE, on_session_unsubscribed, []}); + <<"$event/message_delivered">> -> + emqx_hooks:put('message.delivered', {?MODULE, on_message_delivered, []}); + <<"$event/message_acked">> -> + emqx_hooks:put('message.acked', {?MODULE, on_message_acked, []}); + <<"$event/message_dropped">> -> + emqx_hooks:put('message.dropped', {?MODULE, on_message_dropped, []}); + _ -> + ok + end + end, Topics). + +disable() -> + Topics = emqx_config:get([event_message, topics], []), + lists:foreach(fun(Topic) -> + case Topic of + <<"$event/client_connected">> -> + emqx_hooks:del('client.connected', {?MODULE, on_client_connected}); + <<"$event/client_disconnected">> -> + emqx_hooks:del('client.disconnected', {?MODULE, on_client_disconnected}); + <<"$event/session_subscribed">> -> + emqx_hooks:del('session.subscribed', {?MODULE, on_session_subscribed}); + <<"$event/session_unsubscribed">> -> + emqx_hooks:del('session.unsubscribed', {?MODULE, on_session_unsubscribed}); + <<"$event/message_delivered">> -> + emqx_hooks:del('message.delivered', {?MODULE, on_message_delivered}); + <<"$event/message_acked">> -> + emqx_hooks:del('message.acked', {?MODULE, on_message_acked}); + <<"$event/message_dropped">> -> + emqx_hooks:del('message.dropped', {?MODULE, on_message_dropped}); + _ -> + ok + end + end, ?BASE_TOPICS -- Topics). + +%%-------------------------------------------------------------------- +%% Callbacks +%%-------------------------------------------------------------------- + +on_client_connected(ClientInfo, ConnInfo) -> + Payload0 = common_infos(ClientInfo, ConnInfo), + Payload = Payload0#{ + connack => 0, %% XXX: connack will be removed in 5.0 + keepalive => maps:get(keepalive, ConnInfo, 0), + clean_start => maps:get(clean_start, ConnInfo, true), + expiry_interval => maps:get(expiry_interval, ConnInfo, 0), + connected_at => maps:get(connected_at, ConnInfo) + }, + publish_event_msg(<<"$event/client_connected">>, Payload). + +on_client_disconnected(ClientInfo, + Reason, ConnInfo = #{disconnected_at := DisconnectedAt}) -> + + Payload0 = common_infos(ClientInfo, ConnInfo), + Payload = Payload0#{ + reason => reason(Reason), + disconnected_at => DisconnectedAt + }, + publish_event_msg(<<"$event/client_disconnected">>, Payload). + +on_session_subscribed(_ClientInfo = #{clientid := ClientId, + username := Username}, + Topic, SubOpts) -> + Payload = #{clientid => ClientId, + username => Username, + topic => Topic, + subopts => SubOpts, + ts => erlang:system_time(millisecond) + }, + publish_event_msg(<<"$event/session_subscribed">>, Payload). + +on_session_unsubscribed(_ClientInfo = #{clientid := ClientId, + username := Username}, + Topic, _SubOpts) -> + Payload = #{clientid => ClientId, + username => Username, + topic => Topic, + ts => erlang:system_time(millisecond) + }, + publish_event_msg(<<"$event/session_unsubscribed">>, Payload). + +on_message_dropped(Message = #message{from = ClientId}, _, Reason) -> + case ignore_sys_message(Message) of + true -> ok; + false -> + Payload0 = base_message(Message), + Payload = Payload0#{ + reason => Reason, + clientid => ClientId, + username => emqx_message:get_header(username, Message, undefined), + peerhost => ntoa(emqx_message:get_header(peerhost, Message, undefined)) + }, + publish_event_msg(<<"$event/message_dropped">>, Payload) + end, + {ok, Message}. + +on_message_delivered(_ClientInfo = #{ + peerhost := PeerHost, + clientid := ReceiverCId, + username := ReceiverUsername}, + #message{from = ClientId} = Message) -> + case ignore_sys_message(Message) of + true -> ok; + false -> + Payload0 = base_message(Message), + Payload = Payload0#{ + from_clientid => ClientId, + from_username => emqx_message:get_header(username, Message, undefined), + clientid => ReceiverCId, + username => ReceiverUsername, + peerhost => ntoa(PeerHost) + }, + publish_event_msg(<<"$event/message_delivered">>, Payload) + end, + {ok, Message}. + +on_message_acked(_ClientInfo = #{ + peerhost := PeerHost, + clientid := ReceiverCId, + username := ReceiverUsername}, + #message{from = ClientId} = Message) -> + case ignore_sys_message(Message) of + true -> ok; + false -> + Payload0 = base_message(Message), + Payload = Payload0#{ + from_clientid => ClientId, + from_username => emqx_message:get_header(username, Message, undefined), + clientid => ReceiverCId, + username => ReceiverUsername, + peerhost => ntoa(PeerHost) + }, + publish_event_msg(<<"$event/message_acked">>, Payload) + end, + {ok, Message}. + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- +common_infos( + _ClientInfo = #{clientid := ClientId, + username := Username, + peerhost := PeerHost, + sockport := SockPort + }, + _ConnInfo = #{proto_name := ProtoName, + proto_ver := ProtoVer + }) -> + #{clientid => ClientId, + username => Username, + ipaddress => ntoa(PeerHost), + sockport => SockPort, + proto_name => ProtoName, + proto_ver => ProtoVer, + ts => erlang:system_time(millisecond) + }. + +make_msg(Topic, Payload) -> + emqx_message:set_flag( + sys, emqx_message:make( + ?MODULE, 0, Topic, iolist_to_binary(Payload))). + +-compile({inline, [reason/1]}). +reason(Reason) when is_atom(Reason) -> Reason; +reason({shutdown, Reason}) when is_atom(Reason) -> Reason; +reason({Error, _}) when is_atom(Error) -> Error; +reason(_) -> internal_error. + +ntoa(undefined) -> undefined; +ntoa({IpAddr, Port}) -> + iolist_to_binary([inet:ntoa(IpAddr), ":", integer_to_list(Port)]); +ntoa(IpAddr) -> + iolist_to_binary(inet:ntoa(IpAddr)). + +printable_maps(undefined) -> #{}; +printable_maps(Headers) -> + maps:fold( + fun (K, V0, AccIn) when K =:= peerhost; K =:= peername; K =:= sockname -> + AccIn#{K => ntoa(V0)}; + ('User-Property', V0, AccIn) when is_list(V0) -> + AccIn#{ + 'User-Property' => maps:from_list(V0), + 'User-Property-Pairs' => [#{ + key => Key, + value => Value + } || {Key, Value} <- V0] + }; + (K, V0, AccIn) -> AccIn#{K => V0} + end, #{}, Headers). + +base_message(Message) -> + #message{ + id = Id, + qos = QoS, + flags = Flags, + topic = Topic, + headers = Headers, + payload = Payload, + timestamp = Timestamp} = Message, + #{ + id => emqx_guid:to_hexstr(Id), + payload => Payload, + topic => Topic, + qos => QoS, + flags => Flags, + headers => printable_maps(Headers), + pub_props => printable_maps(emqx_message:get_header(properties, Message, #{})), + publish_received_at => Timestamp + }. + +ignore_sys_message(#message{flags = Flags}) -> + maps:get(sys, Flags, false). + +publish_event_msg(Topic, Payload) -> + _ = emqx_broker:safe_publish(make_msg(Topic, emqx_json:encode(Payload))), + ok. diff --git a/apps/emqx_modules/src/emqx_mod_api_topic_metrics.erl b/apps/emqx_modules/src/emqx_mod_api_topic_metrics.erl deleted file mode 100644 index 5ccef4c6b..000000000 --- a/apps/emqx_modules/src/emqx_mod_api_topic_metrics.erl +++ /dev/null @@ -1,209 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_mod_api_topic_metrics). - --import(minirest, [return/1]). - --rest_api(#{name => list_all_topic_metrics, - method => 'GET', - path => "/topic-metrics", - func => list, - descr => "A list of all topic metrics of all nodes in the cluster"}). - --rest_api(#{name => list_topic_metrics, - method => 'GET', - path => "/topic-metrics/:bin:topic", - func => list, - descr => "A list of specfied topic metrics of all nodes in the cluster"}). - --rest_api(#{name => register_topic_metrics, - method => 'POST', - path => "/topic-metrics", - func => register, - descr => "Register topic metrics"}). - --rest_api(#{name => unregister_all_topic_metrics, - method => 'DELETE', - path => "/topic-metrics", - func => unregister, - descr => "Unregister all topic metrics"}). - --rest_api(#{name => unregister_topic_metrics, - method => 'DELETE', - path => "/topic-metrics/:bin:topic", - func => unregister, - descr => "Unregister topic metrics"}). - --export([ list/2 - , register/2 - , unregister/2 - ]). - --export([ get_topic_metrics/2 - , register_topic_metrics/2 - , unregister_topic_metrics/2 - , unregister_all_topic_metrics/1 - ]). - -list(#{topic := Topic0}, _Params) -> - execute_when_enabled(fun() -> - Topic = emqx_mgmt_util:urldecode(Topic0), - case safe_validate(Topic) of - true -> - case get_topic_metrics(Topic) of - {error, Reason} -> return({error, Reason}); - Metrics -> return({ok, maps:from_list(Metrics)}) - end; - false -> - return({error, invalid_topic_name}) - end - end); - -list(_Bindings, _Params) -> - execute_when_enabled(fun() -> - case get_all_topic_metrics() of - {error, Reason} -> return({error, Reason}); - Metrics -> return({ok, Metrics}) - end - end). - -register(_Bindings, Params) -> - execute_when_enabled(fun() -> - case proplists:get_value(<<"topic">>, Params) of - undefined -> - return({error, missing_required_params}); - Topic -> - case safe_validate(Topic) of - true -> - register_topic_metrics(Topic), - return(ok); - false -> - return({error, invalid_topic_name}) - end - end - end). - -unregister(Bindings, _Params) when map_size(Bindings) =:= 0 -> - execute_when_enabled(fun() -> - unregister_all_topic_metrics(), - return(ok) - end); - -unregister(#{topic := Topic0}, _Params) -> - execute_when_enabled(fun() -> - Topic = emqx_mgmt_util:urldecode(Topic0), - case safe_validate(Topic) of - true -> - unregister_topic_metrics(Topic), - return(ok); - false -> - return({error, invalid_topic_name}) - end - end). - -execute_when_enabled(Fun) -> - Enabled = case emqx_modules:find_module(emqx_mod_topic_metrics) of - [{_, false}] -> false; - [{_, true}] -> true - end, - case Enabled of - true -> - Fun(); - false -> - return({error, module_not_loaded}) - end. - -safe_validate(Topic) -> - try emqx_topic:validate(name, Topic) of - true -> true - catch - error:_Error -> - false - end. - -get_all_topic_metrics() -> - lists:foldl(fun(Topic, Acc) -> - case get_topic_metrics(Topic) of - {error, _Reason} -> - Acc; - Metrics -> - [#{topic => Topic, metrics => Metrics} | Acc] - end - end, [], emqx_mod_topic_metrics:all_registered_topics()). - -get_topic_metrics(Topic) -> - lists:foldl(fun(Node, Acc) -> - case get_topic_metrics(Node, Topic) of - {error, _Reason} -> - Acc; - Metrics -> - case Acc of - [] -> Metrics; - _ -> - lists:foldl(fun({K, V}, Acc0) -> - [{K, V + proplists:get_value(K, Metrics, 0)} | Acc0] - end, [], Acc) - end - end - end, [], ekka_mnesia:running_nodes()). - -get_topic_metrics(Node, Topic) when Node =:= node() -> - emqx_mod_topic_metrics:metrics(Topic); -get_topic_metrics(Node, Topic) -> - rpc_call(Node, get_topic_metrics, [Node, Topic]). - -register_topic_metrics(Topic) -> - Results = [register_topic_metrics(Node, Topic) || Node <- ekka_mnesia:running_nodes()], - case lists:any(fun(Item) -> Item =:= ok end, Results) of - true -> ok; - false -> lists:last(Results) - end. - -register_topic_metrics(Node, Topic) when Node =:= node() -> - emqx_mod_topic_metrics:register(Topic); -register_topic_metrics(Node, Topic) -> - rpc_call(Node, register_topic_metrics, [Node, Topic]). - -unregister_topic_metrics(Topic) -> - Results = [unregister_topic_metrics(Node, Topic) || Node <- ekka_mnesia:running_nodes()], - case lists:any(fun(Item) -> Item =:= ok end, Results) of - true -> ok; - false -> lists:last(Results) - end. - -unregister_topic_metrics(Node, Topic) when Node =:= node() -> - emqx_mod_topic_metrics:unregister(Topic); -unregister_topic_metrics(Node, Topic) -> - rpc_call(Node, unregister_topic_metrics, [Node, Topic]). - -unregister_all_topic_metrics() -> - Results = [unregister_all_topic_metrics(Node) || Node <- ekka_mnesia:running_nodes()], - case lists:any(fun(Item) -> Item =:= ok end, Results) of - true -> ok; - false -> lists:last(Results) - end. - -unregister_all_topic_metrics(Node) when Node =:= node() -> - emqx_mod_topic_metrics:unregister_all(); -unregister_all_topic_metrics(Node) -> - rpc_call(Node, unregister_topic_metrics, [Node]). - -rpc_call(Node, Fun, Args) -> - case rpc:call(Node, ?MODULE, Fun, Args) of - {badrpc, Reason} -> {error, Reason}; - Res -> Res - end. diff --git a/apps/emqx_modules/src/emqx_mod_presence.erl b/apps/emqx_modules/src/emqx_mod_presence.erl deleted file mode 100644 index 7ba147c9a..000000000 --- a/apps/emqx_modules/src/emqx_mod_presence.erl +++ /dev/null @@ -1,130 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_mod_presence). - --behaviour(emqx_gen_mod). - --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/logger.hrl"). - --logger_header("[Presence]"). - -%% emqx_gen_mod callbacks --export([ load/1 - , unload/1 - , description/0 - ]). - --export([ on_client_connected/3 - , on_client_disconnected/4 - ]). - --ifdef(TEST). --export([reason/1]). --endif. - -load(Env) -> - emqx_hooks:put('client.connected', {?MODULE, on_client_connected, [Env]}), - emqx_hooks:put('client.disconnected', {?MODULE, on_client_disconnected, [Env]}). - -unload(_Env) -> - emqx_hooks:del('client.connected', {?MODULE, on_client_connected}), - emqx_hooks:del('client.disconnected', {?MODULE, on_client_disconnected}). - -description() -> - "EMQ X Presence Module". -%%-------------------------------------------------------------------- -%% Callbacks -%%-------------------------------------------------------------------- - -on_client_connected(ClientInfo = #{clientid := ClientId}, ConnInfo, Env) -> - Presence = connected_presence(ClientInfo, ConnInfo), - case emqx_json:safe_encode(Presence) of - {ok, Payload} -> - emqx_broker:safe_publish( - make_msg(qos(Env), topic(connected, ClientId), Payload)); - {error, _Reason} -> - ?LOG(error, "Failed to encode 'connected' presence: ~p", [Presence]) - end. - -on_client_disconnected(_ClientInfo = #{clientid := ClientId, username := Username}, - Reason, _ConnInfo = #{disconnected_at := DisconnectedAt}, Env) -> - Presence = #{clientid => ClientId, - username => Username, - reason => reason(Reason), - disconnected_at => DisconnectedAt, - ts => erlang:system_time(millisecond) - }, - case emqx_json:safe_encode(Presence) of - {ok, Payload} -> - emqx_broker:safe_publish( - make_msg(qos(Env), topic(disconnected, ClientId), Payload)); - {error, _Reason} -> - ?LOG(error, "Failed to encode 'disconnected' presence: ~p", [Presence]) - end. - -%%-------------------------------------------------------------------- -%% Helper functions -%%-------------------------------------------------------------------- - -connected_presence(#{peerhost := PeerHost, - sockport := SockPort, - clientid := ClientId, - username := Username - }, - #{clean_start := CleanStart, - proto_name := ProtoName, - proto_ver := ProtoVer, - keepalive := Keepalive, - connected_at := ConnectedAt, - expiry_interval := ExpiryInterval - }) -> - #{clientid => ClientId, - username => Username, - ipaddress => ntoa(PeerHost), - sockport => SockPort, - proto_name => ProtoName, - proto_ver => ProtoVer, - keepalive => Keepalive, - connack => 0, %% Deprecated? - clean_start => CleanStart, - expiry_interval => ExpiryInterval, - connected_at => ConnectedAt, - ts => erlang:system_time(millisecond) - }. - -make_msg(QoS, Topic, Payload) -> - emqx_message:set_flag( - sys, emqx_message:make( - ?MODULE, QoS, Topic, iolist_to_binary(Payload))). - -topic(connected, ClientId) -> - emqx_topic:systop(iolist_to_binary(["clients/", ClientId, "/connected"])); -topic(disconnected, ClientId) -> - emqx_topic:systop(iolist_to_binary(["clients/", ClientId, "/disconnected"])). - -qos(Env) -> proplists:get_value(qos, Env, 0). - --compile({inline, [reason/1]}). -reason(Reason) when is_atom(Reason) -> Reason; -reason({shutdown, Reason}) when is_atom(Reason) -> Reason; -reason({Error, _}) when is_atom(Error) -> Error; -reason(_) -> internal_error. - --compile({inline, [ntoa/1]}). -ntoa(IpAddr) -> iolist_to_binary(inet:ntoa(IpAddr)). - diff --git a/apps/emqx_modules/src/emqx_mod_subscription.erl b/apps/emqx_modules/src/emqx_mod_subscription.erl deleted file mode 100644 index 06178aee7..000000000 --- a/apps/emqx_modules/src/emqx_mod_subscription.erl +++ /dev/null @@ -1,65 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_mod_subscription). - --behaviour(emqx_gen_mod). - --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). - -%% emqx_gen_mod callbacks --export([ load/1 - , unload/1 - , description/0 - ]). - -%% APIs --export([on_client_connected/3]). - -%%-------------------------------------------------------------------- -%% Load/Unload Hook -%%-------------------------------------------------------------------- - -load(Topics) -> - emqx_hooks:add('client.connected', {?MODULE, on_client_connected, [Topics]}). - -on_client_connected(#{clientid := ClientId, username := Username}, _ConnInfo = #{proto_ver := ProtoVer}, Topics) -> - Replace = fun(Topic) -> - rep(<<"%u">>, Username, rep(<<"%c">>, ClientId, Topic)) - end, - TopicFilters = case ProtoVer of - ?MQTT_PROTO_V5 -> [{Replace(Topic), SubOpts} || {Topic, SubOpts} <- Topics]; - _ -> [{Replace(Topic), #{qos => Qos}} || {Topic, #{qos := Qos}} <- Topics] - end, - self() ! {subscribe, TopicFilters}. - -unload(_) -> - emqx_hooks:del('client.connected', {?MODULE, on_client_connected}). - -description() -> - "EMQ X Subscription Module". -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -rep(<<"%c">>, ClientId, Topic) -> - emqx_topic:feed_var(<<"%c">>, ClientId, Topic); -rep(<<"%u">>, undefined, Topic) -> - Topic; -rep(<<"%u">>, Username, Topic) -> - emqx_topic:feed_var(<<"%u">>, Username, Topic). - diff --git a/apps/emqx_modules/src/emqx_mod_sup.erl b/apps/emqx_modules/src/emqx_mod_sup.erl deleted file mode 100644 index 755e52a60..000000000 --- a/apps/emqx_modules/src/emqx_mod_sup.erl +++ /dev/null @@ -1,74 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_mod_sup). - --behaviour(supervisor). - --include_lib("emqx/include/types.hrl"). - --export([ start_link/0 - , start_child/1 - , start_child/2 - , stop_child/1 - ]). - --export([init/1]). - -%% Helper macro for declaring children of supervisor --define(CHILD(Mod, Type), #{id => Mod, - start => {Mod, start_link, []}, - restart => permanent, - shutdown => 5000, - type => Type, - modules => [Mod]}). - --spec(start_link() -> startlink_ret()). -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - --spec start_child(supervisor:child_spec()) -> ok. -start_child(ChildSpec) when is_map(ChildSpec) -> - assert_started(supervisor:start_child(?MODULE, ChildSpec)). - --spec start_child(atom(), atom()) -> ok. -start_child(Mod, Type) when is_atom(Mod) andalso is_atom(Type) -> - assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Type))). - --spec(stop_child(any()) -> ok | {error, term()}). -stop_child(ChildId) -> - case supervisor:terminate_child(?MODULE, ChildId) of - ok -> supervisor:delete_child(?MODULE, ChildId); - Error -> Error - end. - -%%-------------------------------------------------------------------- -%% Supervisor callbacks -%%-------------------------------------------------------------------- - -init([]) -> - ok = emqx_tables:new(emqx_modules, [set, public, {write_concurrency, true}]), - {ok, {{one_for_one, 10, 100}, []}}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -assert_started({ok, _Pid}) -> ok; -assert_started({ok, _Pid, _Info}) -> ok; -assert_started({error, {already_tarted, _Pid}}) -> ok; -assert_started({error, Reason}) -> erlang:error(Reason). - diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src index 702652fc2..155f581c0 100644 --- a/apps/emqx_modules/src/emqx_modules.app.src +++ b/apps/emqx_modules/src/emqx_modules.app.src @@ -1,9 +1,9 @@ {application, emqx_modules, - [{description, "EMQ X Module Management"}, - {vsn, "4.3.2"}, + [{description, "EMQ X Modules"}, + {vsn, "5.0.0"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}}, - {registered, [emqx_mod_sup]}, + {registered, [emqx_modules_sup]}, {env, []} ]}. diff --git a/apps/emqx_modules/src/emqx_modules.appup.src b/apps/emqx_modules/src/emqx_modules.appup.src deleted file mode 100644 index aa997c453..000000000 --- a/apps/emqx_modules/src/emqx_modules.appup.src +++ /dev/null @@ -1,23 +0,0 @@ -%% -*-: erlang -*- -{VSN, - [ - {"4.3.1", [ - {load_module, emqx_mod_api_topic_metrics, brutal_purge, soft_purge, []} - ]}, - {"4.3.0", [ - {update, emqx_mod_delayed, {advanced, []}}, - {load_module, emqx_mod_api_topic_metrics, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ], - [ - {"4.3.1", [ - {load_module, emqx_mod_api_topic_metrics, brutal_purge, soft_purge, []} - ]}, - {"4.3.0", [ - {update, emqx_mod_delayed, {advanced, []}}, - {load_module, emqx_mod_api_topic_metrics, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ] -}. diff --git a/apps/emqx_modules/src/emqx_modules.erl b/apps/emqx_modules/src/emqx_modules.erl deleted file mode 100644 index 262e700ab..000000000 --- a/apps/emqx_modules/src/emqx_modules.erl +++ /dev/null @@ -1,197 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_modules). - --include_lib("emqx/include/logger.hrl"). - --logger_header("[Modules]"). - --export([ list/0 - , load/0 - , load/1 - , unload/0 - , unload/1 - , reload/1 - , find_module/1 - , load_module/2 - ]). - --export([cli/1]). - -%% @doc List all available plugins --spec(list() -> [{atom(), boolean()}]). -list() -> - ets:tab2list(?MODULE). - -%% @doc Load all the extended modules. --spec(load() -> ok). -load() -> - case emqx:get_env(modules_loaded_file) of - undefined -> ok; - File -> - load_modules(File) - end. - -load(ModuleName) -> - case find_module(ModuleName) of - [] -> - ?LOG(alert, "Module ~s not found, cannot load it", [ModuleName]), - {error, not_found}; - [{ModuleName, true}] -> - ?LOG(notice, "Module ~s is already started", [ModuleName]), - {error, already_started}; - [{ModuleName, false}] -> - emqx_modules:load_module(ModuleName, true) - end. - -%% @doc Unload all the extended modules. --spec(unload() -> ok). -unload() -> - case emqx:get_env(modules_loaded_file) of - undefined -> ignore; - File -> - unload_modules(File) - end. - -unload(ModuleName) -> - case find_module(ModuleName) of - [] -> - ?LOG(alert, "Module ~s not found, cannot load it", [ModuleName]), - {error, not_found}; - [{ModuleName, false}] -> - ?LOG(error, "Module ~s is not started", [ModuleName]), - {error, not_started}; - [{ModuleName, true}] -> - unload_module(ModuleName, true) - end. - --spec(reload(module()) -> ok | ignore | {error, any()}). -reload(_) -> - ignore. - -find_module(ModuleName) -> - ets:lookup(?MODULE, ModuleName). - -filter_module(ModuleNames) -> - filter_module(ModuleNames, emqx:get_env(modules, [])). -filter_module([], Acc) -> - Acc; -filter_module([{ModuleName, true} | ModuleNames], Acc) -> - filter_module(ModuleNames, lists:keydelete(ModuleName, 1, Acc)); -filter_module([{_, false} | ModuleNames], Acc) -> - filter_module(ModuleNames, Acc). - -load_modules(File) -> - case file:consult(File) of - {ok, ModuleNames} -> - lists:foreach(fun({ModuleName, _}) -> - ets:insert(?MODULE, {ModuleName, false}) - end, filter_module(ModuleNames)), - lists:foreach(fun load_module/1, ModuleNames); - {error, Error} -> - ?LOG(alert, "Failed to read: ~p, error: ~p", [File, Error]) - end. - -load_module({ModuleName, true}) -> - emqx_modules:load_module(ModuleName, false); -load_module({ModuleName, false}) -> - ets:insert(?MODULE, {ModuleName, false}); -load_module(ModuleName) -> - load_module({ModuleName, true}). - -load_module(ModuleName, Persistent) -> - Modules = emqx:get_env(modules, []), - Env = proplists:get_value(ModuleName, Modules, undefined), - case ModuleName:load(Env) of - ok -> - ets:insert(?MODULE, {ModuleName, true}), - ok = write_loaded(Persistent), - ?LOG(info, "Load ~s module successfully.", [ModuleName]); - {error, Error} -> - ?LOG(error, "Load module ~s failed, cannot load for ~0p", [ModuleName, Error]), - {error, Error} - end. - -unload_modules(File) -> - case file:consult(File) of - {ok, ModuleNames} -> - lists:foreach(fun unload_module/1, ModuleNames); - {error, Error} -> - ?LOG(alert, "Failed to read: ~p, error: ~p", [File, Error]) - end. -unload_module({ModuleName, true}) -> - unload_module(ModuleName, false); -unload_module({ModuleName, false}) -> - ets:insert(?MODULE, {ModuleName, false}); -unload_module(ModuleName) -> - unload_module({ModuleName, true}). - -unload_module(ModuleName, Persistent) -> - Modules = emqx:get_env(modules, []), - Env = proplists:get_value(ModuleName, Modules, undefined), - case ModuleName:unload(Env) of - ok -> - ets:insert(?MODULE, {ModuleName, false}), - ok = write_loaded(Persistent), - ?LOG(info, "Unload ~s module successfully.", [ModuleName]); - {error, Error} -> - ?LOG(error, "Unload module ~s failed, cannot unload for ~0p", [ModuleName, Error]) - end. - -write_loaded(true) -> - FilePath = emqx:get_env(modules_loaded_file), - case file:write_file(FilePath, [io_lib:format("~p.~n", [Name]) || Name <- list()]) of - ok -> ok; - {error, Error} -> - ?LOG(error, "Write File ~p Error: ~p", [FilePath, Error]), - ok - end; -write_loaded(false) -> ok. - -%%-------------------------------------------------------------------- -%% @doc Modules Command -cli(["list"]) -> - lists:foreach(fun({Name, Active}) -> - emqx_ctl:print("Module(~s, description=~s, active=~s)~n", - [Name, Name:description(), Active]) - end, emqx_modules:list()); - -cli(["load", Name]) -> - case emqx_modules:load(list_to_atom(Name)) of - ok -> - emqx_ctl:print("Module ~s loaded successfully.~n", [Name]); - {error, Reason} -> - emqx_ctl:print("Load module ~s error: ~p.~n", [Name, Reason]) - end; - -cli(["unload", Name]) -> - case emqx_modules:unload(list_to_atom(Name)) of - ok -> - emqx_ctl:print("Module ~s unloaded successfully.~n", [Name]); - {error, Reason} -> - emqx_ctl:print("Unload module ~s error: ~p.~n", [Name, Reason]) - end; - -cli(["reload", Name]) -> - emqx_ctl:print("Module: ~p does not need to be reloaded.~n", [Name]); - -cli(_) -> - emqx_ctl:usage([{"modules list", "Show loaded modules"}, - {"modules load ", "Load module"}, - {"modules unload ", "Unload module"}, - {"modules reload ", "Reload module"} - ]). diff --git a/apps/emqx_modules/src/emqx_modules_api.erl b/apps/emqx_modules/src/emqx_modules_api.erl deleted file mode 100644 index 3490c116c..000000000 --- a/apps/emqx_modules/src/emqx_modules_api.erl +++ /dev/null @@ -1,164 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_modules_api). - --import(minirest, [return/1]). - --rest_api(#{name => list_all_modules, - method => 'GET', - path => "/modules/", - func => list, - descr => "List all modules in the cluster"}). - --rest_api(#{name => list_node_modules, - method => 'GET', - path => "/nodes/:atom:node/modules/", - func => list, - descr => "List all modules on a node"}). - --rest_api(#{name => load_node_module, - method => 'PUT', - path => "/nodes/:atom:node/modules/:atom:module/load", - func => load, - descr => "Load a module"}). - --rest_api(#{name => unload_node_module, - method => 'PUT', - path => "/nodes/:atom:node/modules/:atom:module/unload", - func => unload, - descr => "Unload a module"}). - --rest_api(#{name => reload_node_module, - method => 'PUT', - path => "/nodes/:atom:node/modules/:atom:module/reload", - func => reload, - descr => "Reload a module"}). - --rest_api(#{name => load_module, - method => 'PUT', - path => "/modules/:atom:module/load", - func => load, - descr => "load a module in the cluster"}). - --rest_api(#{name => unload_module, - method => 'PUT', - path => "/modules/:atom:module/unload", - func => unload, - descr => "Unload a module in the cluster"}). - --rest_api(#{name => reload_module, - method => 'PUT', - path => "/modules/:atom:module/reload", - func => reload, - descr => "Reload a module in the cluster"}). - --export([ list/2 - , list_modules/1 - , load/2 - , unload/2 - , reload/2 - ]). - --export([ do_load_module/2 - , do_unload_module/2 - ]). - -list(#{node := Node}, _Params) -> - return({ok, [format(Module) || Module <- list_modules(Node)]}); - -list(_Bindings, _Params) -> - return({ok, [format(Node, Modules) || {Node, Modules} <- list_modules()]}). - -load(#{node := Node, module := Module}, _Params) -> - return(do_load_module(Node, Module)); - -load(#{module := Module}, _Params) -> - Results = [do_load_module(Node, Module) || Node <- ekka_mnesia:running_nodes()], - case lists:filter(fun(Item) -> Item =/= ok end, Results) of - [] -> - return(ok); - Errors -> - return(lists:last(Errors)) - end. - -unload(#{node := Node, module := Module}, _Params) -> - return(do_unload_module(Node, Module)); - -unload(#{module := Module}, _Params) -> - Results = [do_unload_module(Node, Module) || Node <- ekka_mnesia:running_nodes()], - case lists:filter(fun(Item) -> Item =/= ok end, Results) of - [] -> - return(ok); - Errors -> - return(lists:last(Errors)) - end. - -reload(#{node := Node, module := Module}, _Params) -> - case reload_module(Node, Module) of - ignore -> return(ok); - Result -> return(Result) - end; - -reload(#{module := Module}, _Params) -> - Results = [reload_module(Node, Module) || Node <- ekka_mnesia:running_nodes()], - case lists:filter(fun(Item) -> Item =/= ok end, Results) of - [] -> - return(ok); - Errors -> - return(lists:last(Errors)) - end. - -%%------------------------------------------------------------------------------ -%% Internal Functions -%%------------------------------------------------------------------------------ - -format(Node, Modules) -> - #{node => Node, modules => [format(Module) || Module <- Modules]}. - -format({Name, Active}) -> - #{name => Name, - description => iolist_to_binary(Name:description()), - active => Active}. - -list_modules() -> - [{Node, list_modules(Node)} || Node <- ekka_mnesia:running_nodes()]. - -list_modules(Node) when Node =:= node() -> - emqx_modules:list(); -list_modules(Node) -> - rpc_call(Node, list_modules, [Node]). - -do_load_module(Node, Module) when Node =:= node() -> - emqx_modules:load(Module); -do_load_module(Node, Module) -> - rpc_call(Node, do_load_module, [Node, Module]). - -do_unload_module(Node, Module) when Node =:= node() -> - emqx_modules:unload(Module); -do_unload_module(Node, Module) -> - rpc_call(Node, do_unload_module, [Node, Module]). - -reload_module(Node, Module) when Node =:= node() -> - emqx_modules:reload(Module); -reload_module(Node, Module) -> - rpc_call(Node, reload_module, [Node, Module]). - -rpc_call(Node, Fun, Args) -> - case rpc:call(Node, ?MODULE, Fun, Args) of - {badrpc, Reason} -> {error, Reason}; - Res -> Res - end. diff --git a/apps/emqx_modules/src/emqx_modules_app.erl b/apps/emqx_modules/src/emqx_modules_app.erl index a10176829..e969fc6dd 100644 --- a/apps/emqx_modules/src/emqx_modules_app.erl +++ b/apps/emqx_modules/src/emqx_modules_app.erl @@ -18,19 +18,31 @@ -behaviour(application). --export([start/2]). - --export([stop/1]). +-export([ start/2 + , stop/1 + ]). start(_Type, _Args) -> - % the configs for emqx_modules is so far still in emqx application - % Ensure it's loaded - _ = application:load(emqx), - {ok, Pid} = emqx_mod_sup:start_link(), - ok = emqx_modules:load(), - emqx_ctl:register_command(modules, {emqx_modules, cli}, []), - {ok, Pid}. + {ok, Sup} = emqx_modules_sup:start_link(), + maybe_enable_modules(), + {ok, Sup}. stop(_State) -> - emqx_ctl:unregister_command(modules), - emqx_modules:unload(). + maybe_disable_modules(), + ok. + +maybe_enable_modules() -> + emqx_config:get([delayed, enable], true) andalso emqx_delayed:enable(), + emqx_config:get([telemetry, enable], true) andalso emqx_telemetry:enable(), + emqx_config:get([recon, enable], true) andalso emqx_recon:enable(), + emqx_event_message:enable(), + emqx_rewrite:enable(), + emqx_topic_metrics:enable(). + +maybe_disable_modules() -> + emqx_config:get([delayed, enable], true) andalso emqx_delayed:disable(), + emqx_config:get([telemetry, enable], true) andalso emqx_telemetry:disable(), + emqx_config:get([recon, enable], true) andalso emqx_recon:disable(), + emqx_event_message:disable(), + emqx_rewrite:disable(), + emqx_topic_metrics:disable(). diff --git a/apps/emqx_modules/src/emqx_modules_schema.erl b/apps/emqx_modules/src/emqx_modules_schema.erl new file mode 100644 index 000000000..0097fdbbe --- /dev/null +++ b/apps/emqx_modules/src/emqx_modules_schema.erl @@ -0,0 +1,78 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_modules_schema). + +-include_lib("typerefl/include/types.hrl"). + +-behaviour(hocon_schema). + +-export([ structs/0 + , fields/1]). + +structs() -> + ["delayed", + "recon", + "telemetry", + "event_message", + "rewrite", + "topic_metrics"]. + +fields(Name) when Name =:= "recon"; + Name =:= "telemetry" -> + [ {enable, emqx_schema:t(boolean(), undefined, false)} + ]; + +fields("delayed") -> + [ {enable, emqx_schema:t(boolean(), undefined, false)} + , {max_delayed_messages, emqx_schema:t(integer())} + ]; + +fields("rewrite") -> + [ {rules, hoconsc:array(hoconsc:ref(?MODULE, "rules"))} + ]; + +fields("event_message") -> + [ {topics, fun topics/1} + ]; + +fields("topic_metrics") -> + [ {topics, hoconsc:array(binary())} + ]; + +fields("rules") -> + [ {action, hoconsc:enum([publish, subscribe])} + , {source_topic, emqx_schema:t(binary())} + , {re, emqx_schema:t(binary())} + , {dest_topic, emqx_schema:t(binary())} + ]. + +topics(type) -> hoconsc:array(binary()); +topics(default) -> []; +% topics(validator) -> [ +% fun(Conf) -> +% case lists:member(Conf, ["$event/client_connected", +% "$event/client_disconnected", +% "$event/session_subscribed", +% "$event/session_unsubscribed", +% "$event/message_delivered", +% "$event/message_acked", +% "$event/message_dropped"]) of +% true -> ok; +% false -> {error, "Bad event topic"} +% end +% end]; +topics(_) -> undefined. diff --git a/apps/emqx_web_hook/src/emqx_web_hook_sup.erl b/apps/emqx_modules/src/emqx_modules_sup.erl similarity index 59% rename from apps/emqx_web_hook/src/emqx_web_hook_sup.erl rename to apps/emqx_modules/src/emqx_modules_sup.erl index ec46efaa0..570082896 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_sup.erl +++ b/apps/emqx_modules/src/emqx_modules_sup.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_web_hook_sup). +-module(emqx_modules_sup). -behaviour(supervisor). @@ -22,8 +22,22 @@ -export([init/1]). +%% Helper macro for declaring children of supervisor +-define(CHILD(Mod), #{id => Mod, + start => {Mod, start_link, []}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [Mod]}). + start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). +%%-------------------------------------------------------------------- +%% Supervisor callbacks +%%-------------------------------------------------------------------- init([]) -> - {ok, {{one_for_all, 0, 1}, []}}. + {ok, {{one_for_one, 10, 3600}, + [?CHILD(emqx_telemetry), + ?CHILD(emqx_topic_metrics), + ?CHILD(emqx_delayed)]}}. diff --git a/apps/emqx_modules/src/emqx_mod_recon.erl b/apps/emqx_modules/src/emqx_recon.erl similarity index 89% rename from apps/emqx_modules/src/emqx_mod_recon.erl rename to apps/emqx_modules/src/emqx_recon.erl index 38d046b10..b16d1b051 100644 --- a/apps/emqx_modules/src/emqx_mod_recon.erl +++ b/apps/emqx_modules/src/emqx_recon.erl @@ -14,37 +14,24 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_mod_recon). +-module(emqx_recon). --behaviour(emqx_gen_mod). - -%% emqx_gen_mod callbacks --export([ load/1 - , unload/1 - , description/0 +-export([ enable/0 + , disable/0 ]). -export([cmd/1]). %%-------------------------------------------------------------------- -%% Load/Unload +%% enable/disable %%-------------------------------------------------------------------- - --spec(load(list()) -> ok). -load(_Env) -> - load(). - --spec(unload(list()) -> ok). -unload(_Env) -> - unload(). - -description() -> - "EMQ X Recon Module". - -load() -> +enable() -> emqx_ctl:register_command(recon, {?MODULE, cmd}, []). +disable() -> + emqx_ctl:unregister_command(recon). + cmd(["memory"]) -> Print = fun(Key, Keyword) -> emqx_ctl:print("~-20s: ~w~n", [concat(Key, Keyword), recon_alloc:memory(Key, Keyword)]) @@ -76,9 +63,6 @@ cmd(_) -> {"recon remote_load Mod", "recon:remote_load(Mod)"}, {"recon proc_count Attr N","recon:proc_count(Attr, N)"}]). -unload() -> - emqx_ctl:unregister_command(recon). - concat(Key, Keyword) -> lists:concat([atom_to_list(Key), "/", atom_to_list(Keyword)]). diff --git a/apps/emqx_modules/src/emqx_mod_rewrite.erl b/apps/emqx_modules/src/emqx_rewrite.erl similarity index 72% rename from apps/emqx_modules/src/emqx_mod_rewrite.erl rename to apps/emqx_modules/src/emqx_rewrite.erl index c3a550692..20e51ae16 100644 --- a/apps/emqx_modules/src/emqx_mod_rewrite.erl +++ b/apps/emqx_modules/src/emqx_rewrite.erl @@ -14,9 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_mod_rewrite). - --behaviour(emqx_gen_mod). +-module(emqx_rewrite). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). @@ -33,21 +31,29 @@ , rewrite_publish/2 ]). -%% emqx_gen_mod callbacks --export([ load/1 - , unload/1 - , description/0 +-export([ enable/0 + , disable/0 ]). %%-------------------------------------------------------------------- %% Load/Unload %%-------------------------------------------------------------------- -load(RawRules) -> - {PubRules, SubRules} = compile(RawRules), - emqx_hooks:put('client.subscribe', {?MODULE, rewrite_subscribe, [SubRules]}), - emqx_hooks:put('client.unsubscribe', {?MODULE, rewrite_unsubscribe, [SubRules]}), - emqx_hooks:put('message.publish', {?MODULE, rewrite_publish, [PubRules]}). +enable() -> + Rules = emqx_config:get([rewrite, rules], []), + case Rules =:= [] of + true -> ok; + false -> + {PubRules, SubRules} = compile(Rules), + emqx_hooks:put('client.subscribe', {?MODULE, rewrite_subscribe, [SubRules]}), + emqx_hooks:put('client.unsubscribe', {?MODULE, rewrite_unsubscribe, [SubRules]}), + emqx_hooks:put('message.publish', {?MODULE, rewrite_publish, [PubRules]}) + end. + +disable() -> + emqx_hooks:del('client.subscribe', {?MODULE, rewrite_subscribe}), + emqx_hooks:del('client.unsubscribe', {?MODULE, rewrite_unsubscribe}), + emqx_hooks:del('message.publish', {?MODULE, rewrite_publish}). rewrite_subscribe(_ClientInfo, _Properties, TopicFilters, Rules) -> {ok, [{match_and_rewrite(Topic, Rules), Opts} || {Topic, Opts} <- TopicFilters]}. @@ -58,32 +64,28 @@ rewrite_unsubscribe(_ClientInfo, _Properties, TopicFilters, Rules) -> rewrite_publish(Message = #message{topic = Topic}, Rules) -> {ok, Message#message{topic = match_and_rewrite(Topic, Rules)}}. -unload(_) -> - emqx_hooks:del('client.subscribe', {?MODULE, rewrite_subscribe}), - emqx_hooks:del('client.unsubscribe', {?MODULE, rewrite_unsubscribe}), - emqx_hooks:del('message.publish', {?MODULE, rewrite_publish}). - -description() -> - "EMQ X Topic Rewrite Module". %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- compile(Rules) -> - PubRules = [ begin - {ok, MP} = re:compile(Re), - {rewrite, Topic, MP, Dest} - end || {rewrite, pub, Topic, Re, Dest}<- Rules ], - SubRules = [ begin - {ok, MP} = re:compile(Re), - {rewrite, Topic, MP, Dest} - end || {rewrite, sub, Topic, Re, Dest}<- Rules ], - {PubRules, SubRules}. + lists:foldl(fun(#{source_topic := Topic, + re := Re, + dest_topic := Dest, + action := Action}, {Acc1, Acc2}) -> + {ok, MP} = re:compile(Re), + case Action of + publish -> + {[{Topic, MP, Dest} | Acc1], Acc2}; + subscribe -> + {Acc1, [{Topic, MP, Dest} | Acc2]} + end + end, {[], []}, Rules). match_and_rewrite(Topic, []) -> Topic; -match_and_rewrite(Topic, [{rewrite, Filter, MP, Dest} | Rules]) -> +match_and_rewrite(Topic, [{Filter, MP, Dest} | Rules]) -> case emqx_topic:match(Topic, Filter) of true -> rewrite(Topic, MP, Dest); false -> match_and_rewrite(Topic, Rules) diff --git a/apps/emqx_telemetry/src/emqx_telemetry.erl b/apps/emqx_modules/src/emqx_telemetry.erl similarity index 77% rename from apps/emqx_telemetry/src/emqx_telemetry.erl rename to apps/emqx_modules/src/emqx_telemetry.erl index ea4017dc9..c9a78e736 100644 --- a/apps/emqx_telemetry/src/emqx_telemetry.erl +++ b/apps/emqx_modules/src/emqx_telemetry.erl @@ -24,7 +24,7 @@ -include_lib("kernel/include/file.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). --include("emqx_telemetry.hrl"). +-include("emqx_modules.hrl"). %% Mnesia bootstrap -export([mnesia/1]). @@ -32,7 +32,7 @@ -boot_mnesia({mnesia, [boot]}). -copy_mnesia({mnesia, [copy]}). --export([ start_link/1 +-export([ start_link/0 , stop/0 ]). @@ -48,11 +48,15 @@ -export([ enable/0 , disable/0 - , is_enabled/0 - , get_uuid/0 - , get_telemetry/0 ]). +-export([ get_uuid/0 + , get_telemetry/0 + , get_status/0 + ]). + +-export([official_version/1]). + -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). @@ -63,24 +67,16 @@ ]). -record(telemetry, { - id :: non_neg_integer(), - - uuid :: binary(), - - enabled :: boolean() - }). + id :: non_neg_integer(), + uuid :: binary() +}). -record(state, { - uuid :: undefined | binary(), - - enabled :: undefined | boolean(), - - url :: string(), - - report_interval :: undefined | non_neg_integer(), - - timer = undefined :: undefined | reference() - }). + uuid :: undefined | binary(), + url :: string(), + report_interval :: undefined | non_neg_integer(), + timer = undefined :: undefined | reference() +}). %% The count of 100-nanosecond intervals between the UUID epoch %% 1582-10-15 00:00:00 and the UNIX epoch 1970-01-01 00:00:00. @@ -90,6 +86,8 @@ -define(TELEMETRY, emqx_telemetry). +-rlog_shard({?COMMON_SHARD, ?TELEMETRY}). + %%-------------------------------------------------------------------- %% Mnesia bootstrap %%-------------------------------------------------------------------- @@ -108,7 +106,8 @@ mnesia(copy) -> %% API %%-------------------------------------------------------------------- -start_link(Opts) -> +start_link() -> + Opts = emqx_config:get([telemetry], #{}), gen_server:start_link({local, ?MODULE}, ?MODULE, [Opts], []). stop() -> @@ -120,8 +119,8 @@ enable() -> disable() -> gen_server:call(?MODULE, disable). -is_enabled() -> - gen_server:call(?MODULE, is_enabled). +get_status() -> + emqx_config:get([telemetry, enable], true). get_uuid() -> gen_server:call(?MODULE, get_uuid). @@ -139,43 +138,37 @@ get_telemetry() -> %% Given the chance of having two nodes bootstraping with the write %% is very small, it should be safe to ignore. -dialyzer([{nowarn_function, [init/1]}]). -init([Opts]) -> - State = #state{url = ?TELEMETRY_URL, - report_interval = timer:seconds(?REPORT_INTERVAR)}, - NState = case mnesia:dirty_read(?TELEMETRY, ?UNIQUE_ID) of - [] -> - Enabled = proplists:get_value(enabled, Opts, true), - UUID = generate_uuid(), - mnesia:dirty_write(?TELEMETRY, #telemetry{id = ?UNIQUE_ID, - uuid = UUID, - enabled = Enabled}), - State#state{enabled = Enabled, uuid = UUID}; - [#telemetry{uuid = UUID, enabled = Enabled} | _] -> - State#state{enabled = Enabled, uuid = UUID} - end, - case official_version(emqx_app:get_release()) of +init(_Opts) -> + UUID1 = case mnesia:dirty_read(?TELEMETRY, ?UNIQUE_ID) of + [] -> + UUID = generate_uuid(), + ekka_mnesia:dirty_write(?TELEMETRY, #telemetry{id = ?UNIQUE_ID, + uuid = UUID}), + UUID; + [#telemetry{uuid = UUID} | _] -> + UUID + end, + {ok, #state{url = ?TELEMETRY_URL, + report_interval = timer:seconds(?REPORT_INTERVAR), + uuid = UUID1}}. + +handle_call(enable, _From, State) -> + case ?MODULE:official_version(emqx_app:get_release()) of true -> - _ = erlang:send(self(), first_report), - {ok, NState}; + report_telemetry(State), + {reply, ok, ensure_report_timer(State)}; false -> - {ok, NState#state{enabled = false}} - end. + {reply, {error, not_official_version}, State} + end; -handle_call(enable, _From, State = #state{uuid = UUID}) -> - mnesia:dirty_write(?TELEMETRY, #telemetry{id = ?UNIQUE_ID, - uuid = UUID, - enabled = true}), - _ = erlang:send(self(), first_report), - {reply, ok, State#state{enabled = true}}; - -handle_call(disable, _From, State = #state{uuid = UUID}) -> - mnesia:dirty_write(?TELEMETRY, #telemetry{id = ?UNIQUE_ID, - uuid = UUID, - enabled = false}), - {reply, ok, State#state{enabled = false}}; - -handle_call(is_enabled, _From, State = #state{enabled = Enabled}) -> - {reply, Enabled, State}; +handle_call(disable, _From, State = #state{timer = Timer}) -> + case ?MODULE:official_version(emqx_app:get_release()) of + true -> + emqx_misc:cancel_timer(Timer), + {reply, ok, State#state{timer = undefined}}; + false -> + {reply, {error, not_official_version}, State} + end; handle_call(get_uuid, _From, State = #state{uuid = UUID}) -> {reply, {ok, UUID}, State}; @@ -195,20 +188,11 @@ handle_continue(Continue, State) -> ?LOG(error, "Unexpected continue: ~p", [Continue]), {noreply, State}. -handle_info(first_report, State) -> - case is_pid(erlang:whereis(emqx)) of - true -> - report_telemetry(State), - {noreply, ensure_report_timer(State)}; - false -> - _ = erlang:send_after(1000, self(), first_report), - {noreply, State} - end; -handle_info({timeout, TRef, time_to_report_telemetry_data}, State = #state{timer = TRef, - enabled = false}) -> - {noreply, State}; handle_info({timeout, TRef, time_to_report_telemetry_data}, State = #state{timer = TRef}) -> - report_telemetry(State), + case get_status() of + true -> report_telemetry(State); + false -> ok + end, {noreply, ensure_report_timer(State)}; handle_info(Info, State) -> @@ -305,12 +289,8 @@ active_plugins() -> end, [], emqx_plugins:list()). active_modules() -> - lists:foldl(fun({Name, Persistent}, Acc) -> - case Persistent of - true -> [Name | Acc]; - false -> Acc - end - end, [], emqx_modules:list()). + []. + % emqx_modules:list(). num_clients() -> emqx_stats:getstat('connections.max'). diff --git a/apps/emqx_modules/src/emqx_telemetry_api.erl b/apps/emqx_modules/src/emqx_telemetry_api.erl new file mode 100644 index 000000000..af5f40b02 --- /dev/null +++ b/apps/emqx_modules/src/emqx_telemetry_api.erl @@ -0,0 +1,236 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_telemetry_api). + +-behavior(minirest_api). + +-import(emqx_mgmt_util, [ response_schema/1 + , response_schema/2 + , request_body_schema/1 + ]). + +% -export([cli/1]). + +-export([ status/2 + , data/2 + ]). + +-export([enable_telemetry/2]). + +-export([api_spec/0]). + +api_spec() -> + {[status_api(), data_api()], schemas()}. + +schemas() -> + [#{broker_info => #{ + type => object, + properties => #{ + emqx_version => #{ + type => string, + description => <<"EMQ X Version">>}, + license => #{ + type => object, + properties => #{ + edition => #{type => string} + }, + description => <<"EMQ X License">>}, + os_name => #{ + type => string, + description => <<"OS Name">>}, + os_version => #{ + type => string, + description => <<"OS Version">>}, + otp_version => #{ + type => string, + description => <<"Erlang/OTP Version">>}, + up_time => #{ + type => integer, + description => <<"EMQ X Runtime">>}, + uuid => #{ + type => string, + description => <<"EMQ X UUID">>}, + nodes_uuid => #{ + type => array, + items => #{type => string}, + description => <<"EMQ X Cluster Nodes UUID">>}, + active_plugins => #{ + type => array, + items => #{type => string}, + description => <<"EMQ X Active Plugins">>}, + active_modules => #{ + type => array, + items => #{type => string}, + description => <<"EMQ X Active Modules">>}, + num_clients => #{ + type => integer, + description => <<"EMQ X Current Connections">>}, + messages_received => #{ + type => integer, + description => <<"EMQ X Current Received Message">>}, + messages_sent => #{ + type => integer, + description => <<"EMQ X Current Sent Message">>} + } + }}]. + +status_api() -> + Metadata = #{ + get => #{ + description => "Get telemetry status", + responses => #{ + <<"200">> => response_schema(<<"Bad Request">>, + #{ + type => object, + properties => #{enable => #{type => boolean}} + } + ) + } + }, + put => #{ + description => "Enable or disbale telemetry", + 'requestBody' => request_body_schema(#{ + type => object, + properties => #{ + enable => #{ + type => boolean + } + } + }), + responses => #{ + <<"200">> => + response_schema(<<"Enable or disbale telemetry successfully">>), + <<"400">> => + response_schema(<<"Bad Request">>, + #{ + type => object, + properties => #{ + message => #{type => string}, + code => #{type => string} + } + } + ) + } + } + }, + {"/telemetry/status", Metadata, status}. + +data_api() -> + Metadata = #{ + get => #{ + responses => #{ + <<"200">> => response_schema(<<"Get telemetry data">>, <<"broker_info">>) + } + } + }, + {"/telemetry/data", Metadata, data}. + +%%-------------------------------------------------------------------- +%% HTTP API +%%-------------------------------------------------------------------- +status(get, _Request) -> + {200, get_telemetry_status()}; + +status(put, Request) -> + {ok, Body, _} = cowboy_req:read_body(Request), + Params = emqx_json:decode(Body, [return_maps]), + Enable = maps:get(<<"enable">>, Params), + case Enable =:= emqx_telemetry:get_status() of + true -> + Reason = case Enable of + true -> <<"Telemetry status is already enabled">>; + false -> <<"Telemetry status is already disable">> + end, + {400, #{code => "BAD_REQUEST", message => Reason}}; + false -> + enable_telemetry(Enable), + {200} + end. + +data(get, _Request) -> + {200, emqx_json:encode(get_telemetry_data())}. +%%-------------------------------------------------------------------- +%% CLI +%%-------------------------------------------------------------------- +% cli(["enable", Enable0]) -> +% Enable = list_to_atom(Enable0), +% case Enable =:= emqx_telemetry:is_enabled() of +% true -> +% case Enable of +% true -> emqx_ctl:print("Telemetry status is already enabled~n"); +% false -> emqx_ctl:print("Telemetry status is already disable~n") +% end; +% false -> +% enable_telemetry(Enable), +% case Enable of +% true -> emqx_ctl:print("Enable telemetry successfully~n"); +% false -> emqx_ctl:print("Disable telemetry successfully~n") +% end +% end; + +% cli(["get", "status"]) -> +% case get_telemetry_status() of +% [{enabled, true}] -> +% emqx_ctl:print("Telemetry is enabled~n"); +% [{enabled, false}] -> +% emqx_ctl:print("Telemetry is disabled~n") +% end; + +% cli(["get", "data"]) -> +% TelemetryData = get_telemetry_data(), +% case emqx_json:safe_encode(TelemetryData, [pretty]) of +% {ok, Bin} -> +% emqx_ctl:print("~s~n", [Bin]); +% {error, _Reason} -> +% emqx_ctl:print("Failed to get telemetry data") +% end; + +% cli(_) -> +% emqx_ctl:usage([{"telemetry enable", "Enable telemetry"}, +% {"telemetry disable", "Disable telemetry"}, +% {"telemetry get data", "Get reported telemetry data"}]). + +%%-------------------------------------------------------------------- +%% internal function +%%-------------------------------------------------------------------- +enable_telemetry(Enable) -> + lists:foreach(fun(Node) -> + enable_telemetry(Node, Enable) + end, ekka_mnesia:running_nodes()). + +enable_telemetry(Node, Enable) when Node =:= node() -> + case Enable of + true -> + emqx_telemetry:enable(); + false -> + emqx_telemetry:disable() + end; +enable_telemetry(Node, Enable) -> + rpc_call(Node, ?MODULE, enable_telemetry, [Node, Enable]). + +get_telemetry_status() -> + #{enabled => emqx_telemetry:get_status()}. + +get_telemetry_data() -> + {ok, TelemetryData} = emqx_telemetry:get_telemetry(), + TelemetryData. + +rpc_call(Node, Module, Fun, Args) -> + case rpc:call(Node, Module, Fun, Args) of + {badrpc, Reason} -> {error, Reason}; + Result -> Result + end. diff --git a/apps/emqx_modules/src/emqx_mod_topic_metrics.erl b/apps/emqx_modules/src/emqx_topic_metrics.erl similarity index 94% rename from apps/emqx_modules/src/emqx_mod_topic_metrics.erl rename to apps/emqx_modules/src/emqx_topic_metrics.erl index 0196b9b2a..7297878fe 100644 --- a/apps/emqx_modules/src/emqx_mod_topic_metrics.erl +++ b/apps/emqx_modules/src/emqx_topic_metrics.erl @@ -14,10 +14,9 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_mod_topic_metrics). +-module(emqx_topic_metrics). -behaviour(gen_server). --behaviour(emqx_gen_mod). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -25,11 +24,6 @@ -logger_header("[TOPIC_METRICS]"). --export([ load/1 - , unload/1 - , description/0 - ]). - -export([ on_message_publish/1 , on_message_delivered/2 , on_message_dropped/3 @@ -40,11 +34,11 @@ , stop/0 ]). --export([ inc/2 - , inc/3 - , val/2 - , rate/2 - , metrics/1 +-export([ enable/0 + , disable/0 + ]). + +-export([ metrics/1 , register/1 , unregister/1 , unregister_all/0 @@ -52,9 +46,6 @@ , all_registered_topics/0 ]). -%% stats. --export([ rates/2 ]). - %% gen_server callbacks -export([ init/1 , handle_call/3 @@ -63,7 +54,10 @@ , terminate/2 ]). --define(CRefID(Topic), {?MODULE, Topic}). +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-endif. -define(MAX_TOPICS, 512). -define(TAB, ?MODULE). @@ -99,21 +93,15 @@ %%------------------------------------------------------------------------------ %% APIs %%------------------------------------------------------------------------------ - -load(_Env) -> - emqx_mod_sup:start_child(?MODULE, worker), +enable() -> emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}), emqx_hooks:put('message.dropped', {?MODULE, on_message_dropped, []}), emqx_hooks:put('message.delivered', {?MODULE, on_message_delivered, []}). -unload(_Env) -> +disable() -> emqx_hooks:del('message.publish', {?MODULE, on_message_publish}), emqx_hooks:del('message.dropped', {?MODULE, on_message_dropped}), - emqx_hooks:del('message.delivered', {?MODULE, on_message_delivered}), - emqx_mod_sup:stop_child(?MODULE). - -description() -> - "EMQ X Topic Metrics Module". + emqx_hooks:del('message.delivered', {?MODULE, on_message_delivered}). on_message_publish(#message{topic = Topic, qos = QoS}) -> case is_registered(Topic) of @@ -150,55 +138,12 @@ on_message_dropped(#message{topic = Topic}, _, _) -> end. start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + Opts = emqx_config:get([topic_metrics], #{}), + gen_server:start_link({local, ?MODULE}, ?MODULE, [Opts], []). stop() -> gen_server:stop(?MODULE). -try_inc(Topic, Metric) -> - _ = inc(Topic, Metric), - ok. - -inc(Topic, Metric) -> - inc(Topic, Metric, 1). - -inc(Topic, Metric, Val) -> - case get_counters(Topic) of - {error, topic_not_found} -> - {error, topic_not_found}; - CRef -> - case metric_idx(Metric) of - {error, invalid_metric} -> - {error, invalid_metric}; - Idx -> - counters:add(CRef, Idx, Val) - end - end. - -val(Topic, Metric) -> - case ets:lookup(?TAB, Topic) of - [] -> - {error, topic_not_found}; - [{Topic, CRef}] -> - case metric_idx(Metric) of - {error, invalid_metric} -> - {error, invalid_metric}; - Idx -> - counters:get(CRef, Idx) - end - end. - -rate(Topic, Metric) -> - case rates(Topic, Metric) of - #{short := Last} -> - Last; - {error, Reason} -> - {error, Reason} - end. - -rates(Topic, Metric) -> - gen_server:call(?MODULE, {get_rates, Topic, Metric}). - metrics(Topic) -> case ets:lookup(?TAB, Topic) of [] -> @@ -229,7 +174,7 @@ all_registered_topics() -> %% gen_server callbacks %%-------------------------------------------------------------------- -init([]) -> +init([_Opts]) -> erlang:process_flag(trap_exit, true), ok = emqx_tables:new(?TAB, [{read_concurrency, true}]), erlang:send_after(timer:seconds(?TICKING_INTERVAL), self(), ticking), @@ -279,7 +224,7 @@ handle_call({get_rates, Topic, Metric}, _From, State = #state{speeds = Speeds}) undefined -> {reply, {error, invalid_metric}, State}; #speed{last = Short, last_medium = Medium, last_long = Long} -> - {reply, #{ short => Short, medium => Medium, long => Long }, State} + {reply, #{short => Short, medium => Medium, long => Long }, State} end end. @@ -309,6 +254,47 @@ terminate(_Reason, _State) -> %% Internal Functions %%------------------------------------------------------------------------------ +try_inc(Topic, Metric) -> + _ = inc(Topic, Metric), + ok. + +inc(Topic, Metric) -> + inc(Topic, Metric, 1). + +inc(Topic, Metric, Val) -> + case get_counters(Topic) of + {error, topic_not_found} -> + {error, topic_not_found}; + CRef -> + case metric_idx(Metric) of + {error, invalid_metric} -> + {error, invalid_metric}; + Idx -> + counters:add(CRef, Idx, Val) + end + end. + +val(Topic, Metric) -> + case ets:lookup(?TAB, Topic) of + [] -> + {error, topic_not_found}; + [{Topic, CRef}] -> + case metric_idx(Metric) of + {error, invalid_metric} -> + {error, invalid_metric}; + Idx -> + counters:get(CRef, Idx) + end + end. + +rate(Topic, Metric) -> + case gen_server:call(?MODULE, {get_rates, Topic, Metric}) of + #{short := Last} -> + Last; + {error, Reason} -> + {error, Reason} + end. + metric_idx('messages.in') -> 01; metric_idx('messages.out') -> 02; metric_idx('messages.qos0.in') -> 03; diff --git a/apps/emqx_modules/src/emqx_topic_metrics_api.erl b/apps/emqx_modules/src/emqx_topic_metrics_api.erl new file mode 100644 index 000000000..1a2365703 --- /dev/null +++ b/apps/emqx_modules/src/emqx_topic_metrics_api.erl @@ -0,0 +1,207 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_topic_metrics_api). + +% -rest_api(#{name => list_all_topic_metrics, +% method => 'GET', +% path => "/topic-metrics", +% func => list, +% descr => "A list of all topic metrics of all nodes in the cluster"}). + +% -rest_api(#{name => list_topic_metrics, +% method => 'GET', +% path => "/topic-metrics/:bin:topic", +% func => list, +% descr => "A list of specfied topic metrics of all nodes in the cluster"}). + +% -rest_api(#{name => register_topic_metrics, +% method => 'POST', +% path => "/topic-metrics", +% func => register, +% descr => "Register topic metrics"}). + +% -rest_api(#{name => unregister_all_topic_metrics, +% method => 'DELETE', +% path => "/topic-metrics", +% func => unregister, +% descr => "Unregister all topic metrics"}). + +% -rest_api(#{name => unregister_topic_metrics, +% method => 'DELETE', +% path => "/topic-metrics/:bin:topic", +% func => unregister, +% descr => "Unregister topic metrics"}). + +% -export([ list/2 +% , register/2 +% , unregister/2 +% ]). + +% -export([ get_topic_metrics/2 +% , register_topic_metrics/2 +% , unregister_topic_metrics/2 +% , unregister_all_topic_metrics/1 +% ]). + +% list(#{topic := Topic0}, _Params) -> +% execute_when_enabled(fun() -> +% Topic = emqx_mgmt_util:urldecode(Topic0), +% case safe_validate(Topic) of +% true -> +% case get_topic_metrics(Topic) of +% {error, Reason} -> return({error, Reason}); +% Metrics -> return({ok, maps:from_list(Metrics)}) +% end; +% false -> +% return({error, invalid_topic_name}) +% end +% end); + +% list(_Bindings, _Params) -> +% execute_when_enabled(fun() -> +% case get_all_topic_metrics() of +% {error, Reason} -> return({error, Reason}); +% Metrics -> return({ok, Metrics}) +% end +% end). + +% register(_Bindings, Params) -> +% execute_when_enabled(fun() -> +% case proplists:get_value(<<"topic">>, Params) of +% undefined -> +% return({error, missing_required_params}); +% Topic -> +% case safe_validate(Topic) of +% true -> +% register_topic_metrics(Topic), +% return(ok); +% false -> +% return({error, invalid_topic_name}) +% end +% end +% end). + +% unregister(Bindings, _Params) when map_size(Bindings) =:= 0 -> +% execute_when_enabled(fun() -> +% unregister_all_topic_metrics(), +% return(ok) +% end); + +% unregister(#{topic := Topic0}, _Params) -> +% execute_when_enabled(fun() -> +% Topic = emqx_mgmt_util:urldecode(Topic0), +% case safe_validate(Topic) of +% true -> +% unregister_topic_metrics(Topic), +% return(ok); +% false -> +% return({error, invalid_topic_name}) +% end +% end). + +% execute_when_enabled(Fun) -> +% case emqx_modules:find_module(topic_metrics) of +% true -> +% Fun(); +% false -> +% return({error, module_not_loaded}) +% end. + +% safe_validate(Topic) -> +% try emqx_topic:validate(name, Topic) of +% true -> true +% catch +% error:_Error -> +% false +% end. + +% get_all_topic_metrics() -> +% lists:foldl(fun(Topic, Acc) -> +% case get_topic_metrics(Topic) of +% {error, _Reason} -> +% Acc; +% Metrics -> +% [#{topic => Topic, metrics => Metrics} | Acc] +% end +% end, [], emqx_mod_topic_metrics:all_registered_topics()). + +% get_topic_metrics(Topic) -> +% lists:foldl(fun(Node, Acc) -> +% case get_topic_metrics(Node, Topic) of +% {error, _Reason} -> +% Acc; +% Metrics -> +% case Acc of +% [] -> Metrics; +% _ -> +% lists:foldl(fun({K, V}, Acc0) -> +% [{K, V + proplists:get_value(K, Metrics, 0)} | Acc0] +% end, [], Acc) +% end +% end +% end, [], ekka_mnesia:running_nodes()). + +% get_topic_metrics(Node, Topic) when Node =:= node() -> +% emqx_mod_topic_metrics:metrics(Topic); +% get_topic_metrics(Node, Topic) -> +% rpc_call(Node, get_topic_metrics, [Node, Topic]). + +% register_topic_metrics(Topic) -> +% Results = [register_topic_metrics(Node, Topic) || Node <- ekka_mnesia:running_nodes()], +% case lists:any(fun(Item) -> Item =:= ok end, Results) of +% true -> ok; +% false -> lists:last(Results) +% end. + +% register_topic_metrics(Node, Topic) when Node =:= node() -> +% emqx_mod_topic_metrics:register(Topic); +% register_topic_metrics(Node, Topic) -> +% rpc_call(Node, register_topic_metrics, [Node, Topic]). + +% unregister_topic_metrics(Topic) -> +% Results = [unregister_topic_metrics(Node, Topic) || Node <- ekka_mnesia:running_nodes()], +% case lists:any(fun(Item) -> Item =:= ok end, Results) of +% true -> ok; +% false -> lists:last(Results) +% end. + +% unregister_topic_metrics(Node, Topic) when Node =:= node() -> +% emqx_mod_topic_metrics:unregister(Topic); +% unregister_topic_metrics(Node, Topic) -> +% rpc_call(Node, unregister_topic_metrics, [Node, Topic]). + +% unregister_all_topic_metrics() -> +% Results = [unregister_all_topic_metrics(Node) || Node <- ekka_mnesia:running_nodes()], +% case lists:any(fun(Item) -> Item =:= ok end, Results) of +% true -> ok; +% false -> lists:last(Results) +% end. + +% unregister_all_topic_metrics(Node) when Node =:= node() -> +% emqx_mod_topic_metrics:unregister_all(); +% unregister_all_topic_metrics(Node) -> +% rpc_call(Node, unregister_topic_metrics, [Node]). + +% rpc_call(Node, Fun, Args) -> +% case rpc:call(Node, ?MODULE, Fun, Args) of +% {badrpc, Reason} -> {error, Reason}; +% Res -> Res +% end. + +% return(_) -> +% %% TODO: V5 API +% ok. diff --git a/apps/emqx_modules/test/emqx_mod_delayed_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_SUITE.erl similarity index 72% rename from apps/emqx_modules/test/emqx_mod_delayed_SUITE.erl rename to apps/emqx_modules/test/emqx_delayed_SUITE.erl index fcd73bb61..a9af83b1d 100644 --- a/apps/emqx_modules/test/emqx_mod_delayed_SUITE.erl +++ b/apps/emqx_modules/test/emqx_delayed_SUITE.erl @@ -14,9 +14,9 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_mod_delayed_SUITE). +-module(emqx_delayed_SUITE). --import(emqx_mod_delayed, [on_message_publish/1]). +-import(emqx_delayed, [on_message_publish/1]). -compile(export_all). -compile(nowarn_export_all). @@ -35,44 +35,40 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_modules], fun set_special_configs/1), + ekka_mnesia:start(), + ok = emqx_delayed:mnesia(boot), + emqx_ct_helpers:start_apps([emqx_modules]), Config. end_per_suite(_) -> emqx_ct_helpers:stop_apps([emqx_modules]). -set_special_configs(emqx) -> - application:set_env(emqx, modules, [{emqx_mod_delayed, []}]), - application:set_env(emqx, allow_anonymous, false), - application:set_env(emqx, enable_acl_cache, false); -set_special_configs(_App) -> - ok. - %%-------------------------------------------------------------------- %% Test cases %%-------------------------------------------------------------------- t_load_case(_) -> - UnHooks = emqx_hooks:lookup('message.publish'), - ?assertEqual([], UnHooks), - ok = emqx_mod_delayed:load([]), Hooks = emqx_hooks:lookup('message.publish'), - ?assertEqual(1, length(Hooks)), + MFA = {emqx_delayed,on_message_publish,[]}, + ?assertEqual(false, lists:keyfind(MFA, 2, Hooks)), + ok = emqx_delayed:enable(), + Hooks1 = emqx_hooks:lookup('message.publish'), + ?assertNotEqual(false, lists:keyfind(MFA, 2, Hooks1)), ok. t_delayed_message(_) -> - ok = emqx_mod_delayed:load([]), + ok = emqx_delayed:enable(), DelayedMsg = emqx_message:make(?MODULE, 1, <<"$delayed/1/publish">>, <<"delayed_m">>), ?assertEqual({stop, DelayedMsg#message{topic = <<"publish">>, headers = #{allow_publish => false}}}, on_message_publish(DelayedMsg)), Msg = emqx_message:make(?MODULE, 1, <<"no_delayed_msg">>, <<"no_delayed">>), ?assertEqual({ok, Msg}, on_message_publish(Msg)), - [Key] = mnesia:dirty_all_keys(emqx_mod_delayed), - [#delayed_message{msg = #message{payload = Payload}}] = mnesia:dirty_read({emqx_mod_delayed, Key}), + [Key] = mnesia:dirty_all_keys(emqx_delayed), + [#delayed_message{msg = #message{payload = Payload}}] = mnesia:dirty_read({emqx_delayed, Key}), ?assertEqual(<<"delayed_m">>, Payload), timer:sleep(5000), - EmptyKey = mnesia:dirty_all_keys(emqx_mod_delayed), + EmptyKey = mnesia:dirty_all_keys(emqx_delayed), ?assertEqual([], EmptyKey), - ok = emqx_mod_delayed:unload([]). + ok = emqx_delayed:disable(). diff --git a/apps/emqx_modules/test/emqx_event_message_SUITE.erl b/apps/emqx_modules/test/emqx_event_message_SUITE.erl new file mode 100644 index 000000000..97235ac1f --- /dev/null +++ b/apps/emqx_modules/test/emqx_event_message_SUITE.erl @@ -0,0 +1,145 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_event_message_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-define(EVENT_MESSAGE, <<""" +event_message: { + topics : [ + \"$event/client_connected\", + \"$event/client_disconnected\", + \"$event/session_subscribed\", + \"$event/session_unsubscribed\", + \"$event/message_delivered\", + \"$event/message_acked\", + \"$event/message_dropped\" + ]}""">>). + + +all() -> emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:boot_modules(all), + emqx_ct_helpers:start_apps([emqx_modules]), + ok = emqx_config:init_load(emqx_modules_schema, ?EVENT_MESSAGE), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_modules]). + +t_event_topic(_) -> + ok = emqx_event_message:enable(), + {ok, C1} = emqtt:start_link([{clientid, <<"monsys">>}]), + {ok, _} = emqtt:connect(C1), + {ok, _, [?QOS_1]} = emqtt:subscribe(C1, <<"$event/client_connected">>, qos1), + {ok, C2} = emqtt:start_link([{clientid, <<"clientid">>}, + {username, <<"username">>}]), + {ok, _} = emqtt:connect(C2), + ok = recv_connected(<<"clientid">>), + + {ok, _, [?QOS_1]} = emqtt:subscribe(C1, <<"$event/session_subscribed">>, qos1), + _ = receive_publish(100), + timer:sleep(50), + {ok, _, [?QOS_1]} = emqtt:subscribe(C2, <<"test_sub">>, qos1), + ok = recv_subscribed(<<"clientid">>), + emqtt:unsubscribe(C1, <<"$event/session_subscribed">>), + timer:sleep(50), + + {ok, _, [?QOS_1]} = emqtt:subscribe(C1, <<"$event/message_delivered">>, qos1), + {ok, _, [?QOS_1]} = emqtt:subscribe(C1, <<"$event/message_acked">>, qos1), + _ = emqx:publish(emqx_message:make(<<"test">>, ?QOS_1, <<"test_sub">>, <<"test">>)), + {ok, #{qos := QOS1, topic := Topic1}} = receive_publish(100), + {ok, #{qos := QOS2, topic := Topic2}} = receive_publish(100), + recv_message_publish_or_delivered(<<"clientid">>, QOS1, Topic1), + recv_message_publish_or_delivered(<<"clientid">>, QOS2, Topic2), + recv_message_acked(<<"clientid">>), + + {ok, _, [?QOS_1]} = emqtt:subscribe(C1, <<"$event/message_dropped">>, qos1), + ok= emqtt:publish(C2, <<"test_sub1">>, <<"test">>), + recv_message_dropped(<<"clientid">>), + + {ok, _, [?QOS_1]} = emqtt:subscribe(C1, <<"$event/session_unsubscribed">>, qos1), + _ = emqtt:unsubscribe(C2, <<"test_sub">>), + ok = recv_unsubscribed(<<"clientid">>), + + {ok, _, [?QOS_1]} = emqtt:subscribe(C1, <<"$event/client_disconnected">>, qos1), + ok = emqtt:disconnect(C2), + ok = recv_disconnected(<<"clientid">>), + ok = emqtt:disconnect(C1), + ok = emqx_event_message:disable(). + +t_reason(_) -> + ?assertEqual(normal, emqx_event_message:reason(normal)), + ?assertEqual(discarded, emqx_event_message:reason({shutdown, discarded})), + ?assertEqual(tcp_error, emqx_event_message:reason({tcp_error, einval})), + ?assertEqual(internal_error, emqx_event_message:reason(<<"unknown error">>)). + +recv_connected(ClientId) -> + {ok, #{qos := ?QOS_0, topic := Topic, payload := Payload}} = receive_publish(100), + ?assertMatch(<<"$event/client_connected">>, Topic), + ?assertMatch(#{<<"clientid">> := ClientId, + <<"username">> := <<"username">>, + <<"ipaddress">> := <<"127.0.0.1">>, + <<"proto_name">> := <<"MQTT">>, + <<"proto_ver">> := ?MQTT_PROTO_V4, + <<"connack">> := ?RC_SUCCESS, + <<"clean_start">> := true}, emqx_json:decode(Payload, [return_maps])). + +recv_subscribed(_ClientId) -> + {ok, #{qos := ?QOS_0, topic := Topic}} = receive_publish(100), + ?assertMatch(<<"$event/session_subscribed">>, Topic). + +recv_message_dropped(_ClientId) -> + {ok, #{qos := ?QOS_0, topic := Topic}} = receive_publish(100), + ?assertMatch(<<"$event/message_dropped">>, Topic). + +recv_message_publish_or_delivered(_ClientId, 0, Topic) -> + ?assertMatch(<<"$event/message_delivered">>, Topic); +recv_message_publish_or_delivered(_ClientId, 1, Topic) -> + ?assertMatch(<<"test_sub">>, Topic). + +recv_message_acked(_ClientId) -> + {ok, #{qos := ?QOS_0, topic := Topic}} = receive_publish(100), + ?assertMatch(<<"$event/message_acked">>, Topic). + +recv_unsubscribed(_ClientId) -> + {ok, #{qos := ?QOS_0, topic := Topic}} = receive_publish(100), + ?assertMatch(<<"$event/session_unsubscribed">>, Topic). + +recv_disconnected(ClientId) -> + {ok, #{qos := ?QOS_0, topic := Topic, payload := Payload}} = receive_publish(100), + ?assertMatch(<<"$event/client_disconnected">>, Topic), + ?assertMatch(#{<<"clientid">> := ClientId, + <<"username">> := <<"username">>, + <<"reason">> := <<"normal">>}, emqx_json:decode(Payload, [return_maps])). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +receive_publish(Timeout) -> + receive + {publish, Publish} -> + {ok, Publish} + after + Timeout -> {error, timeout} + end. diff --git a/apps/emqx_modules/test/emqx_mod_presence_SUITE.erl b/apps/emqx_modules/test/emqx_mod_presence_SUITE.erl deleted file mode 100644 index fafcc3c2f..000000000 --- a/apps/emqx_modules/test/emqx_mod_presence_SUITE.erl +++ /dev/null @@ -1,88 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_mod_presence_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("eunit/include/eunit.hrl"). - -all() -> emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([emqx_modules]), - %% Ensure all the modules unloaded. - ok = emqx_modules:unload(), - Config. - -end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_modules]). - -%% Test case for emqx_mod_presence -t_mod_presence(_) -> - ok = emqx_mod_presence:load([{qos, ?QOS_1}]), - {ok, C1} = emqtt:start_link([{clientid, <<"monsys">>}]), - {ok, _} = emqtt:connect(C1), - {ok, _Props, [?QOS_1]} = emqtt:subscribe(C1, <<"$SYS/brokers/+/clients/#">>, qos1), - %% Connected Presence - {ok, C2} = emqtt:start_link([{clientid, <<"clientid">>}, - {username, <<"username">>}]), - {ok, _} = emqtt:connect(C2), - ok = recv_and_check_presence(<<"clientid">>, <<"connected">>), - %% Disconnected Presence - ok = emqtt:disconnect(C2), - ok = recv_and_check_presence(<<"clientid">>, <<"disconnected">>), - ok = emqtt:disconnect(C1), - ok = emqx_mod_presence:unload([{qos, ?QOS_1}]). - -t_mod_presence_reason(_) -> - ?assertEqual(normal, emqx_mod_presence:reason(normal)), - ?assertEqual(discarded, emqx_mod_presence:reason({shutdown, discarded})), - ?assertEqual(tcp_error, emqx_mod_presence:reason({tcp_error, einval})), - ?assertEqual(internal_error, emqx_mod_presence:reason(<<"unknown error">>)). - -recv_and_check_presence(ClientId, Presence) -> - {ok, #{qos := ?QOS_1, topic := Topic, payload := Payload}} = receive_publish(100), - ?assertMatch([<<"$SYS">>, <<"brokers">>, _Node, <<"clients">>, ClientId, Presence], - binary:split(Topic, <<"/">>, [global])), - case Presence of - <<"connected">> -> - ?assertMatch(#{<<"clientid">> := <<"clientid">>, - <<"username">> := <<"username">>, - <<"ipaddress">> := <<"127.0.0.1">>, - <<"proto_name">> := <<"MQTT">>, - <<"proto_ver">> := ?MQTT_PROTO_V4, - <<"connack">> := ?RC_SUCCESS, - <<"clean_start">> := true}, emqx_json:decode(Payload, [return_maps])); - <<"disconnected">> -> - ?assertMatch(#{<<"clientid">> := <<"clientid">>, - <<"username">> := <<"username">>, - <<"reason">> := <<"normal">>}, emqx_json:decode(Payload, [return_maps])) - end. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -receive_publish(Timeout) -> - receive - {publish, Publish} -> {ok, Publish} - after - Timeout -> {error, timeout} - end. diff --git a/apps/emqx_modules/test/emqx_mod_subscription_SUITE.erl b/apps/emqx_modules/test/emqx_mod_subscription_SUITE.erl deleted file mode 100644 index c2905754b..000000000 --- a/apps/emqx_modules/test/emqx_mod_subscription_SUITE.erl +++ /dev/null @@ -1,92 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_mod_subscription_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("eunit/include/eunit.hrl"). - -all() -> emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), - Config. - -end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). - -t_on_client_connected(_) -> - ?assertEqual(ok, emqx_mod_subscription:load([{<<"connected/%c/%u">>, #{qos => ?QOS_0}}])), - {ok, C} = emqtt:start_link([{host, "localhost"}, - {clientid, "myclient"}, - {username, "admin"}]), - {ok, _} = emqtt:connect(C), - emqtt:publish(C, <<"connected/myclient/admin">>, <<"Hello world">>, ?QOS_0), - {ok, #{topic := Topic, payload := Payload}} = receive_publish(100), - ?assertEqual(<<"connected/myclient/admin">>, Topic), - ?assertEqual(<<"Hello world">>, Payload), - ok = emqtt:disconnect(C), - ?assertEqual(ok, emqx_mod_subscription:unload([{<<"connected/%c/%u">>, #{qos => ?QOS_0}}])). - -t_on_undefined_client_connected(_) -> - ?assertEqual(ok, emqx_mod_subscription:load([{<<"connected/undefined">>, #{qos => ?QOS_1}}])), - {ok, C} = emqtt:start_link([{host, "localhost"}]), - {ok, _} = emqtt:connect(C), - emqtt:publish(C, <<"connected/undefined">>, <<"Hello world">>, ?QOS_1), - {ok, #{topic := Topic, payload := Payload}} = receive_publish(100), - ?assertEqual(<<"connected/undefined">>, Topic), - ?assertEqual(<<"Hello world">>, Payload), - ok = emqtt:disconnect(C), - ?assertEqual(ok, emqx_mod_subscription:unload([{<<"connected/undefined">>, #{qos => ?QOS_1}}])). - -t_suboption(_) -> - Client_info = fun(Key, Client) -> maps:get(Key, maps:from_list(emqtt:info(Client)), undefined) end, - Suboption = #{qos => ?QOS_2, nl => 1, rap => 1, rh => 2}, - ?assertEqual(ok, emqx_mod_subscription:load([{<<"connected/%c/%u">>, Suboption}])), - {ok, C1} = emqtt:start_link([{proto_ver, v5}]), - {ok, _} = emqtt:connect(C1), - timer:sleep(200), - [CPid1] = emqx_cm:lookup_channels(Client_info(clientid, C1)), - [ Sub1 | _ ] = ets:lookup(emqx_subscription,CPid1), - [ Suboption1 | _ ] = ets:lookup(emqx_suboption,Sub1), - ?assertMatch({Sub1, #{qos := 2, nl := 1, rap := 1, rh := 2, subid := _}}, Suboption1), - ok = emqtt:disconnect(C1), - %% The subscription option is not valid for MQTT V3.1.1 - {ok, C2} = emqtt:start_link([{proto_ver, v4}]), - {ok, _} = emqtt:connect(C2), - timer:sleep(200), - [CPid2] = emqx_cm:lookup_channels(Client_info(clientid, C2)), - [ Sub2 | _ ] = ets:lookup(emqx_subscription,CPid2), - [ Suboption2 | _ ] = ets:lookup(emqx_suboption,Sub2), - ok = emqtt:disconnect(C2), - ?assertMatch({Sub2, #{qos := 2, nl := 0, rap := 0, rh := 0, subid := _}}, Suboption2), - - ?assertEqual(ok, emqx_mod_subscription:unload([{<<"connected/undefined">>, Suboption}])). - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -receive_publish(Timeout) -> - receive - {publish, Publish} -> {ok, Publish} - after - Timeout -> {error, timeout} - end. diff --git a/apps/emqx_modules/test/emqx_mod_sup_SUITE.erl b/apps/emqx_modules/test/emqx_mod_sup_SUITE.erl deleted file mode 100644 index 59d0ffde2..000000000 --- a/apps/emqx_modules/test/emqx_mod_sup_SUITE.erl +++ /dev/null @@ -1,49 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_mod_sup_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). - -all() -> emqx_ct:all(?MODULE). - -%%-------------------------------------------------------------------- -%% Test cases -%%-------------------------------------------------------------------- - -t_start(_) -> - ?assertEqual([], supervisor:which_children(emqx_mod_sup)). - -t_start_child(_) -> - %% Set the emqx_mod_sup child with emqx_hooks for test - Mod = emqx_hooks, - Spec = #{id => Mod, - start => {Mod, start_link, []}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [Mod]}, - - ok = emqx_mod_sup:start_child(Mod, worker), - ?assertError({already_started, _}, emqx_mod_sup:start_child(Spec)), - - ok = emqx_mod_sup:stop_child(Mod), - {error, not_found} = emqx_mod_sup:stop_child(Mod), - ok. - diff --git a/apps/emqx_modules/test/emqx_mod_topic_metrics_SUITE.erl b/apps/emqx_modules/test/emqx_mod_topic_metrics_SUITE.erl deleted file mode 100644 index d41150e4a..000000000 --- a/apps/emqx_modules/test/emqx_mod_topic_metrics_SUITE.erl +++ /dev/null @@ -1,95 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_mod_topic_metrics_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). - -all() -> emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([emqx_modules]), - Config. - -end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_modules]). - -t_nonexistent_topic_metrics(_) -> - emqx_mod_topic_metrics:load([]), - ?assertEqual({error, topic_not_found}, emqx_mod_topic_metrics:val(<<"a/b/c">>, 'messages.in')), - ?assertEqual({error, topic_not_found}, emqx_mod_topic_metrics:inc(<<"a/b/c">>, 'messages.in')), - ?assertEqual({error, topic_not_found}, emqx_mod_topic_metrics:rate(<<"a/b/c">>, 'messages.in')), - ?assertEqual({error, topic_not_found}, emqx_mod_topic_metrics:rates(<<"a/b/c">>, 'messages.in')), - emqx_mod_topic_metrics:register(<<"a/b/c">>), - ?assertEqual(0, emqx_mod_topic_metrics:val(<<"a/b/c">>, 'messages.in')), - ?assertEqual({error, invalid_metric}, emqx_mod_topic_metrics:val(<<"a/b/c">>, 'invalid.metrics')), - ?assertEqual({error, invalid_metric}, emqx_mod_topic_metrics:inc(<<"a/b/c">>, 'invalid.metrics')), - ?assertEqual({error, invalid_metric}, emqx_mod_topic_metrics:rate(<<"a/b/c">>, 'invalid.metrics')), - ?assertEqual({error, invalid_metric}, emqx_mod_topic_metrics:rates(<<"a/b/c">>, 'invalid.metrics')), - emqx_mod_topic_metrics:unregister(<<"a/b/c">>), - emqx_mod_topic_metrics:unload([]). - -t_topic_metrics(_) -> - emqx_mod_topic_metrics:load([]), - - ?assertEqual(false, emqx_mod_topic_metrics:is_registered(<<"a/b/c">>)), - ?assertEqual([], emqx_mod_topic_metrics:all_registered_topics()), - emqx_mod_topic_metrics:register(<<"a/b/c">>), - ?assertEqual(true, emqx_mod_topic_metrics:is_registered(<<"a/b/c">>)), - ?assertEqual([<<"a/b/c">>], emqx_mod_topic_metrics:all_registered_topics()), - - ?assertEqual(0, emqx_mod_topic_metrics:val(<<"a/b/c">>, 'messages.in')), - ?assertEqual(ok, emqx_mod_topic_metrics:inc(<<"a/b/c">>, 'messages.in')), - ?assertEqual(1, emqx_mod_topic_metrics:val(<<"a/b/c">>, 'messages.in')), - ?assert(emqx_mod_topic_metrics:rate(<<"a/b/c">>, 'messages.in') =:= 0), - ?assert(emqx_mod_topic_metrics:rates(<<"a/b/c">>, 'messages.in') =:= #{long => 0,medium => 0,short => 0}), - emqx_mod_topic_metrics:unregister(<<"a/b/c">>), - emqx_mod_topic_metrics:unload([]). - -t_hook(_) -> - emqx_mod_topic_metrics:load([]), - emqx_mod_topic_metrics:register(<<"a/b/c">>), - - ?assertEqual(0, emqx_mod_topic_metrics:val(<<"a/b/c">>, 'messages.in')), - ?assertEqual(0, emqx_mod_topic_metrics:val(<<"a/b/c">>, 'messages.qos0.in')), - ?assertEqual(0, emqx_mod_topic_metrics:val(<<"a/b/c">>, 'messages.out')), - ?assertEqual(0, emqx_mod_topic_metrics:val(<<"a/b/c">>, 'messages.qos0.out')), - ?assertEqual(0, emqx_mod_topic_metrics:val(<<"a/b/c">>, 'messages.dropped')), - - {ok, C} = emqtt:start_link([{host, "localhost"}, - {clientid, "myclient"}, - {username, "myuser"}]), - {ok, _} = emqtt:connect(C), - emqtt:publish(C, <<"a/b/c">>, <<"Hello world">>, 0), - ct:sleep(100), - ?assertEqual(1, emqx_mod_topic_metrics:val(<<"a/b/c">>, 'messages.in')), - ?assertEqual(1, emqx_mod_topic_metrics:val(<<"a/b/c">>, 'messages.qos0.in')), - ?assertEqual(1, emqx_mod_topic_metrics:val(<<"a/b/c">>, 'messages.dropped')), - - emqtt:subscribe(C, <<"a/b/c">>), - emqtt:publish(C, <<"a/b/c">>, <<"Hello world">>, 0), - ct:sleep(100), - ?assertEqual(2, emqx_mod_topic_metrics:val(<<"a/b/c">>, 'messages.in')), - ?assertEqual(2, emqx_mod_topic_metrics:val(<<"a/b/c">>, 'messages.qos0.in')), - ?assertEqual(1, emqx_mod_topic_metrics:val(<<"a/b/c">>, 'messages.out')), - ?assertEqual(1, emqx_mod_topic_metrics:val(<<"a/b/c">>, 'messages.qos0.out')), - ?assertEqual(1, emqx_mod_topic_metrics:val(<<"a/b/c">>, 'messages.dropped')), - emqx_mod_topic_metrics:unregister(<<"a/b/c">>), - emqx_mod_topic_metrics:unload([]). diff --git a/apps/emqx_modules/test/emqx_modules_SUITE.erl b/apps/emqx_modules/test/emqx_modules_SUITE.erl deleted file mode 100644 index 0ce8b0c5f..000000000 --- a/apps/emqx_modules/test/emqx_modules_SUITE.erl +++ /dev/null @@ -1,202 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_modules_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). - --define(CONTENT_TYPE, "application/x-www-form-urlencoded"). - --define(HOST, "http://127.0.0.1:8081/"). - --define(API_VERSION, "v4"). - --define(BASE_PATH, "api"). - -all() -> emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_management, emqx_modules], fun set_special_cfg/1), - emqx_ct_http:create_default_app(), - Config. - -set_special_cfg(_) -> - application:set_env(emqx, modules_loaded_file, emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_modules")), - ok. - -end_per_suite(_Config) -> - emqx_ct_http:delete_default_app(), - emqx_ct_helpers:stop_apps([emqx_modules, emqx_management]). - -t_load(_) -> - ?assertEqual(ok, emqx_modules:unload()), - ?assertEqual(ok, emqx_modules:load()), - ?assertEqual({error, not_found}, emqx_modules:load(not_existed_module)), - ?assertEqual({error, not_started}, emqx_modules:unload(emqx_mod_rewrite)), - ?assertEqual(ignore, emqx_modules:reload(emqx_mod_rewrite)). - -t_list(_) -> - ?assertMatch([{_, _} | _ ], emqx_modules:list()). - -t_modules_api(_) -> - emqx_modules:load_module(emqx_mod_presence, false), - timer:sleep(50), - {ok, Modules1} = request_api(get, api_path(["modules"]), auth_header_()), - [Modules11] = filter(get(<<"data">>, Modules1), <<"node">>, atom_to_binary(node(), utf8)), - [Module1] = filter(maps:get(<<"modules">>, Modules11), <<"name">>, <<"emqx_mod_presence">>), - ?assertEqual(<<"emqx_mod_presence">>, maps:get(<<"name">>, Module1)), - ?assertEqual(true, maps:get(<<"active">>, Module1)), - - {ok, _} = request_api(put, - api_path(["modules", - atom_to_list(emqx_mod_presence), - "unload"]), - auth_header_()), - {ok, Error1} = request_api(put, - api_path(["modules", - atom_to_list(emqx_mod_presence), - "unload"]), - auth_header_()), - ?assertEqual(<<"not_started">>, get(<<"message">>, Error1)), - {ok, Modules2} = request_api(get, - api_path(["nodes", atom_to_list(node()), "modules"]), - auth_header_()), - [Module2] = filter(get(<<"data">>, Modules2), <<"name">>, <<"emqx_mod_presence">>), - ?assertEqual(<<"emqx_mod_presence">>, maps:get(<<"name">>, Module2)), - ?assertEqual(false, maps:get(<<"active">>, Module2)), - - {ok, _} = request_api(put, - api_path(["nodes", - atom_to_list(node()), - "modules", - atom_to_list(emqx_mod_presence), - "load"]), - auth_header_()), - {ok, Modules3} = request_api(get, - api_path(["nodes", atom_to_list(node()), "modules"]), - auth_header_()), - [Module3] = filter(get(<<"data">>, Modules3), <<"name">>, <<"emqx_mod_presence">>), - ?assertEqual(<<"emqx_mod_presence">>, maps:get(<<"name">>, Module3)), - ?assertEqual(true, maps:get(<<"active">>, Module3)), - - {ok, _} = request_api(put, - api_path(["nodes", - atom_to_list(node()), - "modules", - atom_to_list(emqx_mod_presence), - "unload"]), - auth_header_()), - {ok, Error2} = request_api(put, - api_path(["nodes", - atom_to_list(node()), - "modules", - atom_to_list(emqx_mod_presence), - "unload"]), - auth_header_()), - ?assertEqual(<<"not_started">>, get(<<"message">>, Error2)), - emqx_modules:unload(emqx_mod_presence). - - -t_modules_cmd(_) -> - mock_print(), - meck:new(emqx_modules, [non_strict, passthrough]), - meck:expect(emqx_modules, load, fun(_) -> ok end), - meck:expect(emqx_modules, unload, fun(_) -> ok end), - meck:expect(emqx_modules, reload, fun(_) -> ok end), - ?assertEqual(emqx_modules:cli(["list"]), ok), - ?assertEqual(emqx_modules:cli(["load", "emqx_mod_presence"]), - "Module emqx_mod_presence loaded successfully.\n"), - ?assertEqual(emqx_modules:cli(["unload", "emqx_mod_presence"]), - "Module emqx_mod_presence unloaded successfully.\n"), - unmock_print(). - -%% For: https://github.com/emqx/emqx/issues/4511 -t_join_cluster(_) -> - %% Started by emqx application - {error, {already_started, emqx_modules}} = application:start(emqx_modules), - %% After clustered - emqx:shutdown(), - emqx:reboot(), - {error,{already_started,emqx_modules}} = application:start(emqx_modules), - %% After emqx reboot, we should not interfere with other tests - _ = end_per_suite([]), - _ = init_per_suite([]), - ok. - -mock_print() -> - catch meck:unload(emqx_ctl), - meck:new(emqx_ctl, [non_strict, passthrough]), - meck:expect(emqx_ctl, print, fun(Arg) -> emqx_ctl:format(Arg) end), - meck:expect(emqx_ctl, print, fun(Msg, Arg) -> emqx_ctl:format(Msg, Arg) end), - meck:expect(emqx_ctl, usage, fun(Usages) -> emqx_ctl:format_usage(Usages) end), - meck:expect(emqx_ctl, usage, fun(Cmd, Descr) -> emqx_ctl:format_usage(Cmd, Descr) end). - -unmock_print() -> - meck:unload(emqx_ctl). - -get(Key, ResponseBody) -> - maps:get(Key, jiffy:decode(list_to_binary(ResponseBody), [return_maps])). - -request_api(Method, Url, Auth) -> - request_api(Method, Url, [], Auth, []). - -request_api(Method, Url, QueryParams, Auth) -> - request_api(Method, Url, QueryParams, Auth, []). - -request_api(Method, Url, QueryParams, Auth, []) -> - NewUrl = case QueryParams of - "" -> Url; - _ -> Url ++ "?" ++ QueryParams - end, - do_request_api(Method, {NewUrl, [Auth]}); -request_api(Method, Url, QueryParams, Auth, Body) -> - NewUrl = case QueryParams of - "" -> Url; - _ -> Url ++ "?" ++ QueryParams - end, - do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). - -do_request_api(Method, Request)-> - ct:pal("Method: ~p, Request: ~p", [Method, Request]), - case httpc:request(Method, Request, [], []) of - {error, socket_closed_remotely} -> - {error, socket_closed_remotely}; - {ok, {{"HTTP/1.1", Code, _}, _, Return} } - when Code =:= 200 orelse Code =:= 201 -> - {ok, Return}; - {ok, {Reason, _, _}} -> - {error, Reason} - end. - -auth_header_() -> - AppId = <<"admin">>, - AppSecret = <<"public">>, - auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)). - -auth_header_(User, Pass) -> - Encoded = base64:encode_to_string(lists:append([User,":",Pass])), - {"Authorization","Basic " ++ Encoded}. - -api_path(Parts)-> - ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts). - -filter(List, Key, Value) -> - lists:filter(fun(Item) -> - maps:get(Key, Item) == Value - end, List). diff --git a/apps/emqx_modules/test/emqx_mod_rewrite_SUITE.erl b/apps/emqx_modules/test/emqx_rewrite_SUITE.erl similarity index 68% rename from apps/emqx_modules/test/emqx_mod_rewrite_SUITE.erl rename to apps/emqx_modules/test/emqx_rewrite_SUITE.erl index 997eff1c2..467fa0e45 100644 --- a/apps/emqx_modules/test/emqx_mod_rewrite_SUITE.erl +++ b/apps/emqx_modules/test/emqx_rewrite_SUITE.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_mod_rewrite_SUITE). +-module(emqx_rewrite_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -22,17 +22,28 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). --define(RULES, [{rewrite, pub, <<"x/#">>,<<"^x/y/(.+)$">>,<<"z/y/$1">>}, - {rewrite, sub, <<"y/+/z/#">>,<<"^y/(.+)/z/(.+)$">>,<<"y/z/$2">>} - ]). +-define(REWRITE, <<""" +rewrite: { + rules : [ + { + action : publish + source_topic : \"x/#\" + re : \"^x/y/(.+)$\" + dest_topic : \"z/y/$1\" + }, + { + action : subscribe + source_topic : \"y/+/z/#\" + re : \"^y/(.+)/z/(.+)$\" + dest_topic : \"y/z/$2\" + } + ]}""">>). all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> emqx_ct_helpers:boot_modules(all), emqx_ct_helpers:start_apps([emqx_modules]), - %% Ensure all the modules unloaded. - ok = emqx_modules:unload(), Config. end_per_suite(_Config) -> @@ -40,7 +51,8 @@ end_per_suite(_Config) -> %% Test case for emqx_mod_write t_mod_rewrite(_Config) -> - ok = emqx_mod_rewrite:load(?RULES), + ok = emqx_config:init_load(emqx_modules_schema, ?REWRITE), + ok = emqx_rewrite:enable(), {ok, C} = emqtt:start_link([{clientid, <<"rewrite_client">>}]), {ok, _} = emqtt:connect(C), PubOrigTopics = [<<"x/y/2">>, <<"x/1/2">>], @@ -48,7 +60,7 @@ t_mod_rewrite(_Config) -> SubOrigTopics = [<<"y/a/z/b">>, <<"y/def">>], SubDestTopics = [<<"y/z/b">>, <<"y/def">>], %% Sub Rules - {ok, _Props, _} = emqtt:subscribe(C, [{Topic, ?QOS_1} || Topic <- SubOrigTopics]), + {ok, _Props1, _} = emqtt:subscribe(C, [{Topic, ?QOS_1} || Topic <- SubOrigTopics]), timer:sleep(100), Subscriptions = emqx_broker:subscriptions(<<"rewrite_client">>), ?assertEqual(SubDestTopics, [Topic || {Topic, _SubOpts} <- Subscriptions]), @@ -62,7 +74,7 @@ t_mod_rewrite(_Config) -> timer:sleep(100), ?assertEqual([], emqx_broker:subscriptions(<<"rewrite_client">>)), %% Pub Rules - {ok, _Props, _} = emqtt:subscribe(C, [{Topic, ?QOS_1} || Topic <- PubDestTopics]), + {ok, _Props2, _} = emqtt:subscribe(C, [{Topic, ?QOS_1} || Topic <- PubDestTopics]), RecvTopics2 = [begin ok = emqtt:publish(C, Topic, <<"payload">>), {ok, #{topic := RecvTopic}} = receive_publish(100), @@ -72,14 +84,19 @@ t_mod_rewrite(_Config) -> {ok, _, _} = emqtt:unsubscribe(C, PubDestTopics), ok = emqtt:disconnect(C), - ok = emqx_mod_rewrite:unload(?RULES). + ok = emqx_rewrite:disable(). t_rewrite_rule(_Config) -> - {PubRules, SubRules} = emqx_mod_rewrite:compile(?RULES), - ?assertEqual(<<"z/y/2">>, emqx_mod_rewrite:match_and_rewrite(<<"x/y/2">>, PubRules)), - ?assertEqual(<<"x/1/2">>, emqx_mod_rewrite:match_and_rewrite(<<"x/1/2">>, PubRules)), - ?assertEqual(<<"y/z/b">>, emqx_mod_rewrite:match_and_rewrite(<<"y/a/z/b">>, SubRules)), - ?assertEqual(<<"y/def">>, emqx_mod_rewrite:match_and_rewrite(<<"y/def">>, SubRules)). + {ok, Rewite} = hocon:binary(?REWRITE), + #{rewrite := #{rules := Rules}} = + hocon_schema:check_plain(emqx_modules_schema, Rewite, + #{atom_key => true}, + ["rewrite"]), + {PubRules, SubRules} = emqx_rewrite:compile(Rules), + ?assertEqual(<<"z/y/2">>, emqx_rewrite:match_and_rewrite(<<"x/y/2">>, PubRules)), + ?assertEqual(<<"x/1/2">>, emqx_rewrite:match_and_rewrite(<<"x/1/2">>, PubRules)), + ?assertEqual(<<"y/z/b">>, emqx_rewrite:match_and_rewrite(<<"y/a/z/b">>, SubRules)), + ?assertEqual(<<"y/def">>, emqx_rewrite:match_and_rewrite(<<"y/def">>, SubRules)). %%-------------------------------------------------------------------- %% Internal functions diff --git a/apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl similarity index 77% rename from apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl rename to apps/emqx_modules/test/emqx_telemetry_SUITE.erl index 39dcbf68c..62c524bdf 100644 --- a/apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl @@ -27,30 +27,22 @@ all() -> emqx_ct:all(?MODULE). -init_per_testcase(_, Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([emqx_telemetry], fun set_special_configs/1), +init_per_suite(Config) -> + ok = ekka_mnesia:start(), + ok = emqx_telemetry:mnesia(boot), + emqx_ct_helpers:start_apps([emqx_modules]), Config. -end_per_testcase(_, _Config) -> - emqx_ct_helpers:stop_apps([emqx_telemetry]). - -set_special_configs(emqx_telemetry) -> - application:set_env(emqx, plugins_etc_dir, - emqx_ct_helpers:deps_path(emqx_telemetry, "test")), - Conf = #{<<"emqx_telemetry">> => #{<<"enabled">> => true}}, - ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_telemetry.conf'), jsx:encode(Conf)), - ok; -set_special_configs(_App) -> - ok. +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_modules]). t_uuid(_) -> UUID = emqx_telemetry:generate_uuid(), Parts = binary:split(UUID, <<"-">>, [global, trim]), ?assertEqual(5, length(Parts)), {ok, UUID2} = emqx_telemetry:get_uuid(), - emqx_telemetry:stop(), - emqx_telemetry:start_link([{enabled, true}]), + emqx_telemetry:disable(), + emqx_telemetry:enable(), {ok, UUID3} = emqx_telemetry:get_uuid(), ?assertEqual(UUID2, UUID3). @@ -77,19 +69,23 @@ t_get_telemetry(_) -> ?assertEqual(0, get_value(num_clients, TelemetryData)). t_enable(_) -> + ok = meck:new(emqx_telemetry, [non_strict, passthrough, no_history, no_link]), + ok = meck:expect(emqx_telemetry, official_version, fun(_) -> true end), ok = emqx_telemetry:enable(), - ?assertEqual(true, emqx_telemetry:is_enabled()), ok = emqx_telemetry:disable(), - ?assertEqual(false, emqx_telemetry:is_enabled()). + meck:unload([emqx_telemetry]). t_send_after_enable(_) -> + ok = meck:new(emqx_telemetry, [non_strict, passthrough, no_history, no_link]), + ok = meck:expect(emqx_telemetry, official_version, fun(_) -> true end), ok = emqx_telemetry:disable(), ok = snabbkaffe:start_trace(), try ok = emqx_telemetry:enable(), ?assertMatch({ok, _}, ?block_until(#{?snk_kind := telemetry_data_reported}, 2000, 100)) after - ok = snabbkaffe:stop() + ok = snabbkaffe:stop(), + meck:unload([emqx_telemetry]) end. bin(L) when is_list(L) -> diff --git a/apps/emqx_modules/test/emqx_topic_metrics_SUITE.erl b/apps/emqx_modules/test/emqx_topic_metrics_SUITE.erl new file mode 100644 index 000000000..5d1c5f84a --- /dev/null +++ b/apps/emqx_modules/test/emqx_topic_metrics_SUITE.erl @@ -0,0 +1,95 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_topic_metrics_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:boot_modules(all), + emqx_ct_helpers:start_apps([emqx_modules]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_modules]). + +t_nonexistent_topic_metrics(_) -> + emqx_topic_metrics:enable(), + ?assertEqual({error, topic_not_found}, emqx_topic_metrics:val(<<"a/b/c">>, 'messages.in')), + ?assertEqual({error, topic_not_found}, emqx_topic_metrics:inc(<<"a/b/c">>, 'messages.in')), + ?assertEqual({error, topic_not_found}, emqx_topic_metrics:rate(<<"a/b/c">>, 'messages.in')), + % ?assertEqual({error, topic_not_found}, emqx_topic_metrics:rates(<<"a/b/c">>, 'messages.in')), + emqx_topic_metrics:register(<<"a/b/c">>), + ?assertEqual(0, emqx_topic_metrics:val(<<"a/b/c">>, 'messages.in')), + ?assertEqual({error, invalid_metric}, emqx_topic_metrics:val(<<"a/b/c">>, 'invalid.metrics')), + ?assertEqual({error, invalid_metric}, emqx_topic_metrics:inc(<<"a/b/c">>, 'invalid.metrics')), + ?assertEqual({error, invalid_metric}, emqx_topic_metrics:rate(<<"a/b/c">>, 'invalid.metrics')), + % ?assertEqual({error, invalid_metric}, emqx_topic_metrics:rates(<<"a/b/c">>, 'invalid.metrics')), + emqx_topic_metrics:unregister(<<"a/b/c">>), + emqx_topic_metrics:disable(). + +t_topic_metrics(_) -> + emqx_topic_metrics:enable(), + + ?assertEqual(false, emqx_topic_metrics:is_registered(<<"a/b/c">>)), + ?assertEqual([], emqx_topic_metrics:all_registered_topics()), + emqx_topic_metrics:register(<<"a/b/c">>), + ?assertEqual(true, emqx_topic_metrics:is_registered(<<"a/b/c">>)), + ?assertEqual([<<"a/b/c">>], emqx_topic_metrics:all_registered_topics()), + + ?assertEqual(0, emqx_topic_metrics:val(<<"a/b/c">>, 'messages.in')), + ?assertEqual(ok, emqx_topic_metrics:inc(<<"a/b/c">>, 'messages.in')), + ?assertEqual(1, emqx_topic_metrics:val(<<"a/b/c">>, 'messages.in')), + ?assert(emqx_topic_metrics:rate(<<"a/b/c">>, 'messages.in') =:= 0), + % ?assert(emqx_topic_metrics:rates(<<"a/b/c">>, 'messages.in') =:= #{long => 0,medium => 0,short => 0}), + emqx_topic_metrics:unregister(<<"a/b/c">>), + emqx_topic_metrics:disable(). + +t_hook(_) -> + emqx_topic_metrics:enable(), + emqx_topic_metrics:register(<<"a/b/c">>), + + ?assertEqual(0, emqx_topic_metrics:val(<<"a/b/c">>, 'messages.in')), + ?assertEqual(0, emqx_topic_metrics:val(<<"a/b/c">>, 'messages.qos0.in')), + ?assertEqual(0, emqx_topic_metrics:val(<<"a/b/c">>, 'messages.out')), + ?assertEqual(0, emqx_topic_metrics:val(<<"a/b/c">>, 'messages.qos0.out')), + ?assertEqual(0, emqx_topic_metrics:val(<<"a/b/c">>, 'messages.dropped')), + + {ok, C} = emqtt:start_link([{host, "localhost"}, + {clientid, "myclient"}, + {username, "myuser"}]), + {ok, _} = emqtt:connect(C), + emqtt:publish(C, <<"a/b/c">>, <<"Hello world">>, 0), + ct:sleep(100), + ?assertEqual(1, emqx_topic_metrics:val(<<"a/b/c">>, 'messages.in')), + ?assertEqual(1, emqx_topic_metrics:val(<<"a/b/c">>, 'messages.qos0.in')), + ?assertEqual(1, emqx_topic_metrics:val(<<"a/b/c">>, 'messages.dropped')), + + emqtt:subscribe(C, <<"a/b/c">>), + emqtt:publish(C, <<"a/b/c">>, <<"Hello world">>, 0), + ct:sleep(100), + ?assertEqual(2, emqx_topic_metrics:val(<<"a/b/c">>, 'messages.in')), + ?assertEqual(2, emqx_topic_metrics:val(<<"a/b/c">>, 'messages.qos0.in')), + ?assertEqual(1, emqx_topic_metrics:val(<<"a/b/c">>, 'messages.out')), + ?assertEqual(1, emqx_topic_metrics:val(<<"a/b/c">>, 'messages.qos0.out')), + ?assertEqual(1, emqx_topic_metrics:val(<<"a/b/c">>, 'messages.dropped')), + emqx_topic_metrics:unregister(<<"a/b/c">>), + emqx_topic_metrics:disable(). diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl index 71119264d..03c5bdc8f 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl @@ -54,5 +54,5 @@ health_check(PoolName, CheckFunc, State) when is_function(CheckFunc) -> end || {_WorkerName, Worker} <- ecpool:workers(PoolName)], case length(Status) > 0 andalso lists:all(fun(St) -> St =:= true end, Status) of true -> {ok, State}; - false -> {error, test_query_failed, State} + false -> {error, health_check_failed, State} end. diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl index 9a17765c6..3fac97e86 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl @@ -45,7 +45,7 @@ -spec save_files_return_opts(opts_input(), atom() | string() | binary(), string() | binary()) -> opts(). save_files_return_opts(Options, SubDir, ResId) -> - Dir = filename:join([emqx:get_env(data_dir), SubDir, ResId]), + Dir = filename:join([emqx_config:get([node, data_dir]), SubDir, ResId]), save_files_return_opts(Options, Dir). %% @doc Parse ssl options input. @@ -57,7 +57,7 @@ save_files_return_opts(Options, Dir) -> Get = fun(Key) -> GetD(Key, undefined) end, KeyFile = Get(keyfile), CertFile = Get(certfile), - CAFile = GetD(cacertfile, Get(cafile)), + CAFile = Get(cacertfile), Key = do_save_file(KeyFile, Dir), Cert = do_save_file(CertFile, Dir), CA = do_save_file(CAFile, Dir), @@ -76,7 +76,7 @@ save_files_return_opts(Options, Dir) -> %% empty string is returned if the input is empty. -spec save_file(file_input(), atom() | string() | binary()) -> string(). save_file(Param, SubDir) -> - Dir = filename:join([emqx:get_env(data_dir), SubDir]), + Dir = filename:join([emqx_config:get([node, data_dir]), SubDir]), do_save_file(Param, Dir). filter([]) -> []; diff --git a/apps/emqx_prometheus/etc/emqx_prometheus.conf b/apps/emqx_prometheus/etc/emqx_prometheus.conf index c450846fe..38ce5e501 100644 --- a/apps/emqx_prometheus/etc/emqx_prometheus.conf +++ b/apps/emqx_prometheus/etc/emqx_prometheus.conf @@ -1,7 +1,8 @@ ##-------------------------------------------------------------------- ## emqx_prometheus for EMQ X ##-------------------------------------------------------------------- -emqx_prometheus:{ +prometheus: { push_gateway_server: "http://127.0.0.1:9091" interval: "15s" + enable: true } diff --git a/apps/emqx_prometheus/include/emqx_prometheus.hrl b/apps/emqx_prometheus/include/emqx_prometheus.hrl index e69de29bb..589bbd024 100644 --- a/apps/emqx_prometheus/include/emqx_prometheus.hrl +++ b/apps/emqx_prometheus/include/emqx_prometheus.hrl @@ -0,0 +1 @@ +-define(APP, emqx_prometheus). diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index c11c8f0d7..e369179ee 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -25,14 +25,6 @@ -include_lib("prometheus/include/prometheus.hrl"). -include_lib("prometheus/include/prometheus_model.hrl"). --import(minirest, [return/1]). - --rest_api(#{name => stats, - method => 'GET', - path => "/emqx_prometheus", - func => stats, - descr => "Get emqx all stats info" - }). -import(prometheus_model_helpers, [ create_mf/5 @@ -40,11 +32,8 @@ , counter_metric/1 ]). -%% REST APIs --export([stats/2]). - %% APIs --export([start_link/2]). +-export([start_link/1]). %% gen_server callbacks -export([ init/1 @@ -61,6 +50,8 @@ , collect_metrics/2 ]). +-export([collect/1]). + -define(C(K, L), proplists:get_value(K, L, 0)). -define(TIMER_MSG, '#interval'). @@ -71,25 +62,17 @@ %% APIs %%-------------------------------------------------------------------- -start_link(PushGateway, Interval) -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [PushGateway, Interval], []). - -%%-------------------------------------------------------------------- -%% REST APIs - -stats(_Bindings, Params) -> - collect(proplists:get_value(<<"type">>, Params, <<"json">>)). +start_link(Opts) -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [Opts], []). %%-------------------------------------------------------------------- %% gen_server callbacks %%-------------------------------------------------------------------- -init([undefined, Interval]) -> - {ok, #state{interval = Interval}}; - -init([PushGateway, Interval]) -> - Ref = erlang:start_timer(Interval, self(), ?TIMER_MSG), - {ok, #state{timer = Ref, push_gateway = PushGateway, interval = Interval}}. +init([Opts]) -> + Interval = maps:get(interval, Opts), + PushGateway = maps:get(push_gateway_server, Opts), + {ok, ensure_timer(#state{push_gateway = PushGateway, interval = Interval})}. handle_call(_Msg, _From, State) -> {noreply, State}. @@ -97,12 +80,12 @@ handle_call(_Msg, _From, State) -> handle_cast(_Msg, State) -> {noreply, State}. -handle_info({timeout, R, ?TIMER_MSG}, S = #state{interval=I, timer=R, push_gateway=Uri}) -> +handle_info({timeout, R, ?TIMER_MSG}, State = #state{timer=R, push_gateway=Uri}) -> [Name, Ip] = string:tokens(atom_to_list(node()), "@"), Url = lists:concat([Uri, "/metrics/job/", Name, "/instance/",Name, "~", Ip]), Data = prometheus_text_format:format(), httpc:request(post, {Url, [], "text/plain", Data}, [{autoredirect, true}], []), - {noreply, S#state{timer = erlang:start_timer(I, self(), ?TIMER_MSG)}}; + {noreply, ensure_timer(State)}; handle_info(_Msg, State) -> {noreply, State}. @@ -113,6 +96,8 @@ code_change(_OldVsn, State, _Extra) -> terminate(_Reason, _State) -> ok. +ensure_timer(State = #state{interval = Interval}) -> + State#state{timer = emqx_misc:start_timer(Interval, ?TIMER_MSG)}. %%-------------------------------------------------------------------- %% prometheus callbacks %%-------------------------------------------------------------------- @@ -140,18 +125,16 @@ collect(<<"json">>) -> Metrics = emqx_metrics:all(), Stats = emqx_stats:getstats(), VMData = emqx_vm_data(), - Data = [{stats, [collect_stats(Name, Stats) || Name <- emqx_stats()]}, - {metrics, [collect_stats(Name, VMData) || Name <- emqx_vm()]}, - {packets, [collect_stats(Name, Metrics) || Name <- emqx_metrics_packets()]}, - {messages, [collect_stats(Name, Metrics) || Name <- emqx_metrics_messages()]}, - {delivery, [collect_stats(Name, Metrics) || Name <- emqx_metrics_delivery()]}, - {client, [collect_stats(Name, Metrics) || Name <- emqx_metrics_client()]}, - {session, [collect_stats(Name, Metrics) || Name <- emqx_metrics_session()]}], - return({ok, Data}); + [{stats, [collect_stats(Name, Stats) || Name <- emqx_stats()]}, + {metrics, [collect_stats(Name, VMData) || Name <- emqx_vm()]}, + {packets, [collect_stats(Name, Metrics) || Name <- emqx_metrics_packets()]}, + {messages, [collect_stats(Name, Metrics) || Name <- emqx_metrics_messages()]}, + {delivery, [collect_stats(Name, Metrics) || Name <- emqx_metrics_delivery()]}, + {client, [collect_stats(Name, Metrics) || Name <- emqx_metrics_client()]}, + {session, [collect_stats(Name, Metrics) || Name <- emqx_metrics_session()]}]; collect(<<"prometheus">>) -> - Data = prometheus_text_format:format(), - {ok, #{<<"content-type">> => <<"text/plain">>}, Data}. + prometheus_text_format:format(). %% @private collect_stats(Name, Stats) -> @@ -414,8 +397,8 @@ emqx_collect(emqx_client_authenticate, Stats) -> counter_metric(?C('client.authenticate', Stats)); emqx_collect(emqx_client_auth_anonymous, Stats) -> counter_metric(?C('client.auth.anonymous', Stats)); -emqx_collect(emqx_client_check_acl, Stats) -> - counter_metric(?C('client.check_acl', Stats)); +emqx_collect(emqx_client_authorize, Stats) -> + counter_metric(?C('client.authorize', Stats)); emqx_collect(emqx_client_subscribe, Stats) -> counter_metric(?C('client.subscribe', Stats)); emqx_collect(emqx_client_unsubscribe, Stats) -> @@ -567,7 +550,7 @@ emqx_metrics_client() -> [ emqx_client_connected , emqx_client_authenticate , emqx_client_auth_anonymous - , emqx_client_check_acl + , emqx_client_authorize , emqx_client_subscribe , emqx_client_unsubscribe , emqx_client_disconnected diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl new file mode 100644 index 000000000..3b5d686d3 --- /dev/null +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -0,0 +1,138 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_prometheus_api). + +-behaviour(minirest_api). + +-include("emqx_prometheus.hrl"). + +-import(emqx_mgmt_util, [ response_schema/1 + , response_schema/2 + , request_body_schema/1 + ]). + +-export([api_spec/0]). + +-export([ prometheus/2 + % , stats/2 + ]). + +api_spec() -> + {[prometheus_api()], schemas()}. + +schemas() -> + [#{prometheus => #{ + type => object, + properties => #{ + push_gateway_server => #{ + type => string, + description => <<"prometheus PushGateway Server">>, + example => get_raw(<<"push_gateway_server">>, <<"http://127.0.0.1:9091">>)}, + interval => #{ + type => string, + description => <<"Interval">>, + example => get_raw(<<"interval">>, <<"15s">>)}, + enable => #{ + type => boolean, + description => <<"Prometheus status">>, + example => get_raw(<<"enable">>, false)} + } + }}]. + +prometheus_api() -> + Metadata = #{ + get => #{ + description => <<"Get Prometheus info">>, + responses => #{ + <<"200">> => response_schema(prometheus) + } + }, + put => #{ + description => <<"Update Prometheus">>, + 'requestBody' => request_body_schema(prometheus), + responses => #{ + <<"200">> => + response_schema(<<"Update Prometheus successfully">>), + <<"400">> => + response_schema(<<"Bad Request">>, #{ + type => object, + properties => #{ + message => #{type => string}, + code => #{type => string} + } + }) + } + } + }, + {"/prometheus", Metadata, prometheus}. + +% prometheus_data_api() -> +% Metadata = #{ +% get => #{ +% description => <<"Get Prometheus Data">>, +% parameters => [#{ +% name => format_type, +% in => path, +% schema => #{type => string} +% }], +% responses => #{ +% <<"200">> => +% response_schema(<<"Update Prometheus successfully">>), +% <<"400">> => +% response_schema(<<"Bad Request">>, #{ +% type => object, +% properties => #{ +% message => #{type => string}, +% code => #{type => string} +% } +% }) +% } +% } +% }, +% {"/prometheus/stats", Metadata, stats}. + +prometheus(get, _Request) -> + Response = emqx_config:get_raw([<<"prometheus">>], #{}), + {200, Response}; + +prometheus(put, Request) -> + {ok, Body, _} = cowboy_req:read_body(Request), + Params = emqx_json:decode(Body, [return_maps]), + Enable = maps:get(<<"enable">>, Params), + ok = emqx_config:update([prometheus], Params), + enable_prometheus(Enable). + +% stats(_Bindings, Params) -> +% Type = proplists:get_value(<<"format_type">>, Params, <<"json">>), +% Data = emqx_prometheus:collect(Type), +% case Type of +% <<"json">> -> +% {ok, Data}; +% <<"prometheus">> -> +% {ok, #{<<"content-type">> => <<"text/plain">>}, Data} +% end. + +enable_prometheus(true) -> + ok = emqx_prometheus_sup:stop_child(?APP), + emqx_prometheus_sup:start_child(?APP, emqx_config:get([prometheus], #{})), + {200}; +enable_prometheus(false) -> + _ = emqx_prometheus_sup:stop_child(?APP), + {200}. + +get_raw(Key, Def) -> + emqx_config:get_raw([<<"prometheus">>] ++ [Key], Def). diff --git a/apps/emqx_prometheus/src/emqx_prometheus_app.erl b/apps/emqx_prometheus/src/emqx_prometheus_app.erl index 9024bd583..f4d4fd164 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_app.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_app.erl @@ -18,20 +18,25 @@ -behaviour(application). --emqx_plugin(?MODULE). +-include("emqx_prometheus.hrl"). %% Application callbacks -export([ start/2 , stop/1 ]). --define(APP, emqx_prometheus). - start(_StartType, _StartArgs) -> - PushGateway = emqx_config:get([?APP, push_gateway_server], undefined), - Interval = emqx_config:get([?APP, interval], 15000), - emqx_prometheus_sup:start_link(PushGateway, Interval). + {ok, Sup} = emqx_prometheus_sup:start_link(), + maybe_enable_prometheus(), + {ok, Sup}. stop(_State) -> ok. +maybe_enable_prometheus() -> + case emqx_config:get([prometheus, enable], false) of + true -> + emqx_prometheus_sup:start_child(?APP, emqx_config:get([prometheus], #{})); + false -> + ok + end. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 486362e7b..fa41154d3 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -22,9 +22,10 @@ -export([ structs/0 , fields/1]). -structs() -> ["emqx_prometheus"]. +structs() -> ["prometheus"]. -fields("emqx_prometheus") -> +fields("prometheus") -> [ {push_gateway_server, emqx_schema:t(string())} , {interval, emqx_schema:t(emqx_schema:duration_ms(), undefined, "15s")} + , {enable, emqx_schema:t(boolean(), undefined, false)} ]. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_sup.erl b/apps/emqx_prometheus/src/emqx_prometheus_sup.erl index 8ebbb02ae..fd5223b6d 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_sup.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_sup.erl @@ -18,19 +18,48 @@ -behaviour(supervisor). --export([start_link/2]). +-export([ start_link/0 + , start_child/1 + , start_child/2 + , stop_child/1 + ]). -export([init/1]). -start_link(PushGateway, Interval) -> - supervisor:start_link({local, ?MODULE}, ?MODULE, [PushGateway, Interval]). +%% Helper macro for declaring children of supervisor +-define(CHILD(Mod, Opts), #{id => Mod, + start => {Mod, start_link, [Opts]}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [Mod]}). -init([PushGateway, Interval]) -> - {ok, {#{strategy => one_for_one, intensity => 10, period => 100}, - [#{id => emqx_prometheus, - start => {emqx_prometheus, start_link, [PushGateway, Interval]}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_prometheus]}]}}. +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). +-spec start_child(supervisor:child_spec()) -> ok. +start_child(ChildSpec) when is_map(ChildSpec) -> + assert_started(supervisor:start_child(?MODULE, ChildSpec)). + +-spec start_child(atom(), map()) -> ok. +start_child(Mod, Opts) when is_atom(Mod) andalso is_map(Opts) -> + assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Opts))). + +-spec(stop_child(any()) -> ok | {error, term()}). +stop_child(ChildId) -> + case supervisor:terminate_child(?MODULE, ChildId) of + ok -> supervisor:delete_child(?MODULE, ChildId); + Error -> Error + end. + +init([]) -> + {ok, {{one_for_one, 10, 3600}, []}}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +assert_started({ok, _Pid}) -> ok; +assert_started({ok, _Pid, _Info}) -> ok; +assert_started({error, {already_tarted, _Pid}}) -> ok; +assert_started({error, Reason}) -> erlang:error(Reason). diff --git a/apps/emqx_psk_file/.gitignore b/apps/emqx_psk_file/.gitignore deleted file mode 100644 index 0379a99df..000000000 --- a/apps/emqx_psk_file/.gitignore +++ /dev/null @@ -1,16 +0,0 @@ -.eunit -deps -*.o -*.beam -*.plt -erl_crash.dump -ebin -rel/example_project -.concrete/DEV_MODE -.rebar -.erlang.mk/ -data/ -emqx_actorcloud_schema_parser.d -.DS_Store -_build -rebar.lock diff --git a/apps/emqx_psk_file/README.md b/apps/emqx_psk_file/README.md deleted file mode 100644 index 3ba976b81..000000000 --- a/apps/emqx_psk_file/README.md +++ /dev/null @@ -1,2 +0,0 @@ -## EMQX TLS/DTLS PSK Plugin from file - diff --git a/apps/emqx_psk_file/etc/emqx_psk_file.conf b/apps/emqx_psk_file/etc/emqx_psk_file.conf deleted file mode 100644 index 88c5bbdb1..000000000 --- a/apps/emqx_psk_file/etc/emqx_psk_file.conf +++ /dev/null @@ -1,2 +0,0 @@ -psk.file.path = "{{ platform_etc_dir }}/psk.txt" -psk.file.delimiter = ":" diff --git a/apps/emqx_psk_file/etc/psk.txt b/apps/emqx_psk_file/etc/psk.txt deleted file mode 100644 index 3cf33d814..000000000 --- a/apps/emqx_psk_file/etc/psk.txt +++ /dev/null @@ -1,2 +0,0 @@ -client1:1234 -client2:abcd diff --git a/apps/emqx_psk_file/priv/emqx_psk_file.schema b/apps/emqx_psk_file/priv/emqx_psk_file.schema deleted file mode 100644 index 0c784d99b..000000000 --- a/apps/emqx_psk_file/priv/emqx_psk_file.schema +++ /dev/null @@ -1,10 +0,0 @@ -%%-*- mode: erlang -*- -%% emqx_psk_file config mapping - -{mapping, "psk.file.path", "emqx_psk_file.path", [ - {datatype, string} -]}. - -{mapping, "psk.file.delimiter", "emqx_psk_file.delimiter", [ - {datatype, string} -]}. \ No newline at end of file diff --git a/apps/emqx_psk_file/rebar.config b/apps/emqx_psk_file/rebar.config deleted file mode 100644 index 7ac3b98c8..000000000 --- a/apps/emqx_psk_file/rebar.config +++ /dev/null @@ -1,16 +0,0 @@ -{deps, []}. - -{edoc_opts, [{preprocess, true}]}. -{erl_opts, [warn_unused_vars, - warn_shadow_vars, - warn_unused_import, - warn_obsolete_guard, - debug_info, - {parse_transform}]}. - -{xref_checks, [undefined_function_calls, undefined_functions, - locals_not_used, deprecated_function_calls, - warnings_as_errors, deprecated_functions]}. -{cover_enabled, true}. -{cover_opts, [verbose]}. -{cover_export_enabled, true}. diff --git a/apps/emqx_psk_file/src/emqx_psk_file.app.src b/apps/emqx_psk_file/src/emqx_psk_file.app.src deleted file mode 100644 index ef18c8b69..000000000 --- a/apps/emqx_psk_file/src/emqx_psk_file.app.src +++ /dev/null @@ -1,14 +0,0 @@ -{application, emqx_psk_file, - [{description,"EMQX PSK Plugin from File"}, - {vsn, "4.3.1"}, % strict semver, bump manually! - {modules,[]}, - {registered,[emqx_psk_file_sup]}, - {applications,[kernel,stdlib]}, - {mod,{emqx_psk_file_app,[]}}, - {env, []}, - {licenses, ["Apache-2.0"]}, - {maintainers, ["EMQ X Team "]}, - {links, [{"Homepage", "https://emqx.io/"}, - {"Github", "https://github.com/emqx/emqx-psk-file"} - ]} - ]}. diff --git a/apps/emqx_psk_file/src/emqx_psk_file.appup.src b/apps/emqx_psk_file/src/emqx_psk_file.appup.src deleted file mode 100644 index c34a3f71a..000000000 --- a/apps/emqx_psk_file/src/emqx_psk_file.appup.src +++ /dev/null @@ -1,13 +0,0 @@ -%% -*-: erlang -*- -{VSN, - [ - {"4.3.0", [ - {restart_application, emqx_psk_file} - ]} - ], - [ - {"4.3.0", [ - {restart_application, emqx_psk_file} - ]} - ] -}. diff --git a/apps/emqx_psk_file/src/emqx_psk_file.erl b/apps/emqx_psk_file/src/emqx_psk_file.erl deleted file mode 100644 index 3afd6dc73..000000000 --- a/apps/emqx_psk_file/src/emqx_psk_file.erl +++ /dev/null @@ -1,82 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_psk_file). - --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/logger.hrl"). - --import(proplists, [get_value/2]). - --export([load/1, unload/0]). - -%% Hooks functions --export([on_psk_lookup/2]). - --define(TAB, ?MODULE). --define(LF, 10). - --record(psk_entry, {psk_id :: binary(), - psk_str :: binary()}). - -%% Called when the plugin application start -load(Env) -> - _ = ets:new(?TAB, [set, named_table, {keypos, #psk_entry.psk_id}]), - {ok, PskFile} = file:open(get_value(path, Env), [read, raw, binary, read_ahead]), - preload_psks(PskFile, bin(get_value(delimiter, Env))), - _ = file:close(PskFile), - emqx:hook('tls_handshake.psk_lookup', {?MODULE, on_psk_lookup, []}). - -%% Called when the plugin application stop -unload() -> - emqx:unhook('tls_handshake.psk_lookup', {?MODULE, on_psk_lookup}). - -on_psk_lookup(ClientPSKID, UserState) -> - case ets:lookup(?TAB, ClientPSKID) of - [#psk_entry{psk_str = PskStr}] -> - {stop, PskStr}; - [] -> - {ok, UserState} - end. - -preload_psks(FileHandler, Delimiter) -> - case file:read_line(FileHandler) of - {ok, Line} -> - case binary:split(Line, Delimiter) of - [Key, Rem] -> - ets:insert(?TAB, #psk_entry{psk_id = Key, psk_str = trim_lf(Rem)}), - preload_psks(FileHandler, Delimiter); - [Line] -> - ?LOG(warning, "[~p] - Invalid line: ~p, delimiter: ~p", [?MODULE, Line, Delimiter]) - end; - eof -> - ?LOG(info, "[~p] - PSK file is preloaded", [?MODULE]); - {error, Reason} -> - ?LOG(error, "[~p] - Read lines from PSK file: ~p", [?MODULE, Reason]) - end. - -bin(Str) when is_list(Str) -> list_to_binary(Str); -bin(Bin) when is_binary(Bin) -> Bin. - -%% Trim the tailing LF -trim_lf(<<>>) -> <<>>; -trim_lf(Bin) -> - Size = byte_size(Bin), - case binary:at(Bin, Size-1) of - ?LF -> binary_part(Bin, 0, Size-1); - _ -> Bin - end. - diff --git a/apps/emqx_psk_file/test/emqx_psk_file_SUITE.erl b/apps/emqx_psk_file/test/emqx_psk_file_SUITE.erl deleted file mode 100644 index d0083247d..000000000 --- a/apps/emqx_psk_file/test/emqx_psk_file_SUITE.erl +++ /dev/null @@ -1,24 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_psk_file_SUITE). --compile(nowarn_export_all). --compile(export_all). - -all() -> []. - -groups() -> - []. diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index 710deff0c..8e0624c75 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -257,4 +257,8 @@ proc_name(Mod, Id) -> list_to_atom(lists:concat([Mod, "_", Id])). pick(InstId) -> - gproc_pool:pick_worker(emqx_resource_instance, InstId). + Pid = gproc_pool:pick_worker(emqx_resource_instance, InstId), + case is_pid(Pid) of + true -> Pid; + false -> error({failed_to_pick_worker, emqx_resource_instance, InstId}) + end. diff --git a/apps/emqx_resource/src/emqx_resource_validator.erl b/apps/emqx_resource/src/emqx_resource_validator.erl index e9517f160..ee8cb6067 100644 --- a/apps/emqx_resource/src/emqx_resource_validator.erl +++ b/apps/emqx_resource/src/emqx_resource_validator.erl @@ -20,7 +20,7 @@ , max/2 , equals/2 , enum/1 - , required/1 + , not_empty/1 ]). max(Type, Max) -> @@ -38,8 +38,8 @@ enum(Items) -> err_limit({enum, {is_member_of, Items}, {got, Value}})) end. -required(ErrMsg) -> - fun(undefined) -> {error, ErrMsg}; +not_empty(ErrMsg) -> + fun(<<>>) -> {error, ErrMsg}; (_) -> ok end. diff --git a/apps/emqx_retainer/etc/emqx_retainer.conf b/apps/emqx_retainer/etc/emqx_retainer.conf index 4db438a98..3a96909ff 100644 --- a/apps/emqx_retainer/etc/emqx_retainer.conf +++ b/apps/emqx_retainer/etc/emqx_retainer.conf @@ -5,37 +5,77 @@ ## Where to store the retained messages. ## ## Notice that all nodes in the same cluster have to be configured to -## use the same storage_type. -## -## Value: ram | disc | disc_only -## - ram: memory only -## - disc: both memory and disc -## - disc_only: disc only -## -## Default: ram -retainer.storage_type = ram +emqx_retainer: { + ## enable/disable emqx_retainer + enable: true -## Maximum number of retained messages. 0 means no limit. -## -## Value: Number >= 0 -retainer.max_retained_messages = 0 + ## Periodic interval for cleaning up expired messages. Never clear if the value is 0. + ## + ## Value: Duration + ## - h: hour + ## - m: minute + ## - s: second + ## + ## Examples: + ## - 2h: 2 hours + ## - 30m: 30 minutes + ## - 20s: 20 seconds + ## + ## Default: 0s + msg_clear_interval: 0s -## Maximum retained message size. -## -## Value: Bytes -retainer.max_payload_size = 1MB + ## Message retention time. 0 means message will never be expired. + ## + ## Default: 0s + msg_expiry_interval: 0s -## Expiry interval of the retained messages. Never expire if the value is 0. -## -## Value: Duration -## - h: hour -## - m: minute -## - s: second -## -## Examples: -## - 2h: 2 hours -## - 30m: 30 minutes -## - 20s: 20 seconds -## -## Default: 0 -retainer.expiry_interval = 0 + ## The message read and deliver flow rate control + ## When a client subscribe to a wildcard topic, may many retained messages will be loaded. + ## If you don't want these data loaded to the memory all at once, you can use this to control. + ## The processing flow: + ## load max_read_number retained message from storage -> + ## deliver -> + ## repeat this, until all retianed messages are delivered + ## + flow_control: { + ## The max messages number per read from storage. 0 means no limit + ## + ## Default: 0 + max_read_number: 0 + + ## The max number of retained message can be delivered in emqx per quota_release_interval.0 means no limit + ## + ## Default: 0 + msg_deliver_quota: 0 + + ## deliver quota reset interval + ## + ## Default: 0s + quota_release_interval: 0s + } + + ## Maximum retained message size. + ## + ## Value: Bytes + max_payload_size: 1MB + + ## Storage connect parameters + ## + ## Value: mnesia + ## + connector: + [ + { + type: mnesia + config: { + ## storage_type: ram | disc | disc_only + storage_type: ram + + ## Maximum number of retained messages. 0 means no limit. + ## + ## Value: Number >= 0 + max_retained_messages: 0 + } + } + ] +} diff --git a/apps/emqx_retainer/include/emqx_retainer.hrl b/apps/emqx_retainer/include/emqx_retainer.hrl index a1e229cfb..cd07f0692 100644 --- a/apps/emqx_retainer/include/emqx_retainer.hrl +++ b/apps/emqx_retainer/include/emqx_retainer.hrl @@ -14,7 +14,26 @@ %% limitations under the License. %%-------------------------------------------------------------------- +-include_lib("emqx/include/emqx.hrl"). + -define(APP, emqx_retainer). -define(TAB, ?APP). --record(retained, {topic, msg, expiry_time}). +-define(RETAINER_SHARD, emqx_retainer_shard). +-type topic() :: binary(). +-type payload() :: binary(). +-type message() :: #message{}. + +-type context() :: #{context_id := pos_integer(), + atom() => term()}. + +-define(DELIVER_SEMAPHORE, deliver_remained_quota). +-type semaphore() :: ?DELIVER_SEMAPHORE. +-type cursor() :: undefined | term(). +-type result() :: term(). + +-define(SHARED_CONTEXT_TAB, emqx_retainer_ctx). +-record(shared_context, {key :: atom(), value :: term()}). +-type shared_context_key() :: ?DELIVER_SEMAPHORE. + +-type backend() :: emqx_retainer_storage_mnesia. diff --git a/apps/emqx_retainer/priv/emqx_retainer.schema b/apps/emqx_retainer/priv/emqx_retainer.schema deleted file mode 100644 index e598864e1..000000000 --- a/apps/emqx_retainer/priv/emqx_retainer.schema +++ /dev/null @@ -1,30 +0,0 @@ -%%-*- mode: erlang -*- -%% Retainer config mapping - -%% Storage Type -%% {$configurable} -{mapping, "retainer.storage_type", "emqx_retainer.storage_type", [ - {default, ram}, - {datatype, {enum, [ram, disc, disc_only]}} -]}. - -%% Maximum number of retained messages. -%% {$configurable} -{mapping, "retainer.max_retained_messages", "emqx_retainer.max_retained_messages", [ - {default, 0}, - {datatype, integer} -]}. - -%% Maximum payload size of retained message. -%% {$configurable} -{mapping, "retainer.max_payload_size", "emqx_retainer.max_payload_size", [ - {default, "1MB"}, - {datatype, bytesize} -]}. - -%% Expiry interval of retained message -%% {$configurable} -{mapping, "retainer.expiry_interval", "emqx_retainer.expiry_interval", [ - {default, 0}, - {datatype, [integer, {duration, ms}]} -]}. diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index c5ca7599d..4bc3b962b 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -1,6 +1,6 @@ {application, emqx_retainer, [{description, "EMQ X Retainer"}, - {vsn, "4.3.2"}, % strict semver, bump manually! + {vsn, "5.0.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel,stdlib]}, diff --git a/apps/emqx_retainer/src/emqx_retainer.appup.src b/apps/emqx_retainer/src/emqx_retainer.appup.src deleted file mode 100644 index 759ec56bd..000000000 --- a/apps/emqx_retainer/src/emqx_retainer.appup.src +++ /dev/null @@ -1,15 +0,0 @@ -%% -*-: erlang -*- -{VSN, - [ - {<<"4.3.[0-1]">>, [ - {restart_application, emqx_retainer} - ]}, - {<<".*">>, []} - ], - [ - {<<"4.3.[0-1]">>, [ - {restart_application, emqx_retainer} - ]}, - {<<".*">>, []} - ] -}. diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 9e6f60013..524078153 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -21,24 +21,24 @@ -include("emqx_retainer.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("stdlib/include/ms_transform.hrl"). -logger_header("[Retainer]"). --export([start_link/1]). +-export([start_link/0]). --export([ load/1 - , unload/0 - ]). - --export([ on_session_subscribed/3 +-export([ on_session_subscribed/4 , on_message_publish/2 ]). --export([clean/1]). +-export([ dispatch/4 + , delete_message/2 + , store_retained/2 + , deliver/5]). -%% for emqx_pool task func --export([dispatch/2]). +-export([ get_expiry_time/1 + , update_config/1 + , clean/0 + , delete/1]). %% gen_server callbacks -export([ init/1 @@ -49,49 +49,52 @@ , code_change/3 ]). --record(state, {stats_fun, stats_timer, expiry_timer}). +-type state() :: #{ enable := boolean() + , context_id := non_neg_integer() + , context := undefined | context() + , clear_timer := undefined | reference() + , release_quota_timer := undefined | reference() + , wait_quotas := list() + }. + +-rlog_shard({?RETAINER_SHARD, ?TAB}). + +-define(DEF_MAX_PAYLOAD_SIZE, (1024 * 1024)). +-define(DEF_EXPIRY_INTERVAL, 0). + +-define(CAST(Msg), gen_server:cast(?MODULE, Msg)). + +-callback delete_message(context(), topic()) -> ok. +-callback store_retained(context(), message()) -> ok. +-callback read_message(context(), topic()) -> {ok, list()}. +-callback match_messages(context(), topic(), cursor()) -> {ok, list(), cursor()}. +-callback clear_expired(context()) -> ok. +-callback clean(context()) -> ok. %%-------------------------------------------------------------------- -%% Load/Unload +%% Hook API %%-------------------------------------------------------------------- - -load(Env) -> - _ = emqx:hook('session.subscribed', {?MODULE, on_session_subscribed, []}), - _ = emqx:hook('message.publish', {?MODULE, on_message_publish, [Env]}), - ok. - -unload() -> - emqx:unhook('message.publish', {?MODULE, on_message_publish}), - emqx:unhook('session.subscribed', {?MODULE, on_session_subscribed}). - -on_session_subscribed(_, _, #{share := ShareName}) when ShareName =/= undefined -> +on_session_subscribed(_, _, #{share := ShareName}, _) when ShareName =/= undefined -> ok; -on_session_subscribed(_, Topic, #{rh := Rh, is_new := IsNew}) -> +on_session_subscribed(_, Topic, #{rh := Rh, is_new := IsNew}, Context) -> case Rh =:= 0 orelse (Rh =:= 1 andalso IsNew) of - true -> emqx_pool:async_submit(fun ?MODULE:dispatch/2, [self(), Topic]); + true -> dispatch(Context, Topic); _ -> ok end. -%% @private -dispatch(Pid, Topic) -> - Msgs = case emqx_topic:wildcard(Topic) of - false -> read_messages(Topic); - true -> match_messages(Topic) - end, - [Pid ! {deliver, Topic, Msg} || Msg <- sort_retained(Msgs)]. - %% RETAIN flag set to 1 and payload containing zero bytes on_message_publish(Msg = #message{flags = #{retain := true}, topic = Topic, - payload = <<>>}, _Env) -> - mnesia:dirty_delete(?TAB, topic2tokens(Topic)), + payload = <<>>}, + Context) -> + delete_message(Context, Topic), {ok, Msg}; -on_message_publish(Msg = #message{flags = #{retain := true}}, Env) -> +on_message_publish(Msg = #message{flags = #{retain := true}}, Context) -> Msg1 = emqx_message:set_header(retained, true, Msg), - store_retained(Msg1, Env), + store_retained(Context, Msg1), {ok, Msg}; -on_message_publish(Msg, _Env) -> +on_message_publish(Msg, _) -> {ok, Msg}. %%-------------------------------------------------------------------- @@ -99,64 +102,102 @@ on_message_publish(Msg, _Env) -> %%-------------------------------------------------------------------- %% @doc Start the retainer --spec(start_link(Env :: list()) -> emqx_types:startlink_ret()). -start_link(Env) -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [Env], []). +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). --spec(clean(emqx_types:topic()) -> non_neg_integer()). -clean(Topic) when is_binary(Topic) -> - case emqx_topic:wildcard(Topic) of - true -> match_delete_messages(Topic); +-spec dispatch(context(), pid(), topic(), cursor()) -> ok. +dispatch(Context, Pid, Topic, Cursor) -> + Mod = get_backend_module(), + case Cursor =/= undefined orelse emqx_topic:wildcard(Topic) of false -> - Tokens = topic2tokens(Topic), - Fun = fun() -> - case mnesia:read({?TAB, Tokens}) of - [] -> 0; - [_M] -> mnesia:delete({?TAB, Tokens}), 1 - end - end, - {atomic, N} = mnesia:transaction(Fun), N + {ok, Result} = Mod:read_message(Context, Topic), + deliver(Result, Context, Pid, Topic, undefined); + true -> + {ok, Result, NewCursor} = Mod:match_messages(Context, Topic, Cursor), + deliver(Result, Context, Pid, Topic, NewCursor) end. +deliver([], Context, Pid, Topic, Cursor) -> + case Cursor of + undefined -> + ok; + _ -> + dispatch(Context, Pid, Topic, Cursor) + end; +deliver(Result, #{context_id := Id} = Context, Pid, Topic, Cursor) -> + case erlang:is_process_alive(Pid) of + false -> + ok; + _ -> + #{msg_deliver_quota := MaxDeliverNum} = emqx_config:get([?APP, flow_control]), + case MaxDeliverNum of + 0 -> + _ = [Pid ! {deliver, Topic, Msg} || Msg <- Result], + ok; + _ -> + case do_deliver(Result, Id, Pid, Topic) of + ok -> + deliver([], Context, Pid, Topic, Cursor); + abort -> + ok + end + end + end. + +get_expiry_time(#message{headers = #{properties := #{'Message-Expiry-Interval' := 0}}}) -> + 0; +get_expiry_time(#message{headers = #{properties := #{'Message-Expiry-Interval' := Interval}}, + timestamp = Ts}) -> + Ts + Interval * 1000; +get_expiry_time(#message{timestamp = Ts}) -> + Interval = emqx_config:get([?APP, msg_expiry_interval], ?DEF_EXPIRY_INTERVAL), + case Interval of + 0 -> 0; + _ -> Ts + Interval + end. + +-spec update_config(hocon:config()) -> ok. +update_config(Conf) -> + gen_server:call(?MODULE, {?FUNCTION_NAME, Conf}). + +clean() -> + gen_server:call(?MODULE, ?FUNCTION_NAME). + +delete(Topic) -> + gen_server:call(?MODULE, {?FUNCTION_NAME, Topic}). + %%-------------------------------------------------------------------- %% gen_server callbacks %%-------------------------------------------------------------------- -init([Env]) -> - Copies = case proplists:get_value(storage_type, Env, disc) of - ram -> ram_copies; - disc -> disc_copies; - disc_only -> disc_only_copies - end, - StoreProps = [{ets, [compressed, - {read_concurrency, true}, - {write_concurrency, true}]}, - {dets, [{auto_save, 1000}]}], - ok = ekka_mnesia:create_table(?TAB, [ - {type, set}, - {Copies, [node()]}, - {record_name, retained}, - {attributes, record_info(fields, retained)}, - {storage_properties, StoreProps}]), - ok = ekka_mnesia:copy_table(?TAB, Copies), - case mnesia:table_info(?TAB, storage_type) of - Copies -> ok; - _Other -> - {atomic, ok} = mnesia:change_table_copy_type(?TAB, node(), Copies), - ok - end, - StatsFun = emqx_stats:statsfun('retained.count', 'retained.max'), - {ok, StatsTimer} = timer:send_interval(timer:seconds(1), stats), - State = #state{stats_fun = StatsFun, stats_timer = StatsTimer}, - {ok, start_expire_timer(proplists:get_value(expiry_interval, Env, 0), State)}. +init([]) -> + init_shared_context(), + State = new_state(), + #{enable := Enable} = Cfg = emqx_config:get([?APP]), + {ok, + case Enable of + true -> + enable_retainer(State, Cfg); + _ -> + State + end}. -start_expire_timer(0, State) -> - State; -start_expire_timer(undefined, State) -> - State; -start_expire_timer(Ms, State) -> - {ok, Timer} = timer:send_interval(Ms, expire), - State#state{expiry_timer = Timer}. +handle_call({update_config, Conf}, _, State) -> + State2 = update_config(State, Conf), + emqx_config:put([?APP], Conf), + {reply, ok, State2}; + +handle_call({wait_semaphore, Id}, From, #{wait_quotas := Waits} = State) -> + {noreply, State#{wait_quotas := [{Id, From} | Waits]}}; + +handle_call(clean, _, #{context := Context} = State) -> + clean(Context), + {reply, ok, State}; + +handle_call({delete, Topic}, _, #{context := Context} = State) -> + delete_message(Context, Topic), + {reply, ok, State}; handle_call(Req, _From, State) -> ?LOG(error, "Unexpected call: ~p", [Req]), @@ -166,21 +207,36 @@ handle_cast(Msg, State) -> ?LOG(error, "Unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info(stats, State = #state{stats_fun = StatsFun}) -> - StatsFun(retained_count()), - {noreply, State, hibernate}; +handle_info(clear_expired, #{context := Context} = State) -> + Mod = get_backend_module(), + Mod:clear_expired(Context), + Interval = emqx_config:get([?APP, msg_clear_interval], ?DEF_EXPIRY_INTERVAL), + {noreply, State#{clear_timer := add_timer(Interval, clear_expired)}, hibernate}; -handle_info(expire, State) -> - ok = expire_messages(), - {noreply, State, hibernate}; +handle_info(release_deliver_quota, #{context := Context, wait_quotas := Waits} = State) -> + insert_shared_context(?DELIVER_SEMAPHORE, get_msg_deliver_quota()), + case Waits of + [] -> + ok; + _ -> + #{context_id := NowId} = Context, + Waits2 = lists:reverse(Waits), + lists:foreach(fun({Id, From}) -> + gen_server:reply(From, Id =:= NowId) + end, + Waits2) + end, + Interval = emqx_config:get([?APP, flow_control, quota_release_interval]), + {noreply, State#{release_quota_timer := add_timer(Interval, release_deliver_quota), + wait_quotas := []}}; handle_info(Info, State) -> ?LOG(error, "Unexpected info: ~p", [Info]), {noreply, State}. -terminate(_Reason, #state{stats_timer = TRef1, expiry_timer = TRef2}) -> - _ = timer:cancel(TRef1), - _ = timer:cancel(TRef2), +terminate(_Reason, #{clear_timer := TRef1, release_quota_timer := TRef2}) -> + _ = stop_timer(TRef1), + _ = stop_timer(TRef2), ok. code_change(_OldVsn, State, _Extra) -> @@ -189,117 +245,211 @@ code_change(_OldVsn, State, _Extra) -> %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- +-spec new_state() -> state(). +new_state() -> + #{enable => false, + context_id => 0, + context => undefined, + clear_timer => undefined, + release_quota_timer => undefined, + wait_quotas => []}. -sort_retained([]) -> []; -sort_retained([Msg]) -> [Msg]; -sort_retained(Msgs) -> - lists:sort(fun(#message{timestamp = Ts1}, #message{timestamp = Ts2}) -> - Ts1 =< Ts2 - end, Msgs). +-spec new_context(pos_integer()) -> context(). +new_context(Id) -> + #{context_id => Id}. -store_retained(Msg = #message{topic = Topic, payload = Payload}, Env) -> - case {is_table_full(Env), is_too_big(size(Payload), Env)} of - {false, false} -> - ok = emqx_metrics:inc('messages.retained'), - mnesia:dirty_write(?TAB, #retained{topic = topic2tokens(Topic), - msg = Msg, - expiry_time = get_expiry_time(Msg, Env)}); - {true, false} -> - {atomic, _} = mnesia:transaction( - fun() -> - case mnesia:read(?TAB, Topic) of - [_] -> - mnesia:write(?TAB, #retained{topic = topic2tokens(Topic), - msg = Msg, - expiry_time = get_expiry_time(Msg, Env)}, write); - [] -> - ?LOG(error, "Cannot retain message(topic=~s) for table is full!", [Topic]) - end - end), - ok; - {true, _} -> - ?LOG(error, "Cannot retain message(topic=~s) for table is full!", [Topic]); - {_, true} -> - ?LOG(error, "Cannot retain message(topic=~s, payload_size=~p) " - "for payload is too big!", [Topic, iolist_size(Payload)]) - end. - -is_table_full(Env) -> - Limit = proplists:get_value(max_retained_messages, Env, 0), - Limit > 0 andalso (retained_count() > Limit). - -is_too_big(Size, Env) -> - Limit = proplists:get_value(max_payload_size, Env, 0), +is_too_big(Size) -> + Limit = emqx_config:get([?APP, max_payload_size], ?DEF_MAX_PAYLOAD_SIZE), Limit > 0 andalso (Size > Limit). -get_expiry_time(#message{headers = #{properties := #{'Message-Expiry-Interval' := 0}}}, _Env) -> - 0; -get_expiry_time(#message{headers = #{properties := #{'Message-Expiry-Interval' := Interval}}, timestamp = Ts}, _Env) -> - Ts + Interval * 1000; -get_expiry_time(#message{timestamp = Ts}, Env) -> - case proplists:get_value(expiry_interval, Env, 0) of - 0 -> 0; - Interval -> Ts + Interval +%% @private +dispatch(Context, Topic) -> + emqx_retainer_pool:async_submit(fun ?MODULE:dispatch/4, + [Context, self(), Topic, undefined]). + +-spec delete_message(context(), topic()) -> ok. +delete_message(Context, Topic) -> + Mod = get_backend_module(), + Mod:delete_message(Context, Topic). + +-spec store_retained(context(), message()) -> ok. +store_retained(Context, #message{topic = Topic, payload = Payload} = Msg) -> + case is_too_big(erlang:byte_size(Payload)) of + false -> + Mod = get_backend_module(), + Mod:store_retained(Context, Msg); + _ -> + ?ERROR("Cannot retain message(topic=~s, payload_size=~p) for payload is too big!", + [Topic, iolist_size(Payload)]) end. -%%-------------------------------------------------------------------- -%% Internal funcs -%%-------------------------------------------------------------------- +-spec clean(context()) -> ok. +clean(Context) -> + Mod = get_backend_module(), + Mod:clean(Context). --spec(retained_count() -> non_neg_integer()). -retained_count() -> mnesia:table_info(?TAB, size). - -topic2tokens(Topic) -> - emqx_topic:words(Topic). - -expire_messages() -> - NowMs = erlang:system_time(millisecond), - MsHd = #retained{topic = '$1', msg = '_', expiry_time = '$3'}, - Ms = [{MsHd, [{'=/=','$3',0}, {'<','$3',NowMs}], ['$1']}], - {atomic, _} = mnesia:transaction( - fun() -> - Keys = mnesia:select(?TAB, Ms, write), - lists:foreach(fun(Key) -> mnesia:delete({?TAB, Key}) end, Keys) - end), +-spec do_deliver(list(term()), pos_integer(), pid(), topic()) -> ok | abort. +do_deliver([Msg | T], Id, Pid, Topic) -> + case require_semaphore(?DELIVER_SEMAPHORE, Id) of + true -> + Pid ! {deliver, Topic, Msg}, + do_deliver(T, Id, Pid, Topic); + _ -> + abort + end; +do_deliver([], _, _, _) -> ok. --spec(read_messages(emqx_types:topic()) - -> [emqx_types:message()]). -read_messages(Topic) -> - Tokens = topic2tokens(Topic), - case mnesia:dirty_read(?TAB, Tokens) of - [] -> []; - [#retained{msg = Msg, expiry_time = Et}] -> - case Et =:= 0 orelse Et >= erlang:system_time(millisecond) of - true -> [Msg]; - false -> [] - end +-spec require_semaphore(semaphore(), pos_integer()) -> boolean(). +require_semaphore(Semaphore, Id) -> + Remained = ets:update_counter(?SHARED_CONTEXT_TAB, + Semaphore, + {#shared_context.value, -1, -1, -1}), + wait_semaphore(Remained, Id). + +-spec wait_semaphore(non_neg_integer(), pos_integer()) -> boolean(). +wait_semaphore(X, Id) when X < 0 -> + gen_server:call(?MODULE, {?FUNCTION_NAME, Id}, infinity); +wait_semaphore(_, _) -> + true. + +-spec init_shared_context() -> ok. +init_shared_context() -> + ?SHARED_CONTEXT_TAB = ets:new(?SHARED_CONTEXT_TAB, + [ set, named_table, public + , {keypos, #shared_context.key} + , {write_concurrency, true} + , {read_concurrency, true}]), + lists:foreach(fun({K, V}) -> + insert_shared_context(K, V) + end, + [{?DELIVER_SEMAPHORE, get_msg_deliver_quota()}]). + + +-spec insert_shared_context(shared_context_key(), term()) -> ok. +insert_shared_context(Key, Term) -> + ets:insert(?SHARED_CONTEXT_TAB, #shared_context{key = Key, value = Term}), + ok. + +-spec get_msg_deliver_quota() -> non_neg_integer(). +get_msg_deliver_quota() -> + emqx_config:get([?APP, flow_control, msg_deliver_quota]). + +-spec update_config(state(), hocons:config()) -> state(). +update_config(#{clear_timer := ClearTimer, + release_quota_timer := QuotaTimer} = State, Conf) -> + #{enable := Enable, + connector := [Connector | _], + flow_control := #{quota_release_interval := QuotaInterval}, + msg_clear_interval := ClearInterval} = Conf, + + #{connector := [OldConnector | _]} = emqx_config:get([?APP]), + + case Enable of + true -> + StorageType = maps:get(type, Connector), + OldStrorageType = maps:get(type, OldConnector), + case OldStrorageType of + StorageType -> + State#{clear_timer := check_timer(ClearTimer, + ClearInterval, + clear_expired), + release_quota_timer := check_timer(QuotaTimer, + QuotaInterval, + release_deliver_quota)}; + _ -> + State2 = disable_retainer(State), + enable_retainer(State2, Conf) + end; + _ -> + disable_retainer(State) end. --spec(match_messages(emqx_types:topic()) - -> [emqx_types:message()]). -match_messages(Filter) -> - NowMs = erlang:system_time(millisecond), - Cond = condition(emqx_topic:words(Filter)), - MsHd = #retained{topic = Cond, msg = '$2', expiry_time = '$3'}, - Ms = [{MsHd, [{'=:=','$3',0}], ['$2']}, - {MsHd, [{'>','$3',NowMs}], ['$2']}], - mnesia:dirty_select(?TAB, Ms). +-spec enable_retainer(state(), hocon:config()) -> state(). +enable_retainer(#{context_id := ContextId} = State, + #{msg_clear_interval := ClearInterval, + flow_control := #{quota_release_interval := ReleaseInterval}, + connector := [Connector | _]}) -> + NewContextId = ContextId + 1, + Context = create_resource(new_context(NewContextId), Connector), + load(Context), + State#{enable := true, + context_id := NewContextId, + context := Context, + clear_timer := add_timer(ClearInterval, clear_expired), + release_quota_timer := add_timer(ReleaseInterval, release_deliver_quota)}. --spec(match_delete_messages(emqx_types:topic()) - -> DeletedCnt :: non_neg_integer()). -match_delete_messages(Filter) -> - Cond = condition(emqx_topic:words(Filter)), - MsHd = #retained{topic = Cond, msg = '_', expiry_time = '_'}, - Ms = [{MsHd, [], ['$_']}], - Rs = mnesia:dirty_select(?TAB, Ms), - lists:foreach(fun(R) -> mnesia:dirty_delete_object(?TAB, R) end, Rs), - length(Rs). +-spec disable_retainer(state()) -> state(). +disable_retainer(#{clear_timer := TRef1, + release_quota_timer := TRef2, + context := Context, + wait_quotas := Waits} = State) -> + unload(), + ok = lists:foreach(fun(E) -> gen_server:reply(E, false) end, Waits), + ok = close_resource(Context), + State#{enable := false, + clear_timer := stop_timer(TRef1), + release_quota_timer := stop_timer(TRef2), + wait_quotas := []}. -%% @private -condition(Ws) -> - Ws1 = [case W =:= '+' of true -> '_'; _ -> W end || W <- Ws], - case lists:last(Ws1) =:= '#' of - false -> Ws1; - _ -> (Ws1 -- ['#']) ++ '_' +-spec stop_timer(undefined | reference()) -> undefined. +stop_timer(undefined) -> + undefined; +stop_timer(TimerRef) -> + _ = erlang:cancel_timer(TimerRef), + undefined. + +add_timer(0, _) -> + undefined; +add_timer(undefined, _) -> + undefined; +add_timer(Ms, Content) -> + erlang:send_after(Ms, self(), Content). + +check_timer(undefined, Ms, Context) -> + add_timer(Ms, Context); +check_timer(Timer, 0, _) -> + stop_timer(Timer); +check_timer(Timer, undefined, _) -> + stop_timer(Timer); +check_timer(Timer, _, _) -> + Timer. + +-spec get_backend_module() -> backend(). +get_backend_module() -> + [#{type := Backend} | _] = emqx_config:get([?APP, connector]), + erlang:list_to_existing_atom(io_lib:format("~s_~s", [?APP, Backend])). + +create_resource(Context, #{type := mnesia, config := Cfg}) -> + emqx_retainer_mnesia:create_resource(Cfg), + Context; + +create_resource(Context, #{type := DB, config := Config}) -> + ResourceID = erlang:iolist_to_binary([io_lib:format("~s_~s", [?APP, DB])]), + case emqx_resource:create( + ResourceID, + list_to_existing_atom(io_lib:format("~s_~s", [emqx_connector, DB])), + Config) of + {ok, _} -> + Context#{resource_id => ResourceID}; + {error, already_created} -> + Context#{resource_id => ResourceID}; + {error, Reason} -> + error({load_config_error, Reason}) end. + +-spec close_resource(context()) -> ok | {error, term()}. +close_resource(#{resource_id := ResourceId}) -> + emqx_resource:stop(ResourceId); +close_resource(_) -> + ok. + +-spec load(context()) -> ok. +load(Context) -> + _ = emqx:hook('session.subscribed', {?MODULE, on_session_subscribed, [Context]}), + _ = emqx:hook('message.publish', {?MODULE, on_message_publish, [Context]}), + ok. + +unload() -> + emqx:unhook('message.publish', {?MODULE, on_message_publish}), + emqx:unhook('session.subscribed', {?MODULE, on_session_subscribed}). diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl new file mode 100644 index 000000000..1b5b8adcc --- /dev/null +++ b/apps/emqx_retainer/src/emqx_retainer_api.erl @@ -0,0 +1,67 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_retainer_api). + +-rest_api(#{name => lookup_config, + method => 'GET', + path => "/retainer", + func => lookup_config, + descr => "lookup retainer config" + }). + +-rest_api(#{name => update_config, + method => 'PUT', + path => "/retainer", + func => update_config, + descr => "update retainer config" + }). + +-export([ lookup_config/2 + , update_config/2 + ]). + +lookup_config(_Bindings, _Params) -> + Config = emqx_config:get([emqx_retainer]), + return({ok, Config}). + +update_config(_Bindings, Params) -> + try + ConfigList = proplists:get_value(<<"emqx_retainer">>, Params), + {ok, RawConf} = hocon:binary(jsx:encode(#{<<"emqx_retainer">> => ConfigList}), + #{format => richmap}), + RichConf = hocon_schema:check(emqx_retainer_schema, RawConf, #{atom_key => true}), + #{emqx_retainer := Conf} = hocon_schema:richmap_to_map(RichConf), + Action = proplists:get_value(<<"action">>, Params, undefined), + do_update_config(Action, Conf), + return() + catch _:_:Reason -> + return({error, Reason}) + end. + +%%------------------------------------------------------------------------------ +%% Interval Funcs +%%------------------------------------------------------------------------------ +do_update_config(undefined, Config) -> + emqx_retainer:update_config(Config); +do_update_config(<<"test">>, _) -> + ok. + +%% TODO: V5 API +return() -> + ok. +return(_) -> + ok. diff --git a/apps/emqx_retainer/src/emqx_retainer_app.erl b/apps/emqx_retainer/src/emqx_retainer_app.erl index adca5ae7a..eda6f16fb 100644 --- a/apps/emqx_retainer/src/emqx_retainer_app.erl +++ b/apps/emqx_retainer/src/emqx_retainer_app.erl @@ -18,20 +18,15 @@ -behaviour(application). --emqx_plugin(?MODULE). - -export([ start/2 , stop/1 ]). start(_Type, _Args) -> - Env = application:get_all_env(emqx_retainer), - {ok, Sup} = emqx_retainer_sup:start_link(Env), - emqx_retainer:load(Env), + {ok, Sup} = emqx_retainer_sup:start_link(), emqx_retainer_cli:load(), {ok, Sup}. stop(_State) -> - emqx_retainer_cli:unload(), - emqx_retainer:unload(). + emqx_retainer_cli:unload(). diff --git a/apps/emqx_retainer/src/emqx_retainer_cli.erl b/apps/emqx_retainer/src/emqx_retainer_cli.erl index fe8fa9578..f24d69bed 100644 --- a/apps/emqx_retainer/src/emqx_retainer_cli.erl +++ b/apps/emqx_retainer/src/emqx_retainer_cli.erl @@ -27,26 +27,6 @@ load() -> emqx_ctl:register_command(retainer, {?MODULE, cmd}, []). -cmd(["info"]) -> - emqx_ctl:print("retained/total: ~w~n", [mnesia:table_info(?TAB, size)]); - -cmd(["topics"]) -> - case mnesia:dirty_all_keys(?TAB) of - [] -> ignore; - Topics -> lists:foreach(fun(Topic) -> emqx_ctl:print("~s~n", [Topic]) end, Topics) - end; - -cmd(["clean"]) -> - Size = mnesia:table_info(?TAB, size), - case mnesia:clear_table(?TAB) of - {atomic, ok} -> emqx_ctl:print("Cleaned ~p retained messages~n", [Size]); - {aborted, R} -> emqx_ctl:print("Aborted ~p~n", [R]) - end; - -cmd(["clean", Topic]) -> - Lines = emqx_retainer:clean(list_to_binary(Topic)), - emqx_ctl:print("Cleaned ~p retained messages~n", [Lines]); - cmd(_) -> emqx_ctl:usage([{"retainer info", "Show the count of retained messages"}, {"retainer topics", "Show all topics of retained messages"}, @@ -55,4 +35,3 @@ cmd(_) -> unload() -> emqx_ctl:unregister_command(retainer). - diff --git a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl new file mode 100644 index 000000000..5b6028980 --- /dev/null +++ b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl @@ -0,0 +1,241 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_retainer_mnesia). + +-behaviour(emqx_retainer). + +-include("emqx_retainer.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). +-include_lib("stdlib/include/qlc.hrl"). + +-logger_header("[Retainer]"). + +-export([delete_message/2 + , store_retained/2 + , read_message/2 + , match_messages/3 + , clear_expired/1 + , clean/1]). + +-export([create_resource/1]). + +-define(DEF_MAX_RETAINED_MESSAGES, 0). + +-rlog_shard({?RETAINER_SHARD, ?TAB}). + +-record(retained, {topic, msg, expiry_time}). + +-type batch_read_result() :: + {ok, list(emqx:message()), cursor()}. + +%%-------------------------------------------------------------------- +%% emqx_retainer_storage callbacks +%%-------------------------------------------------------------------- +create_resource(#{storage_type := StorageType}) -> + Copies = case StorageType of + ram -> ram_copies; + disc -> disc_copies; + disc_only -> disc_only_copies + end, + StoreProps = [{ets, [compressed, + {read_concurrency, true}, + {write_concurrency, true}]}, + {dets, [{auto_save, 1000}]}], + ok = ekka_mnesia:create_table(?TAB, [ + {type, set}, + {Copies, [node()]}, + {record_name, retained}, + {attributes, record_info(fields, retained)}, + {storage_properties, StoreProps}]), + ok = ekka_mnesia:copy_table(?TAB, Copies), + ok = ekka_rlog:wait_for_shards([?RETAINER_SHARD], infinity), + case mnesia:table_info(?TAB, storage_type) of + Copies -> ok; + _Other -> + {atomic, ok} = mnesia:change_table_copy_type(?TAB, node(), Copies), + ok + end. + +store_retained(_, Msg =#message{topic = Topic}) -> + ExpiryTime = emqx_retainer:get_expiry_time(Msg), + case is_table_full() of + false -> + ok = emqx_metrics:inc('messages.retained'), + ekka_mnesia:dirty_write(?TAB, + #retained{topic = topic2tokens(Topic), + msg = Msg, + expiry_time = ExpiryTime}); + _ -> + Tokens = topic2tokens(Topic), + Fun = fun() -> + case mnesia:read(?TAB, Tokens) of + [_] -> + mnesia:write(?TAB, + #retained{topic = Tokens, + msg = Msg, + expiry_time = ExpiryTime}, + write); + [] -> + ?LOG(error, + "Cannot retain message(topic=~s) for table is full!", + [Topic]), + ok + end + end, + {atomic, ok} = ekka_mnesia:transaction(?RETAINER_SHARD, Fun), + ok + end. + +clear_expired(_) -> + NowMs = erlang:system_time(millisecond), + MsHd = #retained{topic = '$1', msg = '_', expiry_time = '$3'}, + Ms = [{MsHd, [{'=/=', '$3', 0}, {'<', '$3', NowMs}], ['$1']}], + Fun = fun() -> + Keys = mnesia:select(?TAB, Ms, write), + lists:foreach(fun(Key) -> mnesia:delete({?TAB, Key}) end, Keys) + end, + {atomic, _} = ekka_mnesia:transaction(?RETAINER_SHARD, Fun), + ok. + +delete_message(_, Topic) -> + case emqx_topic:wildcard(Topic) of + true -> match_delete_messages(Topic); + false -> + Tokens = topic2tokens(Topic), + Fun = fun() -> + mnesia:delete({?TAB, Tokens}) + end, + case ekka_mnesia:transaction(?RETAINER_SHARD, Fun) of + {atomic, Result} -> + Result; + ok -> + ok + end + end, + ok. + +read_message(_, Topic) -> + {ok, read_messages(Topic)}. + +match_messages(_, Topic, Cursor) -> + MaxReadNum = emqx_config:get([?APP, flow_control, max_read_number]), + case Cursor of + undefined -> + case MaxReadNum of + 0 -> + {ok, sort_retained(match_messages(Topic)), undefined}; + _ -> + start_batch_read(Topic, MaxReadNum) + end; + _ -> + batch_read_messages(Cursor, MaxReadNum) + end. + +clean(_) -> + ekka_mnesia:clear_table(?TAB), + ok. +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +sort_retained([]) -> []; +sort_retained([Msg]) -> [Msg]; +sort_retained(Msgs) -> + lists:sort(fun(#message{timestamp = Ts1}, #message{timestamp = Ts2}) -> + Ts1 =< Ts2 end, + Msgs). + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- +topic2tokens(Topic) -> + emqx_topic:words(Topic). + +-spec start_batch_read(topic(), pos_integer()) -> batch_read_result(). +start_batch_read(Topic, MaxReadNum) -> + Ms = make_match_spec(Topic), + TabQH = ets:table(?TAB, [{traverse, {select, Ms}}]), + QH = qlc:q([E || E <- TabQH]), + Cursor = qlc:cursor(QH), + batch_read_messages(Cursor, MaxReadNum). + +-spec batch_read_messages(emqx_retainer_storage:cursor(), pos_integer()) -> batch_read_result(). +batch_read_messages(Cursor, MaxReadNum) -> + Answers = qlc:next_answers(Cursor, MaxReadNum), + Orders = sort_retained(Answers), + case erlang:length(Orders) < MaxReadNum of + true -> + qlc:delete_cursor(Cursor), + {ok, Orders, undefined}; + _ -> + {ok, Orders, Cursor} + end. + +-spec(read_messages(emqx_types:topic()) + -> [emqx_types:message()]). +read_messages(Topic) -> + Tokens = topic2tokens(Topic), + case mnesia:dirty_read(?TAB, Tokens) of + [] -> []; + [#retained{msg = Msg, expiry_time = Et}] -> + case Et =:= 0 orelse Et >= erlang:system_time(millisecond) of + true -> [Msg]; + false -> [] + end + end. + +-spec(match_messages(emqx_types:topic()) + -> [emqx_types:message()]). +match_messages(Filter) -> + Ms = make_match_spec(Filter), + mnesia:dirty_select(?TAB, Ms). + +-spec(match_delete_messages(emqx_types:topic()) -> ok). +match_delete_messages(Filter) -> + Cond = condition(emqx_topic:words(Filter)), + MsHd = #retained{topic = Cond, msg = '_', expiry_time = '_'}, + Ms = [{MsHd, [], ['$_']}], + Rs = mnesia:dirty_select(?TAB, Ms), + lists:foreach(fun(R) -> ekka_mnesia:dirty_delete_object(?TAB, R) end, Rs). + +%% @private +condition(Ws) -> + Ws1 = [case W =:= '+' of true -> '_'; _ -> W end || W <- Ws], + case lists:last(Ws1) =:= '#' of + false -> Ws1; + _ -> (Ws1 -- ['#']) ++ '_' + end. + +-spec make_match_spec(topic()) -> ets:match_spec(). +make_match_spec(Filter) -> + NowMs = erlang:system_time(millisecond), + Cond = condition(emqx_topic:words(Filter)), + MsHd = #retained{topic = Cond, msg = '$2', expiry_time = '$3'}, + [{MsHd, [{'=:=', '$3', 0}], ['$2']}, + {MsHd, [{'>', '$3', NowMs}], ['$2']}]. + +-spec is_table_full() -> boolean(). +is_table_full() -> + [#{config := Cfg} | _] = emqx_config:get([?APP, connector]), + Limit = maps:get(max_retained_messages, + Cfg, + ?DEF_MAX_RETAINED_MESSAGES), + Limit > 0 andalso (table_size() >= Limit). + +-spec table_size() -> non_neg_integer(). +table_size() -> + mnesia:table_info(?TAB, size). diff --git a/apps/emqx_retainer/src/emqx_retainer_pool.erl b/apps/emqx_retainer/src/emqx_retainer_pool.erl new file mode 100644 index 000000000..59ea1077a --- /dev/null +++ b/apps/emqx_retainer/src/emqx_retainer_pool.erl @@ -0,0 +1,182 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_retainer_pool). + +-behaviour(gen_server). + +-include_lib("emqx/include/logger.hrl"). + +%% API +-export([start_link/2, + async_submit/2]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3, format_status/2]). + +-define(POOL, ?MODULE). + +%%%=================================================================== +%%% API +%%%=================================================================== +async_submit(Fun, Args) -> + cast({async_submit, {Fun, Args}}). + +%%-------------------------------------------------------------------- +%% @doc +%% Starts the server +%% @end +%%-------------------------------------------------------------------- +-spec start_link(atom(), pos_integer()) -> {ok, Pid :: pid()} | + {error, Error :: {already_started, pid()}} | + {error, Error :: term()} | + ignore. +start_link(Pool, Id) -> + gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)}, + ?MODULE, [Pool, Id], [{hibernate_after, 1000}]). + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Initializes the server +%% @end +%%-------------------------------------------------------------------- +-spec init(Args :: term()) -> {ok, State :: term()} | + {ok, State :: term(), Timeout :: timeout()} | + {ok, State :: term(), hibernate} | + {stop, Reason :: term()} | + ignore. +init([Pool, Id]) -> + true = gproc_pool:connect_worker(Pool, {Pool, Id}), + {ok, #{pool => Pool, id => Id}}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling call messages +%% @end +%%-------------------------------------------------------------------- +-spec handle_call(Request :: term(), From :: {pid(), term()}, State :: term()) -> + {reply, Reply :: term(), NewState :: term()} | + {reply, Reply :: term(), NewState :: term(), Timeout :: timeout()} | + {reply, Reply :: term(), NewState :: term(), hibernate} | + {noreply, NewState :: term()} | + {noreply, NewState :: term(), Timeout :: timeout()} | + {noreply, NewState :: term(), hibernate} | + {stop, Reason :: term(), Reply :: term(), NewState :: term()} | + {stop, Reason :: term(), NewState :: term()}. +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignored, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling cast messages +%% @end +%%-------------------------------------------------------------------- +-spec handle_cast(Request :: term(), State :: term()) -> + {noreply, NewState :: term()} | + {noreply, NewState :: term(), Timeout :: timeout()} | + {noreply, NewState :: term(), hibernate} | + {stop, Reason :: term(), NewState :: term()}. +handle_cast({async_submit, Task}, State) -> + try run(Task) + catch _:Error:Stacktrace -> + ?LOG(error, "Error: ~0p, ~0p", [Error, Stacktrace]) + end, + {noreply, State}; + +handle_cast(Msg, State) -> + ?LOG(error, "Unexpected cast: ~p", [Msg]), + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling all non call/cast messages +%% @end +%%-------------------------------------------------------------------- +-spec handle_info(Info :: timeout() | term(), State :: term()) -> + {noreply, NewState :: term()} | + {noreply, NewState :: term(), Timeout :: timeout()} | + {noreply, NewState :: term(), hibernate} | + {stop, Reason :: normal | term(), NewState :: term()}. +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any +%% necessary cleaning up. When it returns, the gen_server terminates +%% with Reason. The return value is ignored. +%% @end +%%-------------------------------------------------------------------- +-spec terminate(Reason :: normal | shutdown | {shutdown, term()} | term(), + State :: term()) -> any(). +terminate(_Reason, #{pool := Pool, id := Id}) -> + gproc_pool:disconnect_worker(Pool, {Pool, Id}). +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Convert process state when code is changed +%% @end +%%-------------------------------------------------------------------- +-spec code_change(OldVsn :: term() | {down, term()}, + State :: term(), + Extra :: term()) -> {ok, NewState :: term()} | + {error, Reason :: term()}. +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called for changing the form and appearance +%% of gen_server status when it is returned from sys:get_status/1,2 +%% or when it appears in termination error logs. +%% @end +%%-------------------------------------------------------------------- +-spec format_status(Opt :: normal | terminate, + Status :: list()) -> Status :: term(). +format_status(_Opt, Status) -> + Status. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== +%% @private +cast(Msg) -> + gen_server:cast(worker(), Msg). + +%% @private +worker() -> + gproc_pool:pick_worker(?POOL). + +run({M, F, A}) -> + erlang:apply(M, F, A); +run({F, A}) when is_function(F), is_list(A) -> + erlang:apply(F, A); +run(Fun) when is_function(Fun) -> + Fun(). diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl new file mode 100644 index 000000000..96cf80846 --- /dev/null +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -0,0 +1,53 @@ +-module(emqx_retainer_schema). + +-include_lib("typerefl/include/types.hrl"). + +-export([structs/0, fields/1]). + +-define(TYPE(Type), hoconsc:t(Type)). + +structs() -> ["emqx_retainer"]. + +fields("emqx_retainer") -> + [ {enable, t(boolean(), false)} + , {msg_expiry_interval, t(emqx_schema:duration_ms(), "0s")} + , {msg_clear_interval, t(emqx_schema:duration_ms(), "0s")} + , {connector, connector()} + , {flow_control, ?TYPE(hoconsc:ref(?MODULE, flow_control))} + , {max_payload_size, t(emqx_schema:bytesize(), "1MB")} + ]; + +fields(mnesia_connector) -> + [ {type, ?TYPE(hoconsc:union([mnesia]))} + , {config, ?TYPE(hoconsc:ref(?MODULE, mnesia_connector_cfg))} + ]; + +fields(mnesia_connector_cfg) -> + [ {storage_type, t(hoconsc:union([ram, disc, disc_only]), ram)} + , {max_retained_messages, t(integer(), 0, fun is_pos_integer/1)} + ]; + +fields(flow_control) -> + [ {max_read_number, t(integer(), 0, fun is_pos_integer/1)} + , {msg_deliver_quota, t(integer(), 0, fun is_pos_integer/1)} + , {quota_release_interval, t(emqx_schema:duration_ms(), "0ms")} + ]. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +t(Type, Default) -> + hoconsc:t(Type, #{default => Default}). + +t(Type, Default, Validator) -> + hoconsc:t(Type, #{default => Default, + validator => Validator}). + +union_array(Item) when is_list(Item) -> + hoconsc:array(hoconsc:union(Item)). + +is_pos_integer(V) -> + V >= 0. + +connector() -> + #{type => union_array([hoconsc:ref(?MODULE, mnesia_connector)])}. diff --git a/apps/emqx_retainer/src/emqx_retainer_sup.erl b/apps/emqx_retainer/src/emqx_retainer_sup.erl index fef245489..3811ed8f2 100644 --- a/apps/emqx_retainer/src/emqx_retainer_sup.erl +++ b/apps/emqx_retainer/src/emqx_retainer_sup.erl @@ -18,19 +18,21 @@ -behaviour(supervisor). --export([start_link/1]). +-export([start_link/0]). -export([init/1]). -start_link(Env) -> - supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]). +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). -init([Env]) -> - {ok, {{one_for_one, 10, 3600}, +init([]) -> + PoolSpec = emqx_pool_sup:spec([emqx_retainer_pool, random, emqx_vm:schedulers(), + {emqx_retainer_pool, start_link, []}]), + {ok, {{one_for_one, 10, 3600}, [#{id => retainer, - start => {emqx_retainer, start_link, [Env]}, + start => {emqx_retainer, start_link, []}, restart => permanent, shutdown => 5000, type => worker, - modules => [emqx_retainer]}]}}. - + modules => [emqx_retainer]}, + PoolSpec]}}. diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl index 1df042dd9..de2481580 100644 --- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl @@ -19,7 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --define(APP, emqx). +-define(APP, emqx_retainer). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -31,24 +31,51 @@ all() -> emqx_ct:all(?MODULE). %%-------------------------------------------------------------------- init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_retainer]), + application:stop(emqx_retainer), + emqx_ct_helpers:start_apps([emqx_retainer], fun set_special_configs/1), Config. end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([emqx_retainer]). init_per_testcase(TestCase, Config) -> - emqx_retainer:clean(<<"#">>), - case TestCase of - t_message_expiry_2 -> - application:set_env(emqx_retainer, expiry_interval, 2000); - _ -> - application:set_env(emqx_retainer, expiry_interval, 0) - end, - application:stop(emqx_retainer), + emqx_retainer:clean(), + DefaultCfg = new_emqx_retainer_conf(), + NewCfg = case TestCase of + t_message_expiry_2 -> + DefaultCfg#{msg_expiry_interval := 2000}; + t_flow_control -> + DefaultCfg#{flow_control := #{max_read_number => 1, + msg_deliver_quota => 1, + quota_release_interval => timer:seconds(1)}}; + _ -> + DefaultCfg + end, + emqx_retainer:update_config(NewCfg), application:ensure_all_started(emqx_retainer), Config. +set_special_configs(emqx_retainer) -> + init_emqx_retainer_conf(); +set_special_configs(_) -> + ok. + +init_emqx_retainer_conf() -> + emqx_config:put([?APP], new_emqx_retainer_conf()). + +new_emqx_retainer_conf() -> + #{enable => true, + msg_expiry_interval => 0, + msg_clear_interval => 0, + connector => [#{type => mnesia, + config => + #{max_retained_messages => 0, + storage_type => ram}}], + flow_control => #{max_read_number => 0, + msg_deliver_quota => 0, + quota_release_interval => 0}, + max_payload_size => 1024 * 1024}. + %%-------------------------------------------------------------------- %% Test Cases %%-------------------------------------------------------------------- @@ -75,18 +102,35 @@ t_retain_handling(_) -> {ok, C1} = emqtt:start_link([{clean_start, true}, {proto_ver, v5}]), {ok, _} = emqtt:connect(C1), + %% rh = 0, no wildcard, and with empty retained message + {ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained">>, [{qos, 0}, {rh, 0}]), + ?assertEqual(0, length(receive_messages(1))), + {ok, #{}, [0]} = emqtt:unsubscribe(C1, <<"retained">>), + + %% rh = 0, has wildcard, and with empty retained message + {ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained/#">>, [{qos, 0}, {rh, 0}]), + ?assertEqual(0, length(receive_messages(1))), + {ok, #{}, [0]} = emqtt:unsubscribe(C1, <<"retained/#">>), + emqtt:publish(C1, <<"retained">>, <<"this is a retained message">>, [{qos, 0}, {retain, true}]), {ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained">>, [{qos, 0}, {rh, 0}]), ?assertEqual(1, length(receive_messages(1))), + {ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained">>, [{qos, 0}, {rh, 0}]), ?assertEqual(1, length(receive_messages(1))), + + {ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained/#">>, [{qos, 0}, {rh, 0}]), + ?assertEqual(1, length(receive_messages(1))), + {ok, #{}, [0]} = emqtt:unsubscribe(C1, <<"retained">>), {ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained">>, [{qos, 0}, {rh, 1}]), ?assertEqual(1, length(receive_messages(1))), + {ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained">>, [{qos, 0}, {rh, 1}]), ?assertEqual(0, length(receive_messages(1))), + {ok, #{}, [0]} = emqtt:unsubscribe(C1, <<"retained">>), {ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained">>, [{qos, 0}, {rh, 2}]), @@ -162,17 +206,34 @@ t_clean(_) -> emqtt:publish(C1, <<"retained/0">>, <<"this is a retained message 0">>, [{qos, 0}, {retain, true}]), emqtt:publish(C1, <<"retained/1">>, <<"this is a retained message 1">>, [{qos, 0}, {retain, true}]), emqtt:publish(C1, <<"retained/test/0">>, <<"this is a retained message 2">>, [{qos, 0}, {retain, true}]), - {ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained/#">>, [{qos, 0}, {rh, 0}]), ?assertEqual(3, length(receive_messages(3))), - 1 = emqx_retainer:clean(<<"retained/test/0">>), - 2 = emqx_retainer:clean(<<"retained/+">>), + ok = emqx_retainer:delete(<<"retained/test/0">>), + ok = emqx_retainer:delete(<<"retained/+">>), {ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained/#">>, [{qos, 0}, {rh, 0}]), ?assertEqual(0, length(receive_messages(3))), ok = emqtt:disconnect(C1). +t_flow_control(_) -> + {ok, C1} = emqtt:start_link([{clean_start, true}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(C1), + emqtt:publish(C1, <<"retained/0">>, <<"this is a retained message 0">>, [{qos, 0}, {retain, true}]), + emqtt:publish(C1, <<"retained/1">>, <<"this is a retained message 1">>, [{qos, 0}, {retain, true}]), + emqtt:publish(C1, <<"retained/3">>, <<"this is a retained message 3">>, [{qos, 0}, {retain, true}]), + Begin = erlang:system_time(millisecond), + {ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained/#">>, [{qos, 0}, {rh, 0}]), + ?assertEqual(3, length(receive_messages(3))), + End = erlang:system_time(millisecond), + Diff = End - Begin, + + %% msg_deliver_quota = 1 and quota_release_interval = 1, and there has three message + %% so total wait time is between in 1 ~ 2s(may be timer will delay, so plus 0.5s to maximum) + ?assert(Diff > timer:seconds(1) andalso Diff < timer:seconds(2.5)), + + ok = emqtt:disconnect(C1). + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- @@ -192,4 +253,3 @@ receive_messages(Count, Msgs) -> after 2000 -> Msgs end. - diff --git a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl new file mode 100644 index 000000000..6ce64ae2e --- /dev/null +++ b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl @@ -0,0 +1,152 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_retainer_api_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include("emqx_retainer.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-import(emqx_ct_http, [ request_api/3 + , request_api/5 + , get_http_data/1 + , create_default_app/0 + , delete_default_app/0 + , default_auth_header/0 + ]). + +-define(HOST, "http://127.0.0.1:8081/"). +-define(API_VERSION, "v4"). +-define(BASE_PATH, "api"). + +all() -> +%% TODO: V5 API +%% emqx_ct:all(?MODULE). + []. + +groups() -> + []. + +init_per_suite(Config) -> + application:stop(emqx_retainer), + emqx_ct_helpers:start_apps([emqx_retainer, emqx_management], fun set_special_configs/1), + create_default_app(), + Config. + +end_per_suite(_Config) -> + delete_default_app(), + emqx_ct_helpers:stop_apps([emqx_management, emqx_retainer]). + +init_per_testcase(_, Config) -> + Config. + +set_special_configs(emqx_retainer) -> + emqx_retainer_SUITE:init_emqx_retainer_conf(); +set_special_configs(emqx_management) -> + emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + applications =>[#{id => "admin", secret => "public"}]}), + ok; +set_special_configs(_) -> + ok. + +%%------------------------------------------------------------------------------ +%% Test Cases +%%------------------------------------------------------------------------------ + +t_config(_Config) -> + {ok, Return} = request_http_rest_lookup(["retainer"]), + NowCfg = get_http_data(Return), + NewCfg = NowCfg#{<<"msg_expiry_interval">> => timer:seconds(60)}, + RetainerConf = #{<<"emqx_retainer">> => NewCfg}, + + {ok, _} = request_http_rest_update(["retainer?action=test"], RetainerConf), + {ok, TestReturn} = request_http_rest_lookup(["retainer"]), + ?assertEqual(NowCfg, get_http_data(TestReturn)), + + {ok, _} = request_http_rest_update(["retainer"], RetainerConf), + {ok, UpdateReturn} = request_http_rest_lookup(["retainer"]), + ?assertEqual(NewCfg, get_http_data(UpdateReturn)), + ok. + +t_enable_disable(_Config) -> + Conf = switch_emqx_retainer(undefined, true), + + {ok, C1} = emqtt:start_link([{clean_start, true}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(C1), + + emqtt:publish(C1, <<"retained">>, <<"this is a retained message">>, [{qos, 0}, {retain, true}]), + timer:sleep(100), + + {ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained">>, [{qos, 0}, {rh, 0}]), + ?assertEqual(1, length(receive_messages(1))), + + _ = switch_emqx_retainer(Conf, false), + + {ok, #{}, [0]} = emqtt:unsubscribe(C1, <<"retained">>), + emqtt:publish(C1, <<"retained">>, <<"this is a retained message">>, [{qos, 0}, {retain, true}]), + timer:sleep(100), + {ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained">>, [{qos, 0}, {rh, 0}]), + ?assertEqual(0, length(receive_messages(1))), + + ok = emqtt:disconnect(C1). + +%%-------------------------------------------------------------------- +%% HTTP Request +%%-------------------------------------------------------------------- +request_http_rest_lookup(Path) -> + request_api(get, uri([Path]), default_auth_header()). + +request_http_rest_update(Path, Params) -> + request_api(put, uri([Path]), [], default_auth_header(), Params). + +uri(Parts) when is_list(Parts) -> + NParts = [b2l(E) || E <- Parts], + ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION | NParts]). + +%% @private +b2l(B) when is_binary(B) -> + binary_to_list(B); +b2l(L) when is_list(L) -> + L. + +receive_messages(Count) -> + receive_messages(Count, []). +receive_messages(0, Msgs) -> + Msgs; +receive_messages(Count, Msgs) -> + receive + {publish, Msg} -> + ct:log("Msg: ~p ~n", [Msg]), + receive_messages(Count-1, [Msg|Msgs]); + Other -> + ct:log("Other Msg: ~p~n",[Other]), + receive_messages(Count, Msgs) + after 2000 -> + Msgs + end. + +switch_emqx_retainer(undefined, IsEnable) -> + {ok, Return} = request_http_rest_lookup(["retainer"]), + NowCfg = get_http_data(Return), + switch_emqx_retainer(NowCfg, IsEnable); + +switch_emqx_retainer(NowCfg, IsEnable) -> + NewCfg = NowCfg#{<<"enable">> => IsEnable}, + RetainerConf = #{<<"emqx_retainer">> => NewCfg}, + {ok, _} = request_http_rest_update(["retainer"], RetainerConf), + NewCfg. diff --git a/apps/emqx_retainer/test/mqtt_protocol_v5_SUITE.erl b/apps/emqx_retainer/test/mqtt_protocol_v5_SUITE.erl index cec492c6a..70c8a0554 100644 --- a/apps/emqx_retainer/test/mqtt_protocol_v5_SUITE.erl +++ b/apps/emqx_retainer/test/mqtt_protocol_v5_SUITE.erl @@ -27,7 +27,7 @@ init_per_suite(Config) -> %% Meck emqtt ok = meck:new(emqtt, [non_strict, passthrough, no_history, no_link]), %% Start Apps - emqx_ct_helpers:start_apps([emqx_retainer]), + emqx_ct_helpers:start_apps([emqx_retainer], fun set_special_configs/1), Config. end_per_suite(_Config) -> @@ -37,6 +37,10 @@ end_per_suite(_Config) -> %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- +set_special_configs(emqx_retainer) -> + emqx_retainer_SUITE:init_emqx_retainer_conf(); +set_special_configs(_) -> + ok. client_info(Key, Client) -> maps:get(Key, maps:from_list(emqtt:info(Client)), undefined). diff --git a/apps/emqx_rule_actions/README.md b/apps/emqx_rule_actions/README.md new file mode 100644 index 000000000..c17e1a34a --- /dev/null +++ b/apps/emqx_rule_actions/README.md @@ -0,0 +1,11 @@ +# emqx_rule_actions + +This project contains a collection of rule actions/resources. It is mainly for + making unit test easier. Also it's easier for us to create utils that many + modules depends on it. + +## Build +----- + + $ rebar3 compile + diff --git a/apps/emqx_sn/rebar.config b/apps/emqx_rule_actions/rebar.config similarity index 58% rename from apps/emqx_sn/rebar.config rename to apps/emqx_rule_actions/rebar.config index cbdac78f6..097c18a3d 100644 --- a/apps/emqx_sn/rebar.config +++ b/apps/emqx_rule_actions/rebar.config @@ -1,26 +1,25 @@ {deps, []}. -{plugins, [rebar3_proper]}. -{deps, - [{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.7.4"}}} - ]}. - -{edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, warn_shadow_vars, warn_unused_import, warn_obsolete_guard, - debug_info, - {parse_transform}]}. - -{dialyzer, [{warnings, [unmatched_returns, error_handling, race_conditions]} + no_debug_info, + compressed, %% for edge + {parse_transform} ]}. +{overrides, [{add, [{erl_opts, [no_debug_info, compressed]}]}]}. + +{edoc_opts, [{preprocess, true}]}. + {xref_checks, [undefined_function_calls, undefined_functions, locals_not_used, deprecated_function_calls, - warnings_as_errors, deprecated_functions]}. + warnings_as_errors, deprecated_functions + ]}. + {cover_enabled, true}. {cover_opts, [verbose]}. {cover_export_enabled, true}. -{plugins, [coveralls]}. +{plugins, [rebar3_proper]}. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl b/apps/emqx_rule_actions/src/emqx_bridge_mqtt_actions.erl similarity index 97% rename from apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl rename to apps/emqx_rule_actions/src/emqx_bridge_mqtt_actions.erl index 3f685a72a..8d17ee6f5 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl +++ b/apps/emqx_rule_actions/src/emqx_bridge_mqtt_actions.erl @@ -119,10 +119,9 @@ }, disk_cache => #{ order => 8, - type => string, + type => boolean, required => false, - default => <<"off">>, - enum => [<<"on">>, <<"off">>], + default => false, title => #{en => <<"Disk Cache">>, zh => <<"磁盘缓存"/utf8>>}, description => #{en => <<"The flag which determines whether messages " @@ -300,10 +299,9 @@ }, disk_cache => #{ order => 6, - type => string, + type => boolean, required => false, - default => <<"off">>, - enum => [<<"on">>, <<"off">>], + default => false, title => #{en => <<"Disk Cache">>, zh => <<"磁盘缓存"/utf8>>}, description => #{en => <<"The flag which determines whether messages " @@ -508,7 +506,7 @@ connect(Options) when is_list(Options) -> connect(Options = #{disk_cache := DiskCache, ecpool_worker_id := Id, pool_name := Pool}) -> Options0 = case DiskCache of true -> - DataDir = filename:join([emqx:get_env(data_dir), replayq, Pool, integer_to_list(Id)]), + DataDir = filename:join([emqx_config:get([node, data_dir]), replayq, Pool, integer_to_list(Id)]), QueueOption = #{replayq_dir => DataDir}, Options#{queue => QueueOption}; false -> @@ -526,7 +524,7 @@ connect(Options = #{disk_cache := DiskCache, ecpool_worker_id := Id, pool_name : end end, Options2 = maps:without([ecpool_worker_id, pool_name, append], Options1), - emqx_bridge_worker:start_link(name(Pool, Id), Options2). + emqx_bridge_worker:start_link(Options2#{name => name(Pool, Id)}). name(Pool, Id) -> list_to_atom(atom_to_list(Pool) ++ ":" ++ integer_to_list(Id)). pool_name(ResId) -> @@ -538,9 +536,9 @@ options(Options, PoolName, ResId) -> Address = Get(<<"address">>), [{max_inflight_batches, 32}, {forward_mountpoint, str(Get(<<"mountpoint">>))}, - {disk_cache, cuttlefish_flag:parse(str(GetD(<<"disk_cache">>, "off")))}, + {disk_cache, GetD(<<"disk_cache">>, false)}, {start_type, auto}, - {reconnect_delay_ms, cuttlefish_duration:parse(str(Get(<<"reconnect_interval">>)), ms)}, + {reconnect_delay_ms, hocon_postprocess:duration(str(Get(<<"reconnect_interval">>)))}, {if_record_metrics, false}, {pool_size, GetD(<<"pool_size">>, 1)}, {pool_name, PoolName} @@ -556,11 +554,11 @@ options(Options, PoolName, ResId) -> {clientid, str(Get(<<"clientid">>))}, {append, Get(<<"append">>)}, {connect_module, emqx_bridge_mqtt}, - {keepalive, cuttlefish_duration:parse(str(Get(<<"keepalive">>)), s)}, + {keepalive, hocon_postprocess:duration(str(Get(<<"keepalive">>))) div 1000}, {username, str(Get(<<"username">>))}, {password, str(Get(<<"password">>))}, {proto_ver, mqtt_ver(Get(<<"proto_ver">>))}, - {retry_interval, cuttlefish_duration:parse(str(GetD(<<"retry_interval">>, "30s")), s)} + {retry_interval, hocon_postprocess:duration(str(GetD(<<"retry_interval">>, "30s"))) div 1000} | maybe_ssl(Options, Get(<<"ssl">>), ResId)] end. diff --git a/apps/emqx_rule_actions/src/emqx_rule_actions.app.src b/apps/emqx_rule_actions/src/emqx_rule_actions.app.src new file mode 100644 index 000000000..fd95c3572 --- /dev/null +++ b/apps/emqx_rule_actions/src/emqx_rule_actions.app.src @@ -0,0 +1,11 @@ +{application, emqx_rule_actions, + [{description, "Rule actions"}, + {vsn, "5.0.0"}, + {registered, []}, + {applications, + [kernel,stdlib]}, + {env,[]}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_rule_actions/src/emqx_web_hook_actions.erl similarity index 98% rename from apps/emqx_web_hook/src/emqx_web_hook_actions.erl rename to apps/emqx_rule_actions/src/emqx_web_hook_actions.erl index 8bb1c7de0..ef21ab864 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_rule_actions/src/emqx_web_hook_actions.erl @@ -295,7 +295,7 @@ parse_action_params(Params = #{<<"url">> := URL}) -> path => merge_path(CommonPath, maps:get(<<"path">>, Params, <<>>)), headers => NHeaders, body => maps:get(<<"body">>, Params, <<>>), - request_timeout => cuttlefish_duration:parse(str(maps:get(<<"request_timeout">>, Params, <<"5s">>))), + request_timeout => hocon_postprocess:duration(str(maps:get(<<"request_timeout">>, Params, <<"5s">>))), pool => maps:get(<<"pool">>, Params)} catch _:_ -> throw({invalid_params, Params}) @@ -338,7 +338,7 @@ pool_opts(Params = #{<<"url">> := URL}, ResId) -> scheme := Scheme}} = emqx_http_lib:uri_parse(URL), PoolSize = maps:get(<<"pool_size">>, Params, 32), ConnectTimeout = - cuttlefish_duration:parse(str(maps:get(<<"connect_timeout">>, Params, <<"5s">>))), + hocon_postprocess:duration(str(maps:get(<<"connect_timeout">>, Params, <<"5s">>))), TransportOpts0 = case Scheme =:= https of true -> [get_ssl_opts(Params, ResId)]; diff --git a/apps/emqx_rule_engine/etc/emqx_rule_engine.conf b/apps/emqx_rule_engine/etc/emqx_rule_engine.conf index 556c59970..c1637d66d 100644 --- a/apps/emqx_rule_engine/etc/emqx_rule_engine.conf +++ b/apps/emqx_rule_engine/etc/emqx_rule_engine.conf @@ -1,42 +1,6 @@ ##==================================================================== -## Rule Engine for EMQ X R4.0 +## Rule Engine for EMQ X R5.0 ##==================================================================== - -rule_engine.ignore_sys_message = on - -## Event Messages -## -## If enabled (on), rule engine publishes the event as an MQTT message -## with topic='$events/' on the occurrence of an emqx event. -## -## If disabled, rule engine stops publishing the event messages, but -## the event message can still be processed by the rule SQL. e.g. rule SQL: -## -## SELECT * FROM "$events/client_connected" -## -## will still work even if 'rule_engine.events.client_connected' is set to 'off' -## -## EMQ Event to event message mapping: -## -## - client.connected -> $events/client_connected -## - client.disconnected -> $events/client_disconnected -## - session.subscribed -> $events/session_subscribed -## - session.unsubscribed -> $events/session_unsubscribed -## - message.delivered -> $events/message_delivered -## - message.acked -> $events/message_acked -## - message.dropped -> $events/message_dropped -## -## Config Value Format: Toggle, QoS-Level -## -## Toggle: on/off -## -## QoS-Level: qos0/qos1/qos2 - -#rule_engine.events.client_connected = "on, qos1" -rule_engine.events.client_connected = off -rule_engine.events.client_disconnected = off -rule_engine.events.session_subscribed = off -rule_engine.events.session_unsubscribed = off -rule_engine.events.message_delivered = off -rule_engine.events.message_acked = off -rule_engine.events.message_dropped = off +emqx_rule_engine:{ + ignore_sys_message: true +} diff --git a/apps/emqx_rule_engine/priv/emqx_rule_engine.schema b/apps/emqx_rule_engine/priv/emqx_rule_engine.schema deleted file mode 100644 index c5549aa36..000000000 --- a/apps/emqx_rule_engine/priv/emqx_rule_engine.schema +++ /dev/null @@ -1,61 +0,0 @@ -%%-*- mode: erlang -*- -%% emqx_rule_engine config mapping - -{mapping, "rule_engine.ignore_sys_message", "emqx_rule_engine.ignore_sys_message", [ - {default, on}, - {datatype, flag} -]}. - -{mapping, "rule_engine.events.$name", "emqx_rule_engine.events", [ - {default, "off, qos1"}, - {datatype, string} -]}. - -{translation, "emqx_rule_engine.events", fun(Conf) -> - SupportedHooks = - [ 'client.connected' - , 'client.disconnected' - , 'session.subscribed' - , 'session.unsubscribed' - , 'message.delivered' - , 'message.acked' - , 'message.dropped' - ], - - HookPoint = fun(Event) -> - case string:split(Event, "_") of - [Prefix, Name] -> - Point = list_to_atom(lists:append([Prefix, ".", Name])), - case lists:member(Point, SupportedHooks) of - true -> Point; - false -> error({unsupported_event, Event}) - end; - [_] -> - error({invalid_event, Event}) - end - end, - - QoS = fun ("qos"++Level = QoSLevel) -> - case list_to_integer(Level) of - QoSL when QoSL =:= 0; QoSL =:= 1; QoSL =:= 2 -> - QoSL; - _ -> - error({invalid_qos_level, QoSLevel}) - end; - (QoSLevel) -> - error({invalid_qos, QoSLevel}) - end, - - lists:foldl( - fun({EE=[_,"events",EvtName], Val}, Acc) -> - case string:split(string:trim(Val), ",", all) of - ["on"++_, Snd] -> - [{HookPoint(EvtName), on, QoS(string:trim(Snd))} | Acc]; - ["on"++_] -> - [{HookPoint(EvtName), on, 1} | Acc]; - [_] -> - Acc - end; - ({_, _}, Acc) -> Acc - end, [], cuttlefish_variable:filter_by_prefix("rule_engine.events", Conf)) -end}. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index aebb73150..ff25dcfd3 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.3"}, % strict semver, bump manually! + {vsn, "5.0.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src deleted file mode 100644 index 01c07c124..000000000 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ /dev/null @@ -1,38 +0,0 @@ -%% -*-: erlang -*- -{"4.3.3", - [ {"4.3.0", - [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []} - , {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} - , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} - , {apply, {emqx_stats, cancel_update, [rule_registery_stats]}} - ]}, - {"4.3.1", - [ {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} - , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} - , {apply, {emqx_stats, cancel_update, [rule_registery_stats]}} - ]}, - {"4.3.2", - [ {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} - , {apply, {emqx_stats, cancel_update, [rule_registery_stats]}} - ]}, - {<<".*">>, []} - ], - [ - {"4.3.0", - [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []} - , {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} - , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} - , {apply, {emqx_stats, cancel_update, [rule_registery_stats]}} - ]}, - {"4.3.1", - [ {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} - , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} - , {apply, {emqx_stats, cancel_update, [rule_registery_stats]}} - ]}, - {"4.3.2", - [ {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} - , {apply, {emqx_stats, cancel_update, [rule_registery_stats]}} - ]}, - {<<".*">>, []} - ] -}. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 4fa3b8aa3..24b4d2c13 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -21,8 +21,6 @@ -logger_header("[RuleEngineAPI]"). --import(minirest, [return/1]). - -rest_api(#{name => create_rule, method => 'POST', path => "/rules/", @@ -552,3 +550,6 @@ get_rule_metrics(Id) -> get_action_metrics(Id) -> [maps:put(node, Node, rpc:call(Node, emqx_rule_metrics, get_action_metrics, [Id])) || Node <- ekka_mnesia:running_nodes()]. + +%% TODO: V5 API +return(_) -> ok. \ No newline at end of file diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl index e00d717d1..5893f9827 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl @@ -18,8 +18,6 @@ -behaviour(application). --emqx_plugin(?MODULE). - -export([start/2]). -export([stop/1]). diff --git a/apps/emqx_psk_file/src/emqx_psk_file_sup.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl similarity index 73% rename from apps/emqx_psk_file/src/emqx_psk_file_sup.erl rename to apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 041eecdb6..f7658c208 100644 --- a/apps/emqx_psk_file/src/emqx_psk_file_sup.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -14,19 +14,16 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_psk_file_sup). +-module(emqx_rule_engine_schema). --behaviour(supervisor). +-include_lib("typerefl/include/types.hrl"). -%% API --export([start_link/0]). +-behaviour(hocon_schema). -%% Supervisor callbacks --export([init/1]). +-export([ structs/0 + , fields/1]). -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -init([]) -> - {ok, { {one_for_one, 0, 1}, []} }. +structs() -> ["emqx_rule_engine"]. +fields("emqx_rule_engine") -> + [{ignore_sys_message, emqx_schema:t(boolean(), undefined, true)}]. diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index 97e40439d..5865f224c 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -64,8 +64,7 @@ load(Topic) -> HookPoint = event_name(Topic), - emqx_hooks:put(HookPoint, {?MODULE, hook_fun(HookPoint), - [hook_conf(HookPoint, env())]}). + emqx_hooks:put(HookPoint, {?MODULE, hook_fun(HookPoint), [[]]}). unload() -> lists:foreach(fun(HookPoint) -> @@ -76,64 +75,62 @@ unload(Topic) -> HookPoint = event_name(Topic), emqx_hooks:del(HookPoint, {?MODULE, hook_fun(HookPoint)}). -env() -> - application:get_all_env(?APP). - %%-------------------------------------------------------------------- %% Callbacks %%-------------------------------------------------------------------- - -on_message_publish(Message = #message{flags = #{event := true}}, - _Env) -> - {ok, Message}; -on_message_publish(Message = #message{flags = #{sys := true}}, - #{ignore_sys_message := true}) -> - {ok, Message}; on_message_publish(Message = #message{topic = Topic}, _Env) -> - case emqx_rule_registry:get_rules_for(Topic) of - [] -> ok; - Rules -> emqx_rule_runtime:apply_rules(Rules, eventmsg_publish(Message)) + case ignore_sys_message(Message) of + true -> + ok; + false -> + case emqx_rule_registry:get_rules_for(Topic) of + [] -> ok; + Rules -> emqx_rule_runtime:apply_rules(Rules, eventmsg_publish(Message)) + end end, {ok, Message}. on_client_connected(ClientInfo, ConnInfo, Env) -> - may_publish_and_apply('client.connected', + apply_event('client.connected', fun() -> eventmsg_connected(ClientInfo, ConnInfo) end, Env). on_client_disconnected(ClientInfo, Reason, ConnInfo, Env) -> - may_publish_and_apply('client.disconnected', + apply_event('client.disconnected', fun() -> eventmsg_disconnected(ClientInfo, ConnInfo, Reason) end, Env). on_session_subscribed(ClientInfo, Topic, SubOpts, Env) -> - may_publish_and_apply('session.subscribed', + apply_event('session.subscribed', fun() -> eventmsg_sub_or_unsub('session.subscribed', ClientInfo, Topic, SubOpts) end, Env). on_session_unsubscribed(ClientInfo, Topic, SubOpts, Env) -> - may_publish_and_apply('session.unsubscribed', + apply_event('session.unsubscribed', fun() -> eventmsg_sub_or_unsub('session.unsubscribed', ClientInfo, Topic, SubOpts) end, Env). -on_message_dropped(Message = #message{flags = #{sys := true}}, - _, _, #{ignore_sys_message := true}) -> - {ok, Message}; on_message_dropped(Message, _, Reason, Env) -> - may_publish_and_apply('message.dropped', - fun() -> eventmsg_dropped(Message, Reason) end, Env), + case ignore_sys_message(Message) of + true -> ok; + false -> + apply_event('message.dropped', + fun() -> eventmsg_dropped(Message, Reason) end, Env) + end, {ok, Message}. -on_message_delivered(_ClientInfo, Message = #message{flags = #{sys := true}}, - #{ignore_sys_message := true}) -> - {ok, Message}; on_message_delivered(ClientInfo, Message, Env) -> - may_publish_and_apply('message.delivered', - fun() -> eventmsg_delivered(ClientInfo, Message) end, Env), + case ignore_sys_message(Message) of + true -> ok; + false -> + apply_event('message.delivered', + fun() -> eventmsg_delivered(ClientInfo, Message) end, Env) + end, {ok, Message}. -on_message_acked(_ClientInfo, Message = #message{flags = #{sys := true}}, - #{ignore_sys_message := true}) -> - {ok, Message}; on_message_acked(ClientInfo, Message, Env) -> - may_publish_and_apply('message.acked', - fun() -> eventmsg_acked(ClientInfo, Message) end, Env), + case ignore_sys_message(Message) of + true -> ok; + false -> + apply_event('message.acked', + fun() -> eventmsg_acked(ClientInfo, Message) end, Env) + end, {ok, Message}. %%-------------------------------------------------------------------- @@ -185,7 +182,7 @@ eventmsg_connected(_ClientInfo = #{ keepalive => Keepalive, clean_start => CleanStart, receive_maximum => RcvMax, - expiry_interval => ExpiryInterval, + expiry_interval => ExpiryInterval div 1000, is_bridge => IsBridge, conn_props => printable_maps(ConnProps), connected_at => ConnectedAt @@ -297,31 +294,15 @@ with_basic_columns(EventName, Data) when is_map(Data) -> }. %%-------------------------------------------------------------------- -%% Events publishing and rules applying +%% rules applying %%-------------------------------------------------------------------- - -may_publish_and_apply(EventName, GenEventMsg, #{enabled := true, qos := QoS}) -> - EventTopic = event_topic(EventName), - EventMsg = GenEventMsg(), - case emqx_json:safe_encode(EventMsg) of - {ok, Payload} -> - _ = emqx_broker:safe_publish(make_msg(QoS, EventTopic, Payload)), - ok; - {error, _Reason} -> - ?LOG(error, "Failed to encode event msg for ~p, msg: ~p", [EventName, EventMsg]) - end, - emqx_rule_runtime:apply_rules(emqx_rule_registry:get_rules_for(EventTopic), EventMsg); -may_publish_and_apply(EventName, GenEventMsg, _Env) -> +apply_event(EventName, GenEventMsg, _Env) -> EventTopic = event_topic(EventName), case emqx_rule_registry:get_rules_for(EventTopic) of [] -> ok; Rules -> emqx_rule_runtime:apply_rules(Rules, GenEventMsg()) end. -make_msg(QoS, Topic, Payload) -> - emqx_message:set_flags(#{sys => true, event => true}, - emqx_message:make(emqx_events, QoS, Topic, iolist_to_binary(Payload))). - %%-------------------------------------------------------------------- %% Columns %%-------------------------------------------------------------------- @@ -559,14 +540,6 @@ columns_with_exam('session.unsubscribed') -> %% Helper functions %%-------------------------------------------------------------------- -hook_conf(HookPoint, Env) -> - Events = proplists:get_value(events, Env, []), - IgnoreSys = proplists:get_value(ignore_sys_message, Env, true), - case lists:keyfind(HookPoint, 1, Events) of - {_, on, QoS} -> #{enabled => true, qos => QoS, ignore_sys_message => IgnoreSys}; - _ -> #{enabled => false, qos => 1, ignore_sys_message => IgnoreSys} - end. - hook_fun(Event) -> case string:split(atom_to_list(Event), ".") of [Prefix, Name] -> @@ -620,3 +593,7 @@ printable_maps(Headers) -> }; (K, V0, AccIn) -> AccIn#{K => V0} end, #{}, Headers). + +ignore_sys_message(#message{flags = Flags}) -> + maps:get(sys, Flags, false) andalso + emqx_config:get([emqx_rule_engine, ignore_sys_message]). diff --git a/apps/emqx_rule_engine/src/emqx_rule_registry.erl b/apps/emqx_rule_engine/src/emqx_rule_registry.erl index 2d029f8e3..f2d717dba 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_registry.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_registry.erl @@ -19,6 +19,7 @@ -behaviour(gen_server). -include("rule_engine.hrl"). +-include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("stdlib/include/qlc.hrl"). @@ -95,6 +96,11 @@ -define(T_CALL, 10000). +-rlog_shard({?RULE_ENGINE_SHARD, ?RULE_TAB}). +-rlog_shard({?RULE_ENGINE_SHARD, ?ACTION_TAB}). +-rlog_shard({?RULE_ENGINE_SHARD, ?RES_TAB}). +-rlog_shard({?RULE_ENGINE_SHARD, ?RES_TYPE_TAB}). + %%------------------------------------------------------------------------------ %% Mnesia bootstrap %%------------------------------------------------------------------------------ @@ -174,7 +180,7 @@ get_rules_ordered_by_ts() -> Query = qlc:q([E || E <- mnesia:table(?RULE_TAB)]), qlc:e(qlc:keysort(#rule.created_at, Query, [{order, ascending}])) end, - {atomic, List} = mnesia:transaction(F), + {atomic, List} = ekka_mnesia:transaction(?RULE_ENGINE_SHARD, F), List. -spec(get_rules_for(Topic :: binary()) -> list(emqx_rule_engine:rule())). @@ -471,11 +477,18 @@ code_change(_OldVsn, State, _Extra) -> get_all_records(Tab) -> %mnesia:dirty_match_object(Tab, mnesia:table_info(Tab, wild_pattern)). - ets:tab2list(Tab). + %% Wrapping ets to a r/o transaction to avoid reading inconsistent + %% data during shard bootstrap + {atomic, Ret} = + ekka_mnesia:ro_transaction(?RULE_ENGINE_SHARD, + fun() -> + ets:tab2list(Tab) + end), + Ret. trans(Fun) -> trans(Fun, []). trans(Fun, Args) -> - case mnesia:transaction(Fun, Args) of + case ekka_mnesia:transaction(?RULE_ENGINE_SHARD, Fun, Args) of {atomic, Result} -> Result; {aborted, Reason} -> error(Reason) end. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index d8244c018..a056d0c26 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -54,14 +54,15 @@ groups() -> [t_inspect_action ,t_republish_action ]}, - {api, [], - [t_crud_rule_api, - t_list_actions_api, - t_show_action_api, - t_crud_resources_api, - t_list_resource_types_api, - t_show_resource_type_api - ]}, +%% TODO: V5 API +%% {api, [], +%% [t_crud_rule_api, +%% t_list_actions_api, +%% t_show_action_api, +%% t_crud_resources_api, +%% t_list_resource_types_api, +%% t_show_resource_type_api +%% ]}, {cli, [], [t_rules_cli, t_actions_cli, @@ -149,11 +150,11 @@ groups() -> init_per_suite(Config) -> ok = ekka_mnesia:start(), ok = emqx_rule_registry:mnesia(boot), - start_apps(), + ok = emqx_ct_helpers:start_apps([emqx_rule_engine], fun set_special_configs/1), Config. end_per_suite(_Config) -> - stop_apps(), + emqx_ct_helpers:stop_apps([emqx_rule_engine]), ok. on_resource_create(_id, _) -> #{}. @@ -2545,21 +2546,6 @@ init_events_counters() -> %%------------------------------------------------------------------------------ %% Start Apps %%------------------------------------------------------------------------------ - -stop_apps() -> - stopped = mnesia:stop(), - [application:stop(App) || App <- [emqx_rule_engine, emqx]]. - -start_apps() -> - [start_apps(App, SchemaFile, ConfigFile) || - {App, SchemaFile, ConfigFile} - <- [{emqx, emqx_schema, deps_path(emqx, "etc/emqx.conf")}, - {emqx_rule_engine, local_path("priv/emqx_rule_engine.schema"), - local_path("etc/emqx_rule_engine.conf")}]]. - -start_apps(App, Schema, ConfigFile) -> - emqx_ct_helpers:start_app(App, Schema, ConfigFile, fun set_special_configs/1). - deps_path(App, RelativePath) -> Path0 = code:lib_dir(App), Path = case file:read_link(Path0) of diff --git a/apps/emqx_sn/.formatter.exs b/apps/emqx_sn/.formatter.exs deleted file mode 100644 index d2cda26ed..000000000 --- a/apps/emqx_sn/.formatter.exs +++ /dev/null @@ -1,4 +0,0 @@ -# Used by "mix format" -[ - inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] -] diff --git a/apps/emqx_sn/.gitignore b/apps/emqx_sn/.gitignore deleted file mode 100644 index 46861cdec..000000000 --- a/apps/emqx_sn/.gitignore +++ /dev/null @@ -1,40 +0,0 @@ -.eunit -deps -*.o -*.beam -*.plt -erl_crash.dump -ebin -rel/example_project -.concrete/DEV_MODE -.rebar -_rel/ -emqx_sn.d -logs/ -.erlang.mk/ -data/ -.idea/ -*.iml -*.d -_build/ -.rebar3 -rebar3.crashdump -.DS_Store -bbmustache/ -etc/gen.emqx.conf -cuttlefish -rebar.lock -xrefr -intergration_test/emqx-rel/ -intergration_test/paho.mqtt-sn.embedded-c/ -intergration_test/client/*.exe -intergration_test/client/*.txt -.DS_Store -cover/ -ct.coverdata -eunit.coverdata -test/ct.cover.spec -erlang.mk -etc/emqx_sn.conf.rendered -.rebar3/ -*.swp diff --git a/apps/emqx_sn/etc/emqx_sn.conf b/apps/emqx_sn/etc/emqx_sn.conf deleted file mode 100644 index e05f1e7be..000000000 --- a/apps/emqx_sn/etc/emqx_sn.conf +++ /dev/null @@ -1,53 +0,0 @@ -##-------------------------------------------------------------------- -## MQTT-SN -##-------------------------------------------------------------------- - -## The UDP port which emq-sn is listening on. -## -## Value: IP:Port | Port -## -## Examples: 1884, "127.0.0.1:1884", "::1:1884" -mqtt.sn.port = 1884 - -## The duration that emqx-sn broadcast ADVERTISE message through. -## -## Value: Duration -mqtt.sn.advertise_duration = 15m - -## The MQTT-SN Gateway id in ADVERTISE message. -## -## Value: Number -mqtt.sn.gateway_id = 1 - -## To control whether write statistics data into ETS table for dashbord to read. -## -## Value: on | off -mqtt.sn.enable_stats = off - -## To control whether accept and process the received publish message with qos=-1. -## -## Value: on | off -mqtt.sn.enable_qos3 = off - -## MQTT SN idle timeout, specified in seconds. -## -## Value: Duration -mqtt.sn.idle_timeout = 30s - -## The pre-defined topic name corresponding to the pre-defined topic id of N. -## Note that the pre-defined topic id of 0 is reserved. -mqtt.sn.predefined.topic.0 = reserved -mqtt.sn.predefined.topic.1 = "/predefined/topic/name/hello" -mqtt.sn.predefined.topic.2 = "/predefined/topic/name/nice" - -## Default username for MQTT-SN. This parameter is optional. If specified, -## emq-sn will connect EMQ core with this username. It is useful if any auth -## plug-in is enabled. -## -## Value: String -mqtt.sn.username = mqtt_sn_user - -## This parameter is optional. Pair with username above. -## -## Value: String -mqtt.sn.password = abc diff --git a/apps/emqx_sn/examples/simple_example.erl b/apps/emqx_sn/examples/simple_example.erl deleted file mode 100644 index ce19c4133..000000000 --- a/apps/emqx_sn/examples/simple_example.erl +++ /dev/null @@ -1,126 +0,0 @@ --module(simple_example). - --include("emqx_sn.hrl"). - --define(HOST, {127,0,0,1}). --define(PORT, 1884). - --export([start/0]). - -start() -> - io:format("start to connect ~p:~p~n", [?HOST, ?PORT]), - - %% create udp socket - {ok, Socket} = gen_udp:open(0, [binary]), - - %% connect to emqx_sn broker - Packet = gen_connect_packet(<<"client1">>), - ok = gen_udp:send(Socket, ?HOST, ?PORT, Packet), - io:format("send connect packet=~p~n", [Packet]), - %% receive message - wait_response(), - - %% register topic_id - RegisterPacket = gen_register_packet(<<"TopicA">>, 0), - ok = gen_udp:send(Socket, ?HOST, ?PORT, RegisterPacket), - io:format("send register packet=~p~n", [RegisterPacket]), - TopicId = wait_response(), - - %% subscribe - SubscribePacket = gen_subscribe_packet(TopicId), - ok = gen_udp:send(Socket, ?HOST, ?PORT, SubscribePacket), - io:format("send subscribe packet=~p~n", [SubscribePacket]), - wait_response(), - - %% publish - PublishPacket = gen_publish_packet(TopicId, <<"Payload...">>), - ok = gen_udp:send(Socket, ?HOST, ?PORT, PublishPacket), - io:format("send publish packet=~p~n", [PublishPacket]), - wait_response(), - - % wait for subscribed message from broker - wait_response(), - - %% disconnect from emqx_sn broker - DisConnectPacket = gen_disconnect_packet(), - ok = gen_udp:send(Socket, ?HOST, ?PORT, DisConnectPacket), - io:format("send disconnect packet=~p~n", [DisConnectPacket]). - - - -gen_connect_packet(ClientId) -> - Length = 6+byte_size(ClientId), - MsgType = ?SN_CONNECT, - Dup = 0, - QoS = 0, - Retain = 0, - Will = 0, - CleanSession = 1, - TopicIdType = 0, - Flag = <>, - ProtocolId = 1, - Duration = 10, - <>. - -gen_subscribe_packet(TopicId) -> - Length = 7, - MsgType = ?SN_SUBSCRIBE, - Dup = 0, - Retain = 0, - Will = 0, - QoS = 1, - CleanSession = 0, - TopicIdType = 1, - Flag = <>, - MsgId = 1, - <>. - -gen_register_packet(Topic, TopicId) -> - Length = 6+byte_size(Topic), - MsgType = ?SN_REGISTER, - MsgId = 1, - <>. - -gen_publish_packet(TopicId, Payload) -> - Length = 7+byte_size(Payload), - MsgType = ?SN_PUBLISH, - Dup = 0, - QoS = 1, - Retain = 0, - Will = 0, - CleanSession = 0, - MsgId = 1, - TopicIdType = 1, - Flag = <>, - <>. - -gen_disconnect_packet()-> - Length = 2, - MsgType = ?SN_DISCONNECT, - <>. - -wait_response() -> - receive - {udp, _Socket, _, _, Bin} -> - case Bin of - <<_Len:8, ?SN_PUBLISH, _Flag:8, TopicId:16, MsgId:16, Data/binary>> -> - io:format("recv publish TopicId: ~p, MsgId: ~p, Data: ~p~n", [TopicId, MsgId, Data]); - <<_Len:8, ?SN_CONNACK, 0:8>> -> - io:format("recv connect ack~n"); - <<_Len:8, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>> -> - io:format("recv regack TopicId=~p, MsgId=~p~n", [TopicId, MsgId]), - TopicId; - <<_Len:8, ?SN_SUBACK, Flags:8, TopicId:16, MsgId:16, 0:8>> -> - io:format("recv suback Flags=~p TopicId=~p, MsgId=~p~n", [Flags, TopicId, MsgId]); - <<_Len:8, ?SN_PUBACK, TopicId:16, MsgId:16, 0:8>> -> - io:format("recv puback TopicId=~p, MsgId=~p~n", [TopicId, MsgId]); - _ -> - io:format("ignore bin=~p~n", [Bin]) - end; - Any -> - io:format("recv something else from udp socket ~p~n", [Any]) - after - 2000 -> - io:format("Error: receive timeout!~n"), - wait_response() - end. diff --git a/apps/emqx_sn/examples/simple_example2.erl b/apps/emqx_sn/examples/simple_example2.erl deleted file mode 100644 index b9ada6d22..000000000 --- a/apps/emqx_sn/examples/simple_example2.erl +++ /dev/null @@ -1,120 +0,0 @@ --module(simple_example2). - --include("emqx_sn.hrl"). - --define(HOST, "localhost"). --define(PORT, 1884). - --export([start/0]). - -start() -> - io:format("start to connect ~p:~p~n", [?HOST, ?PORT]), - - %% create udp socket - {ok, Socket} = gen_udp:open(0, [binary]), - - %% connect to emqx_sn broker - Packet = gen_connect_packet(<<"client1">>), - ok = gen_udp:send(Socket, ?HOST, ?PORT, Packet), - io:format("send connect packet=~p~n", [Packet]), - %% receive message - wait_response(), - - %% subscribe, SHORT TOPIC NAME - SubscribePacket = gen_subscribe_packet(<<"T1">>), - ok = gen_udp:send(Socket, ?HOST, ?PORT, SubscribePacket), - io:format("send subscribe packet=~p~n", [SubscribePacket]), - wait_response(), - - %% publish, SHORT TOPIC NAME - PublishPacket = gen_publish_packet(<<"T1">>, <<"Payload...">>), - ok = gen_udp:send(Socket, ?HOST, ?PORT, PublishPacket), - io:format("send publish packet=~p~n", [PublishPacket]), - wait_response(), - - % wait for subscribed message from broker - wait_response(), - - %% disconnect from emqx_sn broker - DisConnectPacket = gen_disconnect_packet(), - ok = gen_udp:send(Socket, ?HOST, ?PORT, DisConnectPacket), - io:format("send disconnect packet=~p~n", [DisConnectPacket]). - - - -gen_connect_packet(ClientId) -> - Length = 6+byte_size(ClientId), - MsgType = ?SN_CONNECT, - Dup = 0, - QoS = 0, - Retain = 0, - Will = 0, - CleanSession = 1, - TopicIdType = 0, - Flag = <>, - ProtocolId = 1, - Duration = 10, - <>. - -gen_subscribe_packet(ShortTopic) -> - Length = 7, - MsgType = ?SN_SUBSCRIBE, - Dup = 0, - Retain = 0, - Will = 0, - QoS = 1, - CleanSession = 0, - TopicIdType = 2, % SHORT TOPIC NAME - Flag = <>, - MsgId = 1, - <>. - -gen_register_packet(Topic, TopicId) -> - Length = 6+byte_size(Topic), - MsgType = ?SN_REGISTER, - MsgId = 1, - <>. - -gen_publish_packet(ShortTopic, Payload) -> - Length = 7+byte_size(Payload), - MsgType = ?SN_PUBLISH, - Dup = 0, - QoS = 1, - Retain = 0, - Will = 0, - CleanSession = 0, - MsgId = 1, - TopicIdType = 2, % SHORT TOPIC NAME - Flag = <>, - <>. - -gen_disconnect_packet()-> - Length = 2, - MsgType = ?SN_DISCONNECT, - <>. - -wait_response() -> - receive - {udp, _Socket, _, _, Bin} -> - case Bin of - <<_Len:8, ?SN_PUBLISH, _Flag:8, TopicId:16, MsgId:16, Data/binary>> -> - io:format("recv publish TopicId: ~p, MsgId: ~p, Data: ~p~n", [TopicId, MsgId, Data]); - <<_Len:8, ?SN_CONNACK, 0:8>> -> - io:format("recv connect ack~n"); - <<_Len:8, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>> -> - io:format("recv regack TopicId=~p, MsgId=~p~n", [TopicId, MsgId]), - TopicId; - <<_Len:8, ?SN_SUBACK, Flags:8, TopicId:16, MsgId:16, 0:8>> -> - io:format("recv suback Flags=~p TopicId=~p, MsgId=~p~n", [Flags, TopicId, MsgId]); - <<_Len:8, ?SN_PUBACK, TopicId:16, MsgId:16, 0:8>> -> - io:format("recv puback TopicId=~p, MsgId=~p~n", [TopicId, MsgId]); - _ -> - io:format("ignore bin=~p~n", [Bin]) - end; - Any -> - io:format("recv something else from udp socket ~p~n", [Any]) - after - 2000 -> - io:format("Error: receive timeout!~n"), - wait_response() - end. diff --git a/apps/emqx_sn/examples/simple_example3.erl b/apps/emqx_sn/examples/simple_example3.erl deleted file mode 100644 index 40f0bf572..000000000 --- a/apps/emqx_sn/examples/simple_example3.erl +++ /dev/null @@ -1,120 +0,0 @@ --module(simple_example3). - --include("emqx_sn.hrl"). - --define(HOST, "localhost"). --define(PORT, 1884). - --export([start/0]). - -start() -> - io:format("start to connect ~p:~p~n", [?HOST, ?PORT]), - - %% create udp socket - {ok, Socket} = gen_udp:open(0, [binary]), - - %% connect to emqx_sn broker - Packet = gen_connect_packet(<<"client1">>), - ok = gen_udp:send(Socket, ?HOST, ?PORT, Packet), - io:format("send connect packet=~p~n", [Packet]), - %% receive message - wait_response(), - - %% subscribe normal topic name - SubscribePacket = gen_subscribe_packet(<<"T3">>), - ok = gen_udp:send(Socket, ?HOST, ?PORT, SubscribePacket), - io:format("send subscribe packet=~p~n", [SubscribePacket]), - wait_response(), - - %% publish SHORT TOPIC NAME - PublishPacket = gen_publish_packet(<<"T3">>, <<"Payload...">>), - ok = gen_udp:send(Socket, ?HOST, ?PORT, PublishPacket), - io:format("send publish packet=~p~n", [PublishPacket]), - wait_response(), - - % wait for subscribed message from broker - wait_response(), - - %% disconnect from emqx_sn broker - DisConnectPacket = gen_disconnect_packet(), - ok = gen_udp:send(Socket, ?HOST, ?PORT, DisConnectPacket), - io:format("send disconnect packet=~p~n", [DisConnectPacket]). - - - -gen_connect_packet(ClientId) -> - Length = 6+byte_size(ClientId), - MsgType = ?SN_CONNECT, - Dup = 0, - QoS = 0, - Retain = 0, - Will = 0, - CleanSession = 1, - TopicIdType = 0, - Flag = <>, - ProtocolId = 1, - Duration = 10, - <>. - -gen_subscribe_packet(ShortTopic) -> - Length = 7, - MsgType = ?SN_SUBSCRIBE, - Dup = 0, - Retain = 0, - Will = 0, - QoS = 1, - CleanSession = 0, - TopicIdType = 0, % normal topic name - Flag = <>, - MsgId = 1, - <>. - -gen_register_packet(Topic, TopicId) -> - Length = 6+byte_size(Topic), - MsgType = ?SN_REGISTER, - MsgId = 1, - <>. - -gen_publish_packet(ShortTopic, Payload) -> - Length = 7+byte_size(Payload), - MsgType = ?SN_PUBLISH, - Dup = 0, - QoS = 1, - Retain = 0, - Will = 0, - CleanSession = 0, - MsgId = 1, - TopicIdType = 2, % SHORT TOPIC NAME - Flag = <>, - <>. - -gen_disconnect_packet()-> - Length = 2, - MsgType = ?SN_DISCONNECT, - <>. - -wait_response() -> - receive - {udp, _Socket, _, _, Bin} -> - case Bin of - <<_Len:8, ?SN_PUBLISH, _Flag:8, TopicId:16, MsgId:16, Data/binary>> -> - io:format("recv publish TopicId: ~p, MsgId: ~p, Data: ~p~n", [TopicId, MsgId, Data]); - <<_Len:8, ?SN_CONNACK, 0:8>> -> - io:format("recv connect ack~n"); - <<_Len:8, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>> -> - io:format("recv regack TopicId=~p, MsgId=~p~n", [TopicId, MsgId]), - TopicId; - <<_Len:8, ?SN_SUBACK, Flags:8, TopicId:16, MsgId:16, 0:8>> -> - io:format("recv suback Flags=~p TopicId=~p, MsgId=~p~n", [Flags, TopicId, MsgId]); - <<_Len:8, ?SN_PUBACK, TopicId:16, MsgId:16, 0:8>> -> - io:format("recv puback TopicId=~p, MsgId=~p~n", [TopicId, MsgId]); - _ -> - io:format("ignore bin=~p~n", [Bin]) - end; - Any -> - io:format("recv something else from udp socket ~p~n", [Any]) - after - 2000 -> - io:format("Error: receive timeout!~n"), - wait_response() - end. diff --git a/apps/emqx_sn/examples/simple_example4.erl b/apps/emqx_sn/examples/simple_example4.erl deleted file mode 100644 index 6beb5835c..000000000 --- a/apps/emqx_sn/examples/simple_example4.erl +++ /dev/null @@ -1,151 +0,0 @@ --module(simple_example4). - --include("emqx_sn.hrl"). - --define(HOST, {127,0,0,1}). --define(PORT, 1884). - --export([start/0]). - -start(LoopTimes) -> - io:format("start to connect ~p:~p~n", [?HOST, ?PORT]), - - %% create udp socket - {ok, Socket} = gen_udp:open(0, [binary]), - - %% connect to emqx_sn broker - Packet = gen_connect_packet(<<"client1">>), - ok = gen_udp:send(Socket, ?HOST, ?PORT, Packet), - io:format("send connect packet=~p~n", [Packet]), - %% receive message - wait_response(), - - %% register topic_id - RegisterPacket = gen_register_packet(<<"TopicA">>, 0), - ok = gen_udp:send(Socket, ?HOST, ?PORT, RegisterPacket), - io:format("send register packet=~p~n", [RegisterPacket]), - TopicId = wait_response(), - - %% subscribe - SubscribePacket = gen_subscribe_packet(TopicId), - ok = gen_udp:send(Socket, ?HOST, ?PORT, SubscribePacket), - io:format("send subscribe packet=~p~n", [SubscribePacket]), - wait_response(), - - %% loop publish - [begin - timer:sleep(1000), - io:format("~n-------------------- publish ~p start --------------------~n", [N]), - - PublishPacket = gen_publish_packet(TopicId, <<"Payload...">>), - ok = gen_udp:send(Socket, ?HOST, ?PORT, PublishPacket), - io:format("send publish packet=~p~n", [PublishPacket]), - % wait for publish ack - wait_response(), - % wait for subscribed message from broker - wait_response(), - - PingReqPacket = gen_pingreq_packet(), - ok = gen_udp:send(Socket, ?HOST, ?PORT, PingReqPacket), - % wait for pingresp - wait_response(), - - io:format("--------------------- publish ~p end ---------------------~n", [N]) - end || N <- lists:seq(1, LoopTimes)], - - %% disconnect from emqx_sn broker - DisConnectPacket = gen_disconnect_packet(), - ok = gen_udp:send(Socket, ?HOST, ?PORT, DisConnectPacket), - io:format("send disconnect packet=~p~n", [DisConnectPacket]). - - - -gen_connect_packet(ClientId) -> - Length = 6+byte_size(ClientId), - MsgType = ?SN_CONNECT, - Dup = 0, - QoS = 0, - Retain = 0, - Will = 0, - CleanSession = 1, - TopicIdType = 0, - Flag = <>, - ProtocolId = 1, - Duration = 10, - <>. - -gen_subscribe_packet(TopicId) -> - Length = 7, - MsgType = ?SN_SUBSCRIBE, - Dup = 0, - Retain = 0, - Will = 0, - QoS = 1, - CleanSession = 0, - TopicIdType = 1, - Flag = <>, - MsgId = 1, - <>. - -gen_register_packet(Topic, TopicId) -> - Length = 6+byte_size(Topic), - MsgType = ?SN_REGISTER, - MsgId = 1, - <>. - -gen_publish_packet(TopicId, Payload) -> - Length = 7+byte_size(Payload), - MsgType = ?SN_PUBLISH, - Dup = 0, - QoS = 1, - Retain = 0, - Will = 0, - CleanSession = 0, - MsgId = 1, - TopicIdType = 1, - Flag = <>, - <>. - -gen_puback_packet(TopicId, MsgId) -> - Length = 7, - MsgType = ?SN_PUBACK, - <>. - -gen_pingreq_packet() -> - Length = 2, - MsgType = ?SN_PINGREQ, - <>. - -gen_disconnect_packet()-> - Length = 2, - MsgType = ?SN_DISCONNECT, - <>. - -wait_response() -> - receive - {udp, Socket, _, _, Bin} -> - case Bin of - <<_Len:8, ?SN_PUBLISH, _Flag:8, TopicId:16, MsgId:16, Data/binary>> -> - io:format("recv publish TopicId: ~p, MsgId: ~p, Data: ~p~n", [TopicId, MsgId, Data]), - ok = gen_udp:send(Socket, ?HOST, ?PORT, gen_puback_packet(TopicId, MsgId)); - <<_Len:8, ?SN_CONNACK, 0:8>> -> - io:format("recv connect ack~n"); - <<_Len:8, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>> -> - io:format("recv regack TopicId=~p, MsgId=~p~n", [TopicId, MsgId]), - TopicId; - <<_Len:8, ?SN_SUBACK, Flags:8, TopicId:16, MsgId:16, 0:8>> -> - io:format("recv suback Flags=~p TopicId=~p, MsgId=~p~n", [Flags, TopicId, MsgId]); - <<_Len:8, ?SN_PUBACK, TopicId:16, MsgId:16, 0:8>> -> - io:format("recv puback TopicId=~p, MsgId=~p~n", [TopicId, MsgId]); - <<_Len:8, ?SN_PINGRESP>> -> - io:format("recv pingresp~n"); - _ -> - io:format("ignore bin=~p~n", [Bin]) - end; - Any -> - io:format("recv something else from udp socket ~p~n", [Any]) - after - 2000 -> - io:format("Error: receive timeout!~n"), - wait_response() - end. diff --git a/apps/emqx_sn/priv/emqx_sn.schema b/apps/emqx_sn/priv/emqx_sn.schema deleted file mode 100644 index edc76db37..000000000 --- a/apps/emqx_sn/priv/emqx_sn.schema +++ /dev/null @@ -1,70 +0,0 @@ -%%-*- mode: erlang -*- -%% emqx_sn config mapping -{mapping, "mqtt.sn.port", "emqx_sn.port", [ - {default, 1884}, - {datatype, [integer, ip]} -]}. - -{translation, "emqx_sn.port", fun(Conf) -> - case cuttlefish:conf_get("mqtt.sn.port", Conf, undefined) of - Port when is_integer(Port) -> - {{0,0,0,0}, Port}; - {Ip, Port} -> - case inet:parse_address(Ip) of - {ok ,R} -> {R, Port}; - _ -> {Ip, Port} - end - end -end}. - -{mapping, "mqtt.sn.advertise_duration", "emqx_sn.advertise_duration", [ - {default, "15s"}, - {datatype, {duration, ms}} -]}. - -{mapping, "mqtt.sn.gateway_id", "emqx_sn.gateway_id", [ - {default, 1}, - {datatype, integer} -]}. - -{mapping, "mqtt.sn.username", "emqx_sn.username", [ - {datatype, string} -]}. - -{mapping, "mqtt.sn.password", "emqx_sn.password", [ - {datatype, string} -]}. - -{mapping, "mqtt.sn.idle_timeout", "emqx_sn.idle_timeout", [ - {default, "30s"}, - {datatype, {duration, ms}} -]}. - -{mapping, "mqtt.sn.enable_stats", "emqx_sn.enable_stats", [ - {datatype, flag} -]}. - -{mapping, "mqtt.sn.enable_qos3", "emqx_sn.enable_qos3", [ - {datatype, flag} -]}. - -{mapping, "mqtt.sn.predefined.topic.$id", "emqx_sn.predefined", [ - {datatype, string} -]}. - -{translation, "emqx_sn.username", fun(Conf) -> - Username = cuttlefish:conf_get("mqtt.sn.username", Conf), - list_to_binary(Username) -end}. - -{translation, "emqx_sn.password", fun(Conf) -> - Password = cuttlefish:conf_get("mqtt.sn.password", Conf), - list_to_binary(Password) -end}. - -{translation, "emqx_sn.predefined", fun(Conf) -> - List = cuttlefish_variable:filter_by_prefix("mqtt.sn.predefined.topic", Conf), - TopicIdList = lists:sort([{list_to_integer(I), iolist_to_binary(TopicName)} - || {["mqtt", "sn", "predefined", "topic", I], TopicName} - <- List, I =/= "0"]) -end}. diff --git a/apps/emqx_sn/src/emqx_sn.app.src b/apps/emqx_sn/src/emqx_sn.app.src deleted file mode 100644 index 0e4e53dc8..000000000 --- a/apps/emqx_sn/src/emqx_sn.app.src +++ /dev/null @@ -1,14 +0,0 @@ -{application, emqx_sn, - [{description, "EMQ X MQTT-SN Plugin"}, - {vsn, "4.4.0"}, % strict semver, bump manually! - {modules, []}, - {registered, []}, - {applications, [kernel,stdlib,esockd]}, - {mod, {emqx_sn_app,[]}}, - {env, []}, - {licenses, ["Apache-2.0"]}, - {maintainers, ["EMQ X Team "]}, - {links, [{"Homepage", "https://emqx.io/"}, - {"Github", "https://github.com/emqx/emqx-sn"} - ]} - ]}. diff --git a/apps/emqx_sn/src/emqx_sn.appup.src b/apps/emqx_sn/src/emqx_sn.appup.src deleted file mode 100644 index 2bd6f5646..000000000 --- a/apps/emqx_sn/src/emqx_sn.appup.src +++ /dev/null @@ -1,19 +0,0 @@ -%% -*-: erlang -*- -{VSN, - [ - {"4.3.2", [ - {load_module, emqx_sn_gateway, brutal_purge, soft_purge, []} - ]}, - {<<"4.3.[0-1]">>, [ - {restart_application, emqx_sn} - ]} - ], - [ - {"4.3.2", [ - {load_module, emqx_sn_gateway, brutal_purge, soft_purge, []} - ]}, - {<<"4.3.[0-1]">>, [ - {restart_application, emqx_sn} - ]} - ] -}. diff --git a/apps/emqx_sn/src/emqx_sn_app.erl b/apps/emqx_sn/src/emqx_sn_app.erl deleted file mode 100644 index 9575523f8..000000000 --- a/apps/emqx_sn/src/emqx_sn_app.erl +++ /dev/null @@ -1,148 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_sn_app). - --behaviour(application). - --emqx_plugin(protocol). - --export([ start/2 - , stop/1 - ]). - --export([ start_listeners/0 - , start_listener/1 - , start_listener/3 - , stop_listeners/0 - , stop_listener/1 - , stop_listener/3 - ]). - --define(UDP_SOCKOPTS, []). - --type(listener() :: {esockd:proto(), esockd:listen_on(), [esockd:option()]}). - -%%-------------------------------------------------------------------- -%% Application -%%-------------------------------------------------------------------- - -start(_Type, _Args) -> - Addr = application:get_env(emqx_sn, port, 1884), - GwId = application:get_env(emqx_sn, gateway_id, 1), - PredefTopics = application:get_env(emqx_sn, predefined, []), - {ok, Sup} = emqx_sn_sup:start_link(Addr, GwId, PredefTopics), - start_listeners(), - {ok, Sup}. - -stop(_State) -> - stop_listeners(), - ok. - -%%-------------------------------------------------------------------- -%% Listners -%%-------------------------------------------------------------------- - --spec start_listeners() -> ok. -start_listeners() -> - lists:foreach(fun start_listener/1, listeners_confs()). - --spec start_listener(listener()) -> ok. -start_listener({Proto, ListenOn, Options}) -> - case start_listener(Proto, ListenOn, Options) of - {ok, _} -> io:format("Start mqttsn:~s listener on ~s successfully.~n", - [Proto, format(ListenOn)]); - {error, Reason} -> - io:format(standard_error, "Failed to start mqttsn:~s listener on ~s: ~0p~n", - [Proto, format(ListenOn), Reason]), - error(Reason) - end. - -%% Start MQTTSN listener --spec start_listener(esockd:proto(), esockd:listen_on(), [esockd:option()]) - -> {ok, pid()} | {error, term()}. -start_listener(udp, ListenOn, Options) -> - start_udp_listener('mqttsn:udp', ListenOn, Options); -start_listener(dtls, ListenOn, Options) -> - start_udp_listener('mqttsn:dtls', ListenOn, Options). - -%% @private -start_udp_listener(Name, ListenOn, Options) -> - SockOpts = esockd:parse_opt(Options), - esockd:open_udp(Name, ListenOn, merge_default(SockOpts), - {emqx_sn_gateway, start_link, [Options -- SockOpts]}). - --spec stop_listeners() -> ok. -stop_listeners() -> - lists:foreach(fun stop_listener/1, listeners_confs()). - --spec stop_listener(listener()) -> ok | {error, term()}. -stop_listener({Proto, ListenOn, Opts}) -> - StopRet = stop_listener(Proto, ListenOn, Opts), - case StopRet of - ok -> io:format("Stop mqttsn:~s listener on ~s successfully.~n", - [Proto, format(ListenOn)]); - {error, Reason} -> - io:format(standard_error, "Failed to stop mqttsn:~s listener on ~s: ~0p~n", - [Proto, format(ListenOn), Reason]) - end, - StopRet. - --spec stop_listener(esockd:proto(), esockd:listen_on(), [esockd:option()]) - -> ok | {error, term()}. -stop_listener(udp, ListenOn, _Opts) -> - esockd:close('mqttsn:udp', ListenOn); -stop_listener(dtls, ListenOn, _Opts) -> - esockd:close('mqttsn:dtls', ListenOn). - -%%-------------------------------------------------------------------- -%% Internal funcs -%%-------------------------------------------------------------------- - -%% @private -%% In order to compatible with the old version of the configuration format -listeners_confs() -> - ListenOn = application:get_env(emqx_sn, port, 1884), - GwId = application:get_env(emqx_sn, gateway_id, 1), - Username = application:get_env(emqx_sn, username, undefined), - Password = application:get_env(emqx_sn, password, undefined), - EnableQos3 = application:get_env(emqx_sn, enable_qos3, false), - EnableStats = application:get_env(emqx_sn, enable_stats, false), - IdleTimeout = application:get_env(emqx_sn, idle_timeout, 30000), - [{udp, ListenOn, [{gateway_id, GwId}, - {username, Username}, - {password, Password}, - {enable_qos3, EnableQos3}, - {enable_stats, EnableStats}, - {idle_timeout, IdleTimeout}, - {max_connections, 1024000}, - {max_conn_rate, 1000}, - {udp_options, []}]}]. - -merge_default(Options) -> - case lists:keytake(udp_options, 1, Options) of - {value, {udp_options, TcpOpts}, Options1} -> - [{udp_options, emqx_misc:merge_opts(?UDP_SOCKOPTS, TcpOpts)} | Options1]; - false -> - [{udp_options, ?UDP_SOCKOPTS} | Options] - end. - -format(Port) when is_integer(Port) -> - io_lib:format("0.0.0.0:~w", [Port]); -format({Addr, Port}) when is_list(Addr) -> - io_lib:format("~s:~w", [Addr, Port]); -format({Addr, Port}) when is_tuple(Addr) -> - io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). diff --git a/apps/emqx_sn/src/emqx_sn_asleep_timer.erl b/apps/emqx_sn/src/emqx_sn_asleep_timer.erl deleted file mode 100644 index 37ea67689..000000000 --- a/apps/emqx_sn/src/emqx_sn_asleep_timer.erl +++ /dev/null @@ -1,65 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_sn_asleep_timer). - --export([ init/0 - , ensure/2 - , cancel/1 - ]). - --record(asleep_state, { - %% Time internal (seconds) - duration :: integer(), - %% Timer reference - tref :: reference() | undefined - }). - --type(asleep_state() :: #asleep_state{}). - --export_type([asleep_state/0]). - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - --spec(init() -> asleep_state()). -init() -> - #asleep_state{duration = 0}. - --spec(ensure(undefined | integer(), asleep_state()) -> asleep_state()). -ensure(undefined, State = #asleep_state{duration = Duration}) -> - ensure(Duration, State); -ensure(Duration, State) -> - cancel(State), - State#asleep_state{duration = Duration, tref = start(Duration)}. - -%%-------------------------------------------------------------------- -%% Internal funcs -%%-------------------------------------------------------------------- - --compile({inline, [start/1, cancel/1]}). - -start(Duration) -> - erlang:send_after(timer:seconds(Duration), self(), asleep_timeout). - -cancel(#asleep_state{tref = Timer}) when is_reference(Timer) -> - case erlang:cancel_timer(Timer) of - false -> - receive {timeout, Timer, _} -> ok after 0 -> ok end; - _ -> ok - end; -cancel(_) -> ok. \ No newline at end of file diff --git a/apps/emqx_sn/src/emqx_sn_gateway.erl b/apps/emqx_sn/src/emqx_sn_gateway.erl deleted file mode 100644 index 1bccf0c1a..000000000 --- a/apps/emqx_sn/src/emqx_sn_gateway.erl +++ /dev/null @@ -1,1132 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_sn_gateway). - --behaviour(gen_statem). - --include("emqx_sn.hrl"). --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("emqx/include/logger.hrl"). - --logger_header("[MQTT-SN]"). - -%% API. --export([start_link/3]). - --export([ info/1 - , stats/1 - ]). - --export([ call/2 - , call/3 - ]). - -%% SUB/UNSUB Asynchronously, called by plugins. --export([ subscribe/2 - , unsubscribe/2 - ]). - --export([kick/1]). - -%% state functions --export([ idle/3 - , wait_for_will_topic/3 - , wait_for_will_msg/3 - , connected/3 - , asleep/3 - , awake/3 - ]). - -%% gen_statem callbacks --export([ init/1 - , callback_mode/0 - , handle_event/4 - , terminate/3 - , code_change/4 - ]). - --ifdef(TEST). --compile(export_all). --compile(nowarn_export_all). --endif. - --type(maybe(T) :: T | undefined). - --type(pending_msgs() :: #{integer() => [#mqtt_sn_message{}]}). - --record(will_msg, {retain = false :: boolean(), - qos = ?QOS_0 :: emqx_mqtt_types:qos(), - topic :: maybe(binary()), - payload :: maybe(binary()) - }). - --record(state, {gwid :: integer(), - socket :: port(), - sockpid :: pid(), - sockstate :: emqx_types:sockstate(), - sockname :: {inet:ip_address(), inet:port()}, - peername :: {inet:ip_address(), inet:port()}, - channel :: maybe(emqx_channel:channel()), - clientid :: maybe(binary()), - username :: maybe(binary()), - password :: maybe(binary()), - will_msg :: maybe(#will_msg{}), - keepalive_interval :: maybe(integer()), - connpkt :: term(), - asleep_timer :: tuple(), - enable_stats :: boolean(), - stats_timer :: maybe(reference()), - idle_timeout :: integer(), - enable_qos3 = false :: boolean(), - has_pending_pingresp = false :: boolean(), - pending_topic_ids = #{} :: pending_msgs() - }). - --define(INFO_KEYS, [socktype, peername, sockname, sockstate]). %, active_n]). --define(CONN_STATS, [recv_pkt, recv_msg, send_pkt, send_msg]). --define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). - --define(STAT_TIMEOUT, 10000). --define(IDLE_TIMEOUT, 30000). --define(DEFAULT_CHAN_OPTIONS, [{max_packet_size, 256}, {zone, external}]). - --define(NEG_QOS_CLIENT_ID, <<"NegQoS-Client">>). - --define(NO_PEERCERT, undefined). - --define(CONN_INFO(Sockname, Peername), - #{socktype => udp, - sockname => Sockname, - peername => Peername, - protocol => 'mqtt-sn', - peercert => ?NO_PEERCERT, - conn_mod => ?MODULE - }). - --define(is_non_error_reason(Reason), - Reason =:= normal; - Reason =:= idle_timeout; - Reason =:= asleep_timeout; - Reason =:= keepalive_timeout). - -%%-------------------------------------------------------------------- -%% Exported APIs -%%-------------------------------------------------------------------- - -start_link(Transport, Peername, Options) -> - gen_statem:start_link(?MODULE, [Transport, Peername, Options], [{hibernate_after, 60000}]). - -subscribe(GwPid, TopicTable) -> - gen_statem:cast(GwPid, {subscribe, TopicTable}). - -unsubscribe(GwPid, Topics) -> - gen_statem:cast(GwPid, {unsubscribe, Topics}). - -kick(GwPid) -> - gen_statem:call(GwPid, kick). - -%%-------------------------------------------------------------------- -%% gen_statem callbacks -%%-------------------------------------------------------------------- - -init([{_, SockPid, Sock}, Peername, Options]) -> - GwId = proplists:get_value(gateway_id, Options), - Username = proplists:get_value(username, Options, undefined), - Password = proplists:get_value(password, Options, undefined), - EnableQos3 = proplists:get_value(enable_qos3, Options, false), - IdleTimeout = proplists:get_value(idle_timeout, Options, 30000), - EnableStats = proplists:get_value(enable_stats, Options, false), - case inet:sockname(Sock) of - {ok, Sockname} -> - Channel = emqx_channel:init(?CONN_INFO(Sockname, Peername), ?DEFAULT_CHAN_OPTIONS), - State = #state{gwid = GwId, - username = Username, - password = Password, - socket = Sock, - sockstate = running, - sockpid = SockPid, - sockname = Sockname, - peername = Peername, - channel = Channel, - asleep_timer = emqx_sn_asleep_timer:init(), - enable_stats = EnableStats, - enable_qos3 = EnableQos3, - idle_timeout = IdleTimeout - }, - emqx_logger:set_metadata_peername(esockd:format(Peername)), - {ok, idle, State, [IdleTimeout]}; - {error, Reason} when Reason =:= enotconn; - Reason =:= einval; - Reason =:= closed -> - {stop, normal}; - {error, Reason} -> {stop, Reason} - end. - -callback_mode() -> state_functions. - -idle(cast, {incoming, ?SN_SEARCHGW_MSG(_Radius)}, State = #state{gwid = GwId}) -> - State0 = send_message(?SN_GWINFO_MSG(GwId, <<>>), State), - {keep_state, State0, State0#state.idle_timeout}; - -idle(cast, {incoming, ?SN_CONNECT_MSG(Flags, _ProtoId, Duration, ClientId)}, State) -> - #mqtt_sn_flags{will = Will, clean_start = CleanStart} = Flags, - do_connect(ClientId, CleanStart, Will, Duration, State); - -idle(cast, {incoming, ?SN_ADVERTISE_MSG(_GwId, _Radius)}, State) -> - % ignore - {keep_state, State, State#state.idle_timeout}; - -idle(cast, {incoming, ?SN_DISCONNECT_MSG(_Duration)}, State) -> - % ignore - {keep_state, State, State#state.idle_timeout}; - -idle(cast, {incoming, ?SN_PUBLISH_MSG(_Flag, _TopicId, _MsgId, _Data)}, State = #state{enable_qos3 = false}) -> - ?LOG(debug, "The enable_qos3 is false, ignore the received publish with QoS=-1 in idle mode!"), - {keep_state, State#state.idle_timeout}; - -idle(cast, {incoming, ?SN_PUBLISH_MSG(#mqtt_sn_flags{qos = ?QOS_NEG1, - topic_id_type = TopicIdType - }, TopicId, _MsgId, Data)}, - State = #state{clientid = ClientId}) -> - TopicName = case (TopicIdType =:= ?SN_SHORT_TOPIC) of - false -> emqx_sn_registry:lookup_topic(ClientId, TopicId); - true -> <> - end, - _ = case TopicName =/= undefined of - true -> - Msg = emqx_message:make(?NEG_QOS_CLIENT_ID, ?QOS_0, TopicName, Data), - emqx_broker:publish(Msg); - false -> - ok - end, - ?LOG(debug, "Client id=~p receives a publish with QoS=-1 in idle mode!", [ClientId]), - {keep_state, State#state.idle_timeout}; - -idle(cast, {incoming, PingReq = ?SN_PINGREQ_MSG(_ClientId)}, State) -> - handle_ping(PingReq, State); - -idle(cast, {outgoing, Packet}, State) -> - {keep_state, handle_outgoing(Packet, State)}; - -idle(cast, {connack, ConnAck}, State) -> - {next_state, connected, handle_outgoing(ConnAck, State)}; - -idle(timeout, _Timeout, State) -> - stop(idle_timeout, State); - -idle(EventType, EventContent, State) -> - handle_event(EventType, EventContent, idle, State). - -wait_for_will_topic(cast, {incoming, ?SN_WILLTOPIC_EMPTY_MSG}, State = #state{connpkt = ConnPkt}) -> - %% 6.3: - %% Note that if a client wants to delete only its Will data at connection setup, - %% it could send a CONNECT message with 'CleanSession=false' and 'Will=true', - %% and sends an empty WILLTOPIC message to the GW when prompted to do so - NState = State#state{will_msg = undefined}, - handle_incoming(?CONNECT_PACKET(ConnPkt), NState); - -wait_for_will_topic(cast, {incoming, ?SN_WILLTOPIC_MSG(Flags, Topic)}, State) -> - #mqtt_sn_flags{qos = QoS, retain = Retain} = Flags, - WillMsg = #will_msg{retain = Retain, qos = QoS, topic = Topic}, - State0 = send_message(?SN_WILLMSGREQ_MSG(), State), - {next_state, wait_for_will_msg, State0#state{will_msg = WillMsg}}; - -wait_for_will_topic(cast, {incoming, ?SN_ADVERTISE_MSG(_GwId, _Radius)}, _State) -> - % ignore - keep_state_and_data; - -wait_for_will_topic(cast, {incoming, ?SN_CONNECT_MSG(_Flags, _ProtoId, _Duration, _ClientId)}, _State) -> - ?LOG(warning, "Receive connect packet in wait_for_will_topic state", []), - keep_state_and_data; - -wait_for_will_topic(cast, {outgoing, Packet}, State) -> - {keep_state, handle_outgoing(Packet, State)}; - -wait_for_will_topic(cast, {connack, ConnAck}, State) -> - {next_state, connected, handle_outgoing(ConnAck, State)}; - -wait_for_will_topic(cast, Event, _State) -> - ?LOG(error, "wait_for_will_topic UNEXPECTED Event: ~p", [Event]), - keep_state_and_data; - -wait_for_will_topic(EventType, EventContent, State) -> - handle_event(EventType, EventContent, wait_for_will_topic, State). - -wait_for_will_msg(cast, {incoming, ?SN_WILLMSG_MSG(Payload)}, - State = #state{will_msg = WillMsg, connpkt = ConnPkt}) -> - NState = State#state{will_msg = WillMsg#will_msg{payload = Payload}}, - handle_incoming(?CONNECT_PACKET(ConnPkt), NState); - -wait_for_will_msg(cast, {incoming, ?SN_ADVERTISE_MSG(_GwId, _Radius)}, _State) -> - % ignore - keep_state_and_data; - -wait_for_will_msg(cast, {incoming, ?SN_CONNECT_MSG(_Flags, _ProtoId, _Duration, _ClientId)}, _State) -> - ?LOG(warning, "Receive connect packet in wait_for_will_msg state", []), - keep_state_and_data; - -wait_for_will_msg(cast, {outgoing, Packet}, State) -> - {keep_state, handle_outgoing(Packet, State)}; - -wait_for_will_msg(cast, {connack, ConnAck}, State) -> - {next_state, connected, handle_outgoing(ConnAck, State)}; - -wait_for_will_msg(EventType, EventContent, State) -> - handle_event(EventType, EventContent, wait_for_will_msg, State). - -connected(cast, {incoming, ?SN_REGISTER_MSG(_TopicId, MsgId, TopicName)}, - State = #state{clientid = ClientId}) -> - State0 = - case emqx_sn_registry:register_topic(ClientId, TopicName) of - TopicId when is_integer(TopicId) -> - ?LOG(debug, "register ClientId=~p, TopicName=~p, TopicId=~p", [ClientId, TopicName, TopicId]), - send_message(?SN_REGACK_MSG(TopicId, MsgId, ?SN_RC_ACCEPTED), State); - {error, too_large} -> - ?LOG(error, "TopicId is full! ClientId=~p, TopicName=~p", [ClientId, TopicName]), - send_message(?SN_REGACK_MSG(?SN_INVALID_TOPIC_ID, MsgId, ?SN_RC_NOT_SUPPORTED), State); - {error, wildcard_topic} -> - ?LOG(error, "wildcard topic can not be registered! ClientId=~p, TopicName=~p", [ClientId, TopicName]), - send_message(?SN_REGACK_MSG(?SN_INVALID_TOPIC_ID, MsgId, ?SN_RC_NOT_SUPPORTED), State) - end, - {keep_state, State0}; - -connected(cast, {incoming, ?SN_PUBLISH_MSG(Flags, TopicId, MsgId, Data)}, - State = #state{enable_qos3 = EnableQoS3}) -> - #mqtt_sn_flags{topic_id_type = TopicIdType, qos = QoS} = Flags, - Skip = (EnableQoS3 =:= false) andalso (QoS =:= ?QOS_NEG1), - case Skip of - true -> - ?LOG(debug, "The enable_qos3 is false, ignore the received publish with QoS=-1 in connected mode!"), - {keep_state, State}; - false -> - do_publish(TopicIdType, TopicId, Data, Flags, MsgId, State) - end; - -connected(cast, {incoming, ?SN_PUBACK_MSG(TopicId, MsgId, RC)}, State) -> - do_puback(TopicId, MsgId, RC, connected, State); - -connected(cast, {incoming, ?SN_PUBREC_MSG(PubRec, MsgId)}, State) - when PubRec == ?SN_PUBREC; PubRec == ?SN_PUBREL; PubRec == ?SN_PUBCOMP -> - do_pubrec(PubRec, MsgId, connected, State); - -connected(cast, {incoming, ?SN_SUBSCRIBE_MSG(Flags, MsgId, TopicId)}, State) -> - #mqtt_sn_flags{qos = QoS, topic_id_type = TopicIdType} = Flags, - handle_subscribe(TopicIdType, TopicId, QoS, MsgId, State); - -connected(cast, {incoming, ?SN_UNSUBSCRIBE_MSG(Flags, MsgId, TopicId)}, State) -> - #mqtt_sn_flags{topic_id_type = TopicIdType} = Flags, - handle_unsubscribe(TopicIdType, TopicId, MsgId, State); - -connected(cast, {incoming, PingReq = ?SN_PINGREQ_MSG(_ClientId)}, State) -> - handle_ping(PingReq, State); - -connected(cast, {incoming, ?SN_REGACK_MSG(TopicId, _MsgId, ?SN_RC_ACCEPTED)}, State) -> - {keep_state, replay_no_reg_pending_publishes(TopicId, State)}; -connected(cast, {incoming, ?SN_REGACK_MSG(TopicId, MsgId, ReturnCode)}, State) -> - ?LOG(error, "client does not accept register TopicId=~p, MsgId=~p, ReturnCode=~p", - [TopicId, MsgId, ReturnCode]), - {keep_state, State}; - -connected(cast, {incoming, ?SN_DISCONNECT_MSG(Duration)}, State) -> - State0 = send_message(?SN_DISCONNECT_MSG(undefined), State), - case Duration of - undefined -> - handle_incoming(?DISCONNECT_PACKET(), State0); - _Other -> goto_asleep_state(Duration, State0) - end; - -connected(cast, {incoming, ?SN_WILLTOPICUPD_MSG(Flags, Topic)}, State = #state{will_msg = WillMsg}) -> - WillMsg1 = case Topic of - undefined -> undefined; - _ -> update_will_topic(WillMsg, Flags, Topic) - end, - State0 = send_message(?SN_WILLTOPICRESP_MSG(0), State), - {keep_state, State0#state{will_msg = WillMsg1}}; - -connected(cast, {incoming, ?SN_WILLMSGUPD_MSG(Payload)}, State = #state{will_msg = WillMsg}) -> - State0 = send_message(?SN_WILLMSGRESP_MSG(0), State), - {keep_state, State0#state{will_msg = update_will_msg(WillMsg, Payload)}}; - -connected(cast, {incoming, ?SN_ADVERTISE_MSG(_GwId, _Radius)}, State) -> - % ignore - {keep_state, State}; - -connected(cast, {incoming, ?SN_CONNECT_MSG(_Flags, _ProtoId, _Duration, _ClientId)}, _State) -> - ?LOG(warning, "Receive connect packet in wait_for_will_topic state", []), - keep_state_and_data; - -connected(cast, {outgoing, Packet}, State) -> - {keep_state, handle_outgoing(Packet, State)}; - -%% XXX: It's so strange behavoir!!! -connected(cast, {connack, ConnAck}, State) -> - {keep_state, handle_outgoing(ConnAck, State)}; - -connected(cast, {shutdown, Reason, Packet}, State) -> - stop(Reason, handle_outgoing(Packet, State)); - -connected(cast, {shutdown, Reason}, State) -> - stop(Reason, State); - -connected(cast, {close, Reason}, State) -> - ?LOG(debug, "Force to close the socket due to ~p", [Reason]), - handle_info({sock_closed, Reason}, close_socket(State)); - -connected(EventType, EventContent, State) -> - handle_event(EventType, EventContent, connected, State). - -asleep(cast, {incoming, ?SN_DISCONNECT_MSG(Duration)}, State) -> - State0 = send_message(?SN_DISCONNECT_MSG(undefined), State), - case Duration of - undefined -> - handle_incoming(?DISCONNECT_PACKET(), State0); - _Other -> - goto_asleep_state(Duration, State0) - end; - -asleep(cast, {incoming, ?SN_PINGREQ_MSG(undefined)}, State) -> - % ClientId in PINGREQ is mandatory - {keep_state, State}; - -asleep(cast, {incoming, ?SN_PINGREQ_MSG(ClientIdPing)}, - State = #state{clientid = ClientId, channel = Channel}) -> - inc_ping_counter(), - case ClientIdPing of - ClientId -> - case emqx_session:replay(emqx_channel:get_session(Channel)) of - {ok, [], Session0} -> - State0 = send_message(?SN_PINGRESP_MSG(), State), - {keep_state, State0#state{ - channel = emqx_channel:set_session(Session0, Channel)}}; - {ok, Publishes, Session0} -> - {Packets, Channel1} = emqx_channel:do_deliver(Publishes, - emqx_channel:set_session(Session0, Channel)), - {next_state, awake, - State#state{channel = Channel1, has_pending_pingresp = true}, - outgoing_events(Packets ++ [try_goto_asleep])} - end; - _Other -> - {next_state, asleep, State} - end; - -asleep(cast, {incoming, ?SN_PUBACK_MSG(TopicId, MsgId, ReturnCode)}, State) -> - do_puback(TopicId, MsgId, ReturnCode, asleep, State); - -asleep(cast, {incoming, ?SN_PUBREC_MSG(PubRec, MsgId)}, State) - when PubRec == ?SN_PUBREC; PubRec == ?SN_PUBREL; PubRec == ?SN_PUBCOMP -> - do_pubrec(PubRec, MsgId, asleep, State); - -% NOTE: what about following scenario: -% 1) client go to sleep -% 2) client reboot for manual reset or other reasons -% 3) client send a CONNECT -% 4) emq-sn regard this CONNECT as a signal to connected state, not a bootup CONNECT. For this reason, will procedure is lost -% this should be a bug in mqtt-sn channel. -asleep(cast, {incoming, ?SN_CONNECT_MSG(_Flags, _ProtoId, _Duration, _ClientId)}, - State = #state{channel = Channel, asleep_timer = Timer}) -> - NChannel = emqx_channel:ensure_keepalive(#{}, Channel), - emqx_sn_asleep_timer:cancel(Timer), - {next_state, connected, send_connack(State#state{channel = NChannel, - asleep_timer = emqx_sn_asleep_timer:init()})}; - -asleep(EventType, EventContent, State) -> - handle_event(EventType, EventContent, asleep, State). - -awake(cast, {incoming, ?SN_REGACK_MSG(TopicId, _MsgId, ?SN_RC_ACCEPTED)}, State) -> - {keep_state, replay_no_reg_pending_publishes(TopicId, State)}; - -awake(cast, {incoming, ?SN_REGACK_MSG(TopicId, MsgId, ReturnCode)}, State) -> - ?LOG(error, "client does not accept register TopicId=~p, MsgId=~p, ReturnCode=~p", - [TopicId, MsgId, ReturnCode]), - {keep_state, State}; - -awake(cast, {incoming, PingReq = ?SN_PINGREQ_MSG(_ClientId)}, State) -> - handle_ping(PingReq, State); - -awake(cast, {outgoing, Packet}, State) -> - {keep_state, handle_outgoing(Packet, State)}; - -awake(cast, {incoming, ?SN_PUBACK_MSG(TopicId, MsgId, ReturnCode)}, State) -> - do_puback(TopicId, MsgId, ReturnCode, awake, State); - -awake(cast, {incoming, ?SN_PUBREC_MSG(PubRec, MsgId)}, State) - when PubRec == ?SN_PUBREC; PubRec == ?SN_PUBREL; PubRec == ?SN_PUBCOMP -> - do_pubrec(PubRec, MsgId, awake, State); - -awake(cast, try_goto_asleep, State=#state{channel = Channel, - has_pending_pingresp = PingPending}) -> - Inflight = emqx_session:info(inflight, emqx_channel:get_session(Channel)), - case emqx_inflight:size(Inflight) of - 0 when PingPending =:= true -> - State0 = send_message(?SN_PINGRESP_MSG(), State), - goto_asleep_state(State0#state{has_pending_pingresp = false}); - 0 when PingPending =:= false -> - goto_asleep_state(State); - _Size -> - keep_state_and_data - end; - -awake(EventType, EventContent, State) -> - handle_event(EventType, EventContent, awake, State). - -handle_event({call, From}, Req, _StateName, State) -> - case handle_call(From, Req, State) of - {reply, Reply, NState} -> - gen_server:reply(From, Reply), - {keep_state, NState}; - {stop, Reason, Reply, NState} -> - State0 = case NState#state.sockstate of - running -> - send_message(?SN_DISCONNECT_MSG(undefined), NState); - _ -> NState - end, - gen_server:reply(From, Reply), - stop(Reason, State0) - end; - -handle_event(info, {datagram, SockPid, Data}, StateName, - State = #state{sockpid = SockPid, channel = _Channel}) -> - ?LOG(debug, "RECV ~0p", [Data]), - Oct = iolist_size(Data), - inc_counter(recv_oct, Oct), - try emqx_sn_frame:parse(Data) of - {ok, Msg} -> - inc_counter(recv_cnt, 1), - ?LOG(info, "RECV ~s at state ~s", [emqx_sn_frame:format(Msg), StateName]), - {keep_state, State, next_event({incoming, Msg})} - catch - error:Error:Stacktrace -> - ?LOG(info, "Parse frame error: ~p at state ~s, Stacktrace: ~p", - [Error, StateName, Stacktrace]), - stop(frame_error, State) - end; - -handle_event(info, {deliver, _Topic, Msg}, asleep, - State = #state{channel = Channel, pending_topic_ids = Pendings}) -> - % section 6.14, Support of sleeping clients - ?LOG(debug, "enqueue downlink message in asleep state, msg: ~0p, pending_topic_ids: ~0p", - [Msg, Pendings]), - Session = emqx_session:enqueue(Msg, emqx_channel:get_session(Channel)), - {keep_state, State#state{channel = emqx_channel:set_session(Session, Channel)}}; - -handle_event(info, Deliver = {deliver, _Topic, _Msg}, _StateName, - State = #state{channel = Channel}) -> - handle_return(emqx_channel:handle_deliver([Deliver], Channel), State); - -handle_event(info, {redeliver, {?PUBREL, MsgId}}, _StateName, State) -> - {keep_state, send_message(?SN_PUBREC_MSG(?SN_PUBREL, MsgId), State)}; - -%% FIXME: Is not unused in v4.x -handle_event(info, {timeout, TRef, emit_stats}, _StateName, - State = #state{channel = Channel}) -> - case emqx_channel:info(clientinfo, Channel) of - #{clientid := undefined} -> {keep_state, State}; - _ -> handle_timeout(TRef, {emit_stats, stats(State)}, State) - end; - -handle_event(info, {timeout, TRef, keepalive}, _StateName, State) -> - RecvOct = emqx_pd:get_counter(recv_oct), - handle_timeout(TRef, {keepalive, RecvOct}, State); - -handle_event(info, {timeout, TRef, TMsg}, _StateName, State) -> - handle_timeout(TRef, TMsg, State); - -handle_event(info, asleep_timeout, asleep, State) -> - ?LOG(debug, "asleep timer timeout, shutdown now"), - stop(asleep_timeout, State); - -handle_event(info, asleep_timeout, StateName, State) -> - ?LOG(debug, "asleep timer timeout on StateName=~p, ignore it", [StateName]), - {keep_state, State}; - -handle_event(cast, {close, Reason}, _StateName, State) -> - stop(Reason, State); - -handle_event(cast, {event, connected}, _StateName, State = #state{channel = Channel}) -> - ClientId = emqx_channel:info(clientid, Channel), - emqx_cm:insert_channel_info(ClientId, info(State), stats(State)), - {keep_state, State}; - -handle_event(cast, {event, disconnected}, _StateName, State = #state{channel = Channel}) -> - ClientId = emqx_channel:info(clientid, Channel), - emqx_cm:set_chan_info(ClientId, info(State)), - emqx_cm:connection_closed(ClientId), - {keep_state, State}; - -handle_event(cast, {event, _Other}, _StateName, State = #state{channel = Channel}) -> - ClientId = emqx_channel:info(clientid, Channel), - emqx_cm:set_chan_info(ClientId, info(State)), - emqx_cm:set_chan_stats(ClientId, stats(State)), - {keep_state, State}; - -handle_event(EventType, EventContent, StateName, State) -> - ?LOG(error, "StateName: ~s, Unexpected Event: ~0p", - [StateName, {EventType, EventContent}]), - {keep_state, State}. - -terminate(Reason, _StateName, #state{channel = Channel}) -> - ClientId = emqx_channel:info(clientid, Channel), - case Reason of - {shutdown, takeovered} -> - ok; - _ -> - emqx_sn_registry:unregister_topic(ClientId) - end, - emqx_channel:terminate(Reason, Channel), - ok. - -code_change(_Vsn, StateName, State, _Extra) -> - {ok, StateName, State}. - -%%-------------------------------------------------------------------- -%% Handle Call/Info -%%-------------------------------------------------------------------- - -handle_call(_From, info, State) -> - {reply, info(State), State}; - -handle_call(_From, stats, State) -> - {reply, stats(State), State}; - -handle_call(_From, Req, State = #state{channel = Channel}) -> - case emqx_channel:handle_call(Req, Channel) of - {reply, Reply, NChannel} -> - {reply, Reply, State#state{channel = NChannel}}; - {shutdown, Reason, Reply, NChannel} -> - stop(Reason, Reply, State#state{channel = NChannel}) - end. - -handle_info({sock_closed, Reason} = Info, State = #state{channel = Channel}) -> - maybe_send_will_msg(Reason, State), - handle_return(emqx_channel:handle_info(Info, Channel), State). - -handle_timeout(TRef, TMsg, State = #state{channel = Channel}) -> - handle_return(emqx_channel:handle_timeout(TRef, TMsg, Channel), State). - -handle_return(Return, State) -> - handle_return(Return, State, []). - -handle_return({ok, NChannel}, State, AddEvents) -> - handle_return({ok, AddEvents, NChannel}, State, []); -handle_return({ok, Replies, NChannel}, State, AddEvents) -> - {keep_state, State#state{channel = NChannel}, outgoing_events(append(Replies, AddEvents))}; -handle_return({shutdown, Reason, NChannel}, State, _AddEvents) -> - stop(Reason, State#state{channel = NChannel}); -handle_return({shutdown, Reason, OutPacket, NChannel}, State, _AddEvents) -> - NState = State#state{channel = NChannel}, - stop(Reason, handle_outgoing(OutPacket, NState)). - -outgoing_events(Actions) -> - lists:map(fun outgoing_event/1, Actions). - -outgoing_event(Packet) when is_record(Packet, mqtt_packet); - is_record(Packet, mqtt_sn_message)-> - next_event({outgoing, Packet}); -outgoing_event(Action) -> - next_event(Action). - -close_socket(State = #state{sockstate = closed}) -> State; -close_socket(State = #state{socket = _Socket}) -> - %ok = gen_udp:close(Socket), - State#state{sockstate = closed}. - -%%-------------------------------------------------------------------- -%% Info & Stats -%%-------------------------------------------------------------------- - -%% @doc Get infos of the connection/channel. -info(CPid) when is_pid(CPid) -> - call(CPid, info); -info(State = #state{channel = Channel}) -> - ChanInfo = upgrade_infos(emqx_channel:info(Channel)), - SockInfo = maps:from_list( - info(?INFO_KEYS, State)), - ChanInfo#{sockinfo => SockInfo}. - -info(Keys, State) when is_list(Keys) -> - [{Key, info(Key, State)} || Key <- Keys]; -info(socktype, _State) -> - udp; -info(peername, #state{peername = Peername}) -> - Peername; -info(sockname, #state{sockname = Sockname}) -> - Sockname; -info(sockstate, #state{sockstate = SockSt}) -> - SockSt. - -upgrade_infos(ChanInfo = #{conninfo := ConnInfo}) -> - ChanInfo#{conninfo => ConnInfo#{proto_name => <<"MQTT-SN">>, - proto_ver => 1}}. - -%% @doc Get stats of the connection/channel. -stats(CPid) when is_pid(CPid) -> - call(CPid, stats); -stats(#state{socket = Socket, channel = Channel}) -> - SockStats = case inet:getstat(Socket, ?SOCK_STATS) of - {ok, Ss} -> Ss; - {error, _} -> [] - end, - ConnStats = emqx_pd:get_counters(?CONN_STATS), - ChanStats = emqx_channel:stats(Channel), - ProcStats = emqx_misc:proc_stats(), - lists:append([SockStats, ConnStats, ChanStats, ProcStats]). - -call(Pid, Req) -> - call(Pid, Req, infinity). - -call(Pid, Req, Timeout) -> - gen_server:call(Pid, Req, Timeout). - -%%-------------------------------------------------------------------- -%% Internal Functions -%%-------------------------------------------------------------------- -handle_ping(_PingReq, State) -> - State0 = send_message(?SN_PINGRESP_MSG(), State), - inc_ping_counter(), - {keep_state, State0}. - -inc_ping_counter() -> - inc_counter(recv_msg, 1). - -mqtt2sn(?CONNACK_PACKET(0, _SessPresent), _State) -> - ?SN_CONNACK_MSG(0); - -mqtt2sn(?CONNACK_PACKET(_ReturnCode, _SessPresent), _State) -> - ?SN_CONNACK_MSG(?SN_RC_CONGESTION); - -mqtt2sn(?PUBREC_PACKET(MsgId), _State) -> - ?SN_PUBREC_MSG(?SN_PUBREC, MsgId); - -mqtt2sn(?PUBREL_PACKET(MsgId), _State) -> - ?SN_PUBREC_MSG(?SN_PUBREL, MsgId); - -mqtt2sn(?PUBCOMP_PACKET(MsgId), _State) -> - ?SN_PUBREC_MSG(?SN_PUBCOMP, MsgId); - -mqtt2sn(?UNSUBACK_PACKET(MsgId), _State)-> - ?SN_UNSUBACK_MSG(MsgId); - -mqtt2sn(?PUBLISH_PACKET(QoS, Topic, PacketId, Payload), #state{channel = Channel}) -> - NewPacketId = if QoS =:= ?QOS_0 -> 0; - true -> PacketId - end, - ClientId = emqx_channel:info(clientid, Channel), - {TopicIdType, TopicContent} = case emqx_sn_registry:lookup_topic_id(ClientId, Topic) of - {predef, PredefTopicId} -> - {?SN_PREDEFINED_TOPIC, PredefTopicId}; - TopicId when is_integer(TopicId) -> - {?SN_NORMAL_TOPIC, TopicId}; - undefined -> - {?SN_SHORT_TOPIC, Topic} - end, - - Flags = #mqtt_sn_flags{qos = QoS, topic_id_type = TopicIdType}, - ?SN_PUBLISH_MSG(Flags, TopicContent, NewPacketId, Payload); - -mqtt2sn(?SUBACK_PACKET(MsgId, ReturnCodes), _State)-> - % if success, suback is sent by handle_info({suback, MsgId, [GrantedQoS]}, ...) - % if failure, suback is sent in this function. - [ReturnCode | _ ] = ReturnCodes, - {QoS, TopicId, NewReturnCode} - = case ?IS_QOS(ReturnCode) of - true -> - {ReturnCode, get_topic_id(suback, MsgId), ?SN_RC_ACCEPTED}; - _ -> - {?QOS_0, get_topic_id(suback, MsgId), ?SN_RC_NOT_SUPPORTED} - end, - Flags = #mqtt_sn_flags{qos = QoS}, - ?SN_SUBACK_MSG(Flags, TopicId, MsgId, NewReturnCode); - -mqtt2sn(?PUBACK_PACKET(MsgId, _ReasonCode), _State) -> - TopicIdFinal = get_topic_id(puback, MsgId), - ?SN_PUBACK_MSG(TopicIdFinal, MsgId, ?SN_RC_ACCEPTED). - -send_register(TopicName, TopicId, MsgId, State) -> - send_message(?SN_REGISTER_MSG(TopicId, MsgId, TopicName), State). - -send_connack(State) -> - send_message(?SN_CONNACK_MSG(?SN_RC_ACCEPTED), State). - -send_message(Msg = #mqtt_sn_message{type = Type}, - State = #state{sockpid = SockPid, peername = Peername}) -> - ?LOG(debug, "SEND ~s~n", [emqx_sn_frame:format(Msg)]), - inc_outgoing_stats(Type), - Data = emqx_sn_frame:serialize(Msg), - ok = emqx_metrics:inc('bytes.sent', iolist_size(Data)), - SockPid ! {datagram, Peername, Data}, - State. - -goto_asleep_state(State) -> - goto_asleep_state(undefined, State). -goto_asleep_state(Duration, State=#state{asleep_timer = AsleepTimer, - channel = Channel}) -> - ?LOG(debug, "goto_asleep_state Duration=~p", [Duration]), - NewTimer = emqx_sn_asleep_timer:ensure(Duration, AsleepTimer), - NChannel = emqx_channel:clear_keepalive(Channel), - {next_state, asleep, State#state{asleep_timer = NewTimer, - channel = NChannel}, hibernate}. - -%%-------------------------------------------------------------------- -%% Helper funcs -%%-------------------------------------------------------------------- -stop({shutdown, Reason}, State) -> - stop(Reason, State); -stop(Reason, State) -> - ?LOG(stop_log_level(Reason), "stop due to ~p", [Reason]), - maybe_send_will_msg(Reason, State), - {stop, {shutdown, Reason}, State}. - -stop({shutdown, Reason}, Reply, State) -> - stop(Reason, Reply, State); -stop(Reason, Reply, State) -> - ?LOG(stop_log_level(Reason), "stop due to ~p", [Reason]), - maybe_send_will_msg(Reason, State), - {stop, {shutdown, Reason}, Reply, State}. - -maybe_send_will_msg(normal, _State) -> - ok; -maybe_send_will_msg(_Reason, State) -> - do_publish_will(State). - -stop_log_level(Reason) when ?is_non_error_reason(Reason) -> - debug; -stop_log_level(_) -> - error. - -mqttsn_to_mqtt(?SN_PUBACK, MsgId) -> - ?PUBACK_PACKET(MsgId); -mqttsn_to_mqtt(?SN_PUBREC, MsgId) -> - ?PUBREC_PACKET(MsgId); -mqttsn_to_mqtt(?SN_PUBREL, MsgId) -> - ?PUBREL_PACKET(MsgId); -mqttsn_to_mqtt(?SN_PUBCOMP, MsgId) -> - ?PUBCOMP_PACKET(MsgId). - -do_connect(ClientId, CleanStart, WillFlag, Duration, State) -> - emqx_logger:set_metadata_clientid(ClientId), - %% 6.6 Client’s Publish Procedure - %% At any point in time a client may have only one QoS level 1 or 2 PUBLISH message - %% outstanding, i.e. it has to wait for the termination of this PUBLISH message exchange - %% before it could start a new level 1 or 2 transaction. - OnlyOneInflight = #{'Receive-Maximum' => 1}, - ConnPkt = #mqtt_packet_connect{clientid = ClientId, - clean_start = CleanStart, - username = State#state.username, - password = State#state.password, - proto_name = <<"MQTT-SN">>, - keepalive = Duration, - properties = OnlyOneInflight, - proto_ver = 1 - }, - case WillFlag of - true -> State0 = send_message(?SN_WILLTOPICREQ_MSG(), State), - NState = State0#state{connpkt = ConnPkt, - clientid = ClientId, - keepalive_interval = Duration - }, - {next_state, wait_for_will_topic, NState}; - false -> - NState = State#state{clientid = ClientId, - keepalive_interval = Duration - }, - handle_incoming(?CONNECT_PACKET(ConnPkt), NState) - end. - -handle_subscribe(?SN_NORMAL_TOPIC, TopicName, QoS, MsgId, - State=#state{channel = Channel}) -> - ClientId = emqx_channel:info(clientid, Channel), - case emqx_sn_registry:register_topic(ClientId, TopicName) of - {error, too_large} -> - State0 = send_message(?SN_SUBACK_MSG(#mqtt_sn_flags{qos = QoS}, - ?SN_INVALID_TOPIC_ID, - MsgId, - ?SN_RC_INVALID_TOPIC_ID), State), - {keep_state, State0}; - {error, wildcard_topic} -> - proto_subscribe(TopicName, QoS, MsgId, ?SN_INVALID_TOPIC_ID, State); - NewTopicId when is_integer(NewTopicId) -> - proto_subscribe(TopicName, QoS, MsgId, NewTopicId, State) - end; - -handle_subscribe(?SN_PREDEFINED_TOPIC, TopicId, QoS, MsgId, - State = #state{channel = Channel}) -> - ClientId = emqx_channel:info(clientid, Channel), - case emqx_sn_registry:lookup_topic(ClientId, TopicId) of - undefined -> - State0 = send_message(?SN_SUBACK_MSG(#mqtt_sn_flags{qos = QoS}, - TopicId, - MsgId, - ?SN_RC_INVALID_TOPIC_ID), State), - {next_state, connected, State0}; - PredefinedTopic -> - proto_subscribe(PredefinedTopic, QoS, MsgId, TopicId, State) - end; - -handle_subscribe(?SN_SHORT_TOPIC, TopicId, QoS, MsgId, State) -> - TopicName = case is_binary(TopicId) of - true -> TopicId; - false -> <> - end, - proto_subscribe(TopicName, QoS, MsgId, ?SN_INVALID_TOPIC_ID, State); - -handle_subscribe(_, _TopicId, QoS, MsgId, State) -> - State0 = send_message(?SN_SUBACK_MSG(#mqtt_sn_flags{qos = QoS}, - ?SN_INVALID_TOPIC_ID, - MsgId, - ?SN_RC_INVALID_TOPIC_ID), State), - {keep_state, State0}. - -handle_unsubscribe(?SN_NORMAL_TOPIC, TopicId, MsgId, State) -> - proto_unsubscribe(TopicId, MsgId, State); - -handle_unsubscribe(?SN_PREDEFINED_TOPIC, TopicId, MsgId, - State = #state{channel = Channel}) -> - ClientId = emqx_channel:info(clientid, Channel), - case emqx_sn_registry:lookup_topic(ClientId, TopicId) of - undefined -> - {keep_state, send_message(?SN_UNSUBACK_MSG(MsgId), State)}; - PredefinedTopic -> - proto_unsubscribe(PredefinedTopic, MsgId, State) - end; - -handle_unsubscribe(?SN_SHORT_TOPIC, TopicId, MsgId, State) -> - TopicName = case is_binary(TopicId) of - true -> TopicId; - false -> <> - end, - proto_unsubscribe(TopicName, MsgId, State); - -handle_unsubscribe(_, _TopicId, MsgId, State) -> - {keep_state, send_message(?SN_UNSUBACK_MSG(MsgId), State)}. - -do_publish(?SN_NORMAL_TOPIC, TopicName, Data, Flags, MsgId, State) -> - %% XXX: Handle normal topic id as predefined topic id, to be compatible with paho mqtt-sn library - <> = TopicName, - do_publish(?SN_PREDEFINED_TOPIC, TopicId, Data, Flags, MsgId, State); -do_publish(?SN_PREDEFINED_TOPIC, TopicId, Data, Flags, MsgId, - State=#state{channel = Channel}) -> - #mqtt_sn_flags{qos = QoS, dup = Dup, retain = Retain} = Flags, - NewQoS = get_corrected_qos(QoS), - ClientId = emqx_channel:info(clientid, Channel), - case emqx_sn_registry:lookup_topic(ClientId, TopicId) of - undefined -> - {keep_state, maybe_send_puback(NewQoS, TopicId, MsgId, ?SN_RC_INVALID_TOPIC_ID, - State)}; - TopicName -> - proto_publish(TopicName, Data, Dup, NewQoS, Retain, MsgId, TopicId, State) - end; - -do_publish(?SN_SHORT_TOPIC, STopicName, Data, Flags, MsgId, State) -> - #mqtt_sn_flags{qos = QoS, dup = Dup, retain = Retain} = Flags, - NewQoS = get_corrected_qos(QoS), - <> = STopicName, - case emqx_topic:wildcard(STopicName) of - true -> - {keep_state, maybe_send_puback(NewQoS, TopicId, MsgId, ?SN_RC_NOT_SUPPORTED, - State)}; - false -> - proto_publish(STopicName, Data, Dup, NewQoS, Retain, MsgId, TopicId, State) - end; -do_publish(_, TopicId, _Data, #mqtt_sn_flags{qos = QoS}, MsgId, State) -> - {keep_state, maybe_send_puback(QoS, TopicId, MsgId, ?SN_RC_NOT_SUPPORTED, - State)}. - -do_publish_will(#state{will_msg = undefined}) -> - ok; -do_publish_will(#state{will_msg = #will_msg{payload = undefined}}) -> - ok; -do_publish_will(#state{will_msg = #will_msg{topic = undefined}}) -> - ok; -do_publish_will(#state{will_msg = WillMsg, clientid = ClientId}) -> - #will_msg{qos = QoS, retain = Retain, topic = Topic, payload = Payload} = WillMsg, - Publish = #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, dup = false, - qos = QoS, retain = Retain}, - variable = #mqtt_packet_publish{topic_name = Topic, packet_id = 1000}, - payload = Payload}, - _ = emqx_broker:publish(emqx_packet:to_message(Publish, ClientId)), - ok. - -do_puback(TopicId, MsgId, ReturnCode, StateName, - State=#state{channel = Channel}) -> - case ReturnCode of - ?SN_RC_ACCEPTED -> - handle_incoming(?PUBACK_PACKET(MsgId), StateName, State); - ?SN_RC_INVALID_TOPIC_ID -> - ClientId = emqx_channel:info(clientid, Channel), - case emqx_sn_registry:lookup_topic(ClientId, TopicId) of - undefined -> {keep_state, State}; - TopicName -> - %%notice that this TopicName maybe normal or predefined, - %% involving the predefined topic name in register to enhance the gateway's robustness even inconsistent with MQTT-SN channels - {keep_state, send_register(TopicName, TopicId, MsgId, State)} - end; - _ -> - ?LOG(error, "CAN NOT handle PUBACK ReturnCode=~p", [ReturnCode]), - {keep_state, State} - end. - -do_pubrec(PubRec, MsgId, StateName, State) -> - handle_incoming(mqttsn_to_mqtt(PubRec, MsgId), StateName, State). - -proto_subscribe(TopicName, QoS, MsgId, TopicId, State) -> - ?LOG(debug, "subscribe Topic=~p, MsgId=~p, TopicId=~p", - [TopicName, MsgId, TopicId]), - enqueue_msgid(suback, MsgId, TopicId), - SubOpts = maps:put(qos, QoS, ?DEFAULT_SUBOPTS), - handle_incoming(?SUBSCRIBE_PACKET(MsgId, [{TopicName, SubOpts}]), State). - -proto_unsubscribe(TopicName, MsgId, State) -> - ?LOG(debug, "unsubscribe Topic=~p, MsgId=~p", [TopicName, MsgId]), - handle_incoming(?UNSUBSCRIBE_PACKET(MsgId, [TopicName]), State). - -proto_publish(TopicName, Data, Dup, QoS, Retain, MsgId, TopicId, State) -> - (QoS =/= ?QOS_0) andalso enqueue_msgid(puback, MsgId, TopicId), - Publish = #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, dup = Dup, qos = QoS, retain = Retain}, - variable = #mqtt_packet_publish{topic_name = TopicName, packet_id = MsgId}, - payload = Data}, - ?LOG(debug, "[publish] Msg: ~0p~n", [Publish]), - handle_incoming(Publish, State). - -update_will_topic(undefined, #mqtt_sn_flags{qos = QoS, retain = Retain}, Topic) -> - #will_msg{qos = QoS, retain = Retain, topic = Topic}; -update_will_topic(Will=#will_msg{}, #mqtt_sn_flags{qos = QoS, retain = Retain}, Topic) -> - Will#will_msg{qos = QoS, retain = Retain, topic = Topic}. - -update_will_msg(undefined, Msg) -> - #will_msg{payload = Msg}; -update_will_msg(Will = #will_msg{}, Msg) -> - Will#will_msg{payload = Msg}. - -enqueue_msgid(suback, MsgId, TopicId) -> - put({suback, MsgId}, TopicId); -enqueue_msgid(puback, MsgId, TopicId) -> - put({puback, MsgId}, TopicId). - -dequeue_msgid(suback, MsgId) -> - erase({suback, MsgId}); -dequeue_msgid(puback, MsgId) -> - erase({puback, MsgId}). - -get_corrected_qos(?QOS_NEG1) -> - ?LOG(debug, "Receive a publish with QoS=-1"), - ?QOS_0; -get_corrected_qos(QoS) -> - QoS. - -get_topic_id(Type, MsgId) -> - case dequeue_msgid(Type, MsgId) of - undefined -> 0; - TopicId -> TopicId - end. - -handle_incoming(Packet, State) -> - handle_incoming(Packet, unknown, State). - -handle_incoming(#mqtt_packet{variable = #mqtt_packet_puback{}} = Packet, awake, State) -> - Result = channel_handle_in(Packet, State), - handle_return(Result, State, [try_goto_asleep]); - -handle_incoming(Packet, _StName, State) -> - Result = channel_handle_in(Packet, State), - handle_return(Result, State). - -channel_handle_in(Packet = ?PACKET(Type), #state{channel = Channel}) -> - _ = inc_incoming_stats(Type), - ok = emqx_metrics:inc_recv(Packet), - ?LOG(debug, "RECV ~s", [emqx_packet:format(Packet)]), - emqx_channel:handle_in(Packet, Channel). - -handle_outgoing(Packets, State) when is_list(Packets) -> - lists:foldl(fun(Packet, State0) -> - handle_outgoing(Packet, State0) - end, State, Packets); - -handle_outgoing(PubPkt = ?PUBLISH_PACKET(_, TopicName, _, _), - State = #state{channel = Channel}) -> - ?LOG(debug, "Handle outgoing publish: ~0p", [PubPkt]), - ClientId = emqx_channel:info(clientid, Channel), - TopicId = emqx_sn_registry:lookup_topic_id(ClientId, TopicName), - case (TopicId == undefined) andalso (byte_size(TopicName) =/= 2) of - true -> register_and_notify_client(PubPkt, State); - false -> send_message(mqtt2sn(PubPkt, State), State) - end; - -handle_outgoing(Packet, State) -> - send_message(mqtt2sn(Packet, State), State). - -cache_no_reg_publish_message(Pendings, TopicId, PubPkt, State) -> - ?LOG(debug, "cache non-registered publish message for topic-id: ~p, msg: ~0p, pendings: ~0p", - [TopicId, PubPkt, Pendings]), - Msgs = maps:get(pending_topic_ids, Pendings, []), - Pendings#{TopicId => Msgs ++ [mqtt2sn(PubPkt, State)]}. - -replay_no_reg_pending_publishes(TopicId, #state{pending_topic_ids = Pendings} = State0) -> - ?LOG(debug, "replay non-registered publish message for topic-id: ~p, pendings: ~0p", - [TopicId, Pendings]), - State = lists:foldl(fun(Msg, State1) -> - send_message(Msg, State1) - end, State0, maps:get(TopicId, Pendings, [])), - State#state{pending_topic_ids = maps:remove(TopicId, Pendings)}. - -register_and_notify_client(?PUBLISH_PACKET(QoS, TopicName, PacketId, Payload) = PubPkt, - State = #state{pending_topic_ids = Pendings, channel = Channel}) -> - MsgId = message_id(PacketId), - #mqtt_packet{header = #mqtt_packet_header{dup = Dup, retain = Retain}} = PubPkt, - ClientId = emqx_channel:info(clientid, Channel), - TopicId = emqx_sn_registry:register_topic(ClientId, TopicName), - ?LOG(debug, "Register TopicId=~p, TopicName=~p, Payload=~p, Dup=~p, QoS=~p, " - "Retain=~p, MsgId=~p", [TopicId, TopicName, Payload, Dup, QoS, Retain, MsgId]), - NewPendings = cache_no_reg_publish_message(Pendings, TopicId, PubPkt, State), - send_register(TopicName, TopicId, MsgId, State#state{pending_topic_ids = NewPendings}). - -message_id(undefined) -> - rand:uniform(16#FFFF); -message_id(MsgId) -> MsgId. - -inc_incoming_stats(Type) -> - inc_counter(recv_pkt, 1), - case Type == ?PUBLISH of - true -> - inc_counter(recv_msg, 1), - inc_counter(incoming_pubs, 1); - false -> ok - end. - -inc_outgoing_stats(Type) -> - inc_counter(send_pkt, 1), - case Type =:= ?SN_PUBLISH of - true -> inc_counter(send_msg, 1); - false -> ok - end. - -next_event(Content) -> - {next_event, cast, Content}. - -inc_counter(Key, Inc) -> - _ = emqx_pd:inc_counter(Key, Inc), - ok. - -append(Replies, AddEvents) when is_list(Replies) -> - Replies ++ AddEvents; -append(Replies, AddEvents) -> - [Replies] ++ AddEvents. - -maybe_send_puback(?QOS_0, _TopicId, _MsgId, _ReasonCode, State) -> - State; -maybe_send_puback(_QoS, TopicId, MsgId, ReasonCode, State) -> - send_message(?SN_PUBACK_MSG(TopicId, MsgId, ReasonCode), State). diff --git a/apps/emqx_sn/src/emqx_sn_registry.erl b/apps/emqx_sn/src/emqx_sn_registry.erl deleted file mode 100644 index 4a3b22585..000000000 --- a/apps/emqx_sn/src/emqx_sn_registry.erl +++ /dev/null @@ -1,198 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_sn_registry). - --behaviour(gen_server). - --include("emqx_sn.hrl"). - --define(LOG(Level, Format, Args), - emqx_logger:Level("MQTT-SN(registry): " ++ Format, Args)). - --export([ start_link/1 - , stop/0 - ]). - --export([ register_topic/2 - , unregister_topic/1 - ]). - --export([ lookup_topic/2 - , lookup_topic_id/2 - ]). - -%% gen_server callbacks --export([ init/1 - , handle_call/3 - , handle_cast/2 - , handle_info/2 - , terminate/2 - , code_change/3 - ]). - --define(TAB, ?MODULE). - --record(state, {max_predef_topic_id = 0}). - --record(emqx_sn_registry, {key, value}). - -%% Mnesia bootstrap --export([mnesia/1]). - --boot_mnesia({mnesia, [boot]}). --copy_mnesia({mnesia, [copy]}). - - -%% @doc Create or replicate tables. --spec(mnesia(boot | copy) -> ok). -mnesia(boot) -> - %% Optimize storage - StoreProps = [{ets, [{read_concurrency, true}]}], - ok = ekka_mnesia:create_table(?MODULE, [ - {attributes, record_info(fields, emqx_sn_registry)}, - {ram_copies, [node()]}, - {storage_properties, StoreProps}]); - -mnesia(copy) -> - ok = ekka_mnesia:copy_table(?MODULE, ram_copies). - -%%----------------------------------------------------------------------------- - --spec(start_link(list()) -> {ok, pid()} | ignore | {error, Reason :: term()}). -start_link(PredefTopics) -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [PredefTopics], []). - --spec(stop() -> ok). -stop() -> - gen_server:stop(?MODULE, normal, infinity). - --spec(register_topic(binary(), binary()) -> integer() | {error, term()}). -register_topic(ClientId, TopicName) when is_binary(TopicName) -> - case emqx_topic:wildcard(TopicName) of - false -> - gen_server:call(?MODULE, {register, ClientId, TopicName}); - %% TopicId: in case of “accepted” the value that will be used as topic - %% id by the gateway when sending PUBLISH messages to the client (not - %% relevant in case of subscriptions to a short topic name or to a topic - %% name which contains wildcard characters) - true -> {error, wildcard_topic} - end. - --spec(lookup_topic(binary(), pos_integer()) -> undefined | binary()). -lookup_topic(ClientId, TopicId) when is_integer(TopicId) -> - case lookup_element(?TAB, {predef, TopicId}, 3) of - undefined -> - lookup_element(?TAB, {ClientId, TopicId}, 3); - Topic -> Topic - end. - --spec(lookup_topic_id(binary(), binary()) - -> undefined - | pos_integer() - | {predef, integer()}). -lookup_topic_id(ClientId, TopicName) when is_binary(TopicName) -> - case lookup_element(?TAB, {predef, TopicName}, 3) of - undefined -> - lookup_element(?TAB, {ClientId, TopicName}, 3); - TopicId -> - {predef, TopicId} - end. - -%% @private -lookup_element(Tab, Key, Pos) -> - try ets:lookup_element(Tab, Key, Pos) catch error:badarg -> undefined end. - --spec(unregister_topic(binary()) -> ok). -unregister_topic(ClientId) -> - gen_server:call(?MODULE, {unregister, ClientId}). - -%%----------------------------------------------------------------------------- - -init([PredefTopics]) -> - %% {predef, TopicId} -> TopicName - %% {predef, TopicName} -> TopicId - %% {ClientId, TopicId} -> TopicName - %% {ClientId, TopicName} -> TopicId - MaxPredefId = lists:foldl( - fun({TopicId, TopicName}, AccId) -> - mnesia:dirty_write(#emqx_sn_registry{key = {predef, TopicId}, - value = TopicName}), - mnesia:dirty_write(#emqx_sn_registry{key = {predef, TopicName}, - value = TopicId}), - if TopicId > AccId -> TopicId; true -> AccId end - end, 0, PredefTopics), - {ok, #state{max_predef_topic_id = MaxPredefId}}. - -handle_call({register, ClientId, TopicName}, _From, - State = #state{max_predef_topic_id = PredefId}) -> - case lookup_topic_id(ClientId, TopicName) of - {predef, PredefTopicId} when is_integer(PredefTopicId) -> - {reply, PredefTopicId, State}; - TopicId when is_integer(TopicId) -> - {reply, TopicId, State}; - undefined -> - case next_topic_id(?TAB, PredefId, ClientId) of - TopicId when TopicId >= 16#FFFF -> - {reply, {error, too_large}, State}; - TopicId -> - Fun = fun() -> - mnesia:write(#emqx_sn_registry{key = {ClientId, next_topic_id}, - value = TopicId + 1}), - mnesia:write(#emqx_sn_registry{key = {ClientId, TopicName}, - value = TopicId}), - mnesia:write(#emqx_sn_registry{key = {ClientId, TopicId}, - value = TopicName}) - end, - case mnesia:transaction(Fun) of - {atomic, ok} -> - {reply, TopicId, State}; - {aborted, Error} -> - {reply, {error, Error}, State} - end - end - end; - -handle_call({unregister, ClientId}, _From, State) -> - Registry = mnesia:dirty_match_object({?TAB, {ClientId, '_'}, '_'}), - lists:foreach(fun(R) -> mnesia:dirty_delete_object(R) end, Registry), - {reply, ok, State}; - -handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected request: ~p", [Req]), - {reply, ignored, State}. - -handle_cast(Msg, State) -> - ?LOG(error, "Unexpected msg: ~p", [Msg]), - {noreply, State}. - -handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%----------------------------------------------------------------------------- - -next_topic_id(Tab, PredefId, ClientId) -> - case mnesia:dirty_read(Tab, {ClientId, next_topic_id}) of - [#emqx_sn_registry{value = Id}] -> Id; - [] -> PredefId + 1 - end. diff --git a/apps/emqx_sn/src/emqx_sn_sup.erl b/apps/emqx_sn/src/emqx_sn_sup.erl deleted file mode 100644 index 3d4fe602f..000000000 --- a/apps/emqx_sn/src/emqx_sn_sup.erl +++ /dev/null @@ -1,43 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_sn_sup). - --behaviour(supervisor). - --export([ start_link/3 - , init/1 - ]). - -start_link(Addr, GwId, PredefTopics) -> - supervisor:start_link({local, ?MODULE}, ?MODULE, [Addr, GwId, PredefTopics]). - -init([{_Ip, Port}, GwId, PredefTopics]) -> - Broadcast = #{id => emqx_sn_broadcast, - start => {emqx_sn_broadcast, start_link, [GwId, Port]}, - restart => permanent, - shutdown => brutal_kill, - type => worker, - modules => [emqx_sn_broadcast]}, - Registry = #{id => emqx_sn_registry, - start => {emqx_sn_registry, start_link, [PredefTopics]}, - restart => permanent, - shutdown => brutal_kill, - type => worker, - modules => [emqx_sn_registry]}, - {ok, {{one_for_one, 10, 3600}, [Broadcast, Registry]}}. - - diff --git a/apps/emqx_sn/test/emqx_sn_registry_SUITE.erl b/apps/emqx_sn/test/emqx_sn_registry_SUITE.erl deleted file mode 100644 index 8d320d8ed..000000000 --- a/apps/emqx_sn/test/emqx_sn_registry_SUITE.erl +++ /dev/null @@ -1,121 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_sn_registry_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). - --define(REGISTRY, emqx_sn_registry). --define(MAX_PREDEF_ID, 2). --define(PREDEF_TOPICS, [{1, <<"/predefined/topic/name/hello">>}, - {2, <<"/predefined/topic/name/nice">>}]). - -%%-------------------------------------------------------------------- -%% Setups -%%-------------------------------------------------------------------- - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - _ = application:set_env(emqx_sn, predefined, ?PREDEF_TOPICS), - Config. - -end_per_suite(_Config) -> - ok. - -init_per_testcase(_TestCase, Config) -> - ekka_mnesia:start(), - emqx_sn_registry:mnesia(boot), - mnesia:clear_table(emqx_sn_registry), - PredefTopics = application:get_env(emqx_sn, predefined, []), - {ok, _Pid} = ?REGISTRY:start_link(PredefTopics), - Config. - -end_per_testcase(_TestCase, Config) -> - ?REGISTRY:stop(), - Config. - -%%-------------------------------------------------------------------- -%% Test cases -%%-------------------------------------------------------------------- - -t_register(_Config) -> - ?assertEqual(?MAX_PREDEF_ID+1, ?REGISTRY:register_topic(<<"ClientId">>, <<"Topic1">>)), - ?assertEqual(?MAX_PREDEF_ID+2, ?REGISTRY:register_topic(<<"ClientId">>, <<"Topic2">>)), - ?assertEqual(<<"Topic1">>, ?REGISTRY:lookup_topic(<<"ClientId">>, ?MAX_PREDEF_ID+1)), - ?assertEqual(<<"Topic2">>, ?REGISTRY:lookup_topic(<<"ClientId">>, ?MAX_PREDEF_ID+2)), - ?assertEqual(?MAX_PREDEF_ID+1, ?REGISTRY:lookup_topic_id(<<"ClientId">>, <<"Topic1">>)), - ?assertEqual(?MAX_PREDEF_ID+2, ?REGISTRY:lookup_topic_id(<<"ClientId">>, <<"Topic2">>)), - emqx_sn_registry:unregister_topic(<<"ClientId">>), - ?assertEqual(undefined, ?REGISTRY:lookup_topic(<<"ClientId">>, ?MAX_PREDEF_ID+1)), - ?assertEqual(undefined, ?REGISTRY:lookup_topic(<<"ClientId">>, ?MAX_PREDEF_ID+2)), - ?assertEqual(undefined, ?REGISTRY:lookup_topic_id(<<"ClientId">>, <<"Topic1">>)), - ?assertEqual(undefined, ?REGISTRY:lookup_topic_id(<<"ClientId">>, <<"Topic2">>)). - -t_register_case2(_Config) -> - ?assertEqual(?MAX_PREDEF_ID+1, ?REGISTRY:register_topic(<<"ClientId">>, <<"Topic1">>)), - ?assertEqual(?MAX_PREDEF_ID+2, ?REGISTRY:register_topic(<<"ClientId">>, <<"Topic2">>)), - ?assertEqual(?MAX_PREDEF_ID+1, ?REGISTRY:register_topic(<<"ClientId">>, <<"Topic1">>)), - ?assertEqual(<<"Topic1">>, ?REGISTRY:lookup_topic(<<"ClientId">>, ?MAX_PREDEF_ID+1)), - ?assertEqual(<<"Topic2">>, ?REGISTRY:lookup_topic(<<"ClientId">>, ?MAX_PREDEF_ID+2)), - ?assertEqual(?MAX_PREDEF_ID+1, ?REGISTRY:lookup_topic_id(<<"ClientId">>, <<"Topic1">>)), - ?assertEqual(?MAX_PREDEF_ID+2, ?REGISTRY:lookup_topic_id(<<"ClientId">>, <<"Topic2">>)), - ?assertEqual(undefined, ?REGISTRY:lookup_topic_id(<<"ClientId">>, <<"Topic3">>)), - ?REGISTRY:unregister_topic(<<"ClientId">>), - ?assertEqual(undefined, ?REGISTRY:lookup_topic(<<"ClientId">>, ?MAX_PREDEF_ID+1)), - ?assertEqual(undefined, ?REGISTRY:lookup_topic(<<"ClientId">>, ?MAX_PREDEF_ID+2)), - ?assertEqual(undefined, ?REGISTRY:lookup_topic_id(<<"ClientId">>, <<"Topic1">>)), - ?assertEqual(undefined, ?REGISTRY:lookup_topic_id(<<"ClientId">>, <<"Topic2">>)). - -t_reach_maximum(_Config) -> - register_a_lot(?MAX_PREDEF_ID+1, 16#ffff), - ?assertEqual({error, too_large}, ?REGISTRY:register_topic(<<"ClientId">>, <<"TopicABC">>)), - Topic1 = iolist_to_binary(io_lib:format("Topic~p", [?MAX_PREDEF_ID+1])), - Topic2 = iolist_to_binary(io_lib:format("Topic~p", [?MAX_PREDEF_ID+2])), - ?assertEqual(?MAX_PREDEF_ID+1, ?REGISTRY:lookup_topic_id(<<"ClientId">>, Topic1)), - ?assertEqual(?MAX_PREDEF_ID+2, ?REGISTRY:lookup_topic_id(<<"ClientId">>, Topic2)), - ?REGISTRY:unregister_topic(<<"ClientId">>), - ?assertEqual(undefined, ?REGISTRY:lookup_topic(<<"ClientId">>, ?MAX_PREDEF_ID+1)), - ?assertEqual(undefined, ?REGISTRY:lookup_topic(<<"ClientId">>, ?MAX_PREDEF_ID+2)), - ?assertEqual(undefined, ?REGISTRY:lookup_topic_id(<<"ClientId">>, Topic1)), - ?assertEqual(undefined, ?REGISTRY:lookup_topic_id(<<"ClientId">>, Topic2)). - -t_register_case4(_Config) -> - ?assertEqual(?MAX_PREDEF_ID+1, ?REGISTRY:register_topic(<<"ClientId">>, <<"TopicA">>)), - ?assertEqual(?MAX_PREDEF_ID+2, ?REGISTRY:register_topic(<<"ClientId">>, <<"TopicB">>)), - ?assertEqual(?MAX_PREDEF_ID+3, ?REGISTRY:register_topic(<<"ClientId">>, <<"TopicC">>)), - ?REGISTRY:unregister_topic(<<"ClientId">>), - ?assertEqual(?MAX_PREDEF_ID+1, ?REGISTRY:register_topic(<<"ClientId">>, <<"TopicD">>)). - -t_deny_wildcard_topic(_Config) -> - ?assertEqual({error, wildcard_topic}, ?REGISTRY:register_topic(<<"ClientId">>, <<"/TopicA/#">>)), - ?assertEqual({error, wildcard_topic}, ?REGISTRY:register_topic(<<"ClientId">>, <<"/+/TopicB">>)). - -%%-------------------------------------------------------------------- -%% Helper funcs -%%-------------------------------------------------------------------- - -register_a_lot(Max, Max) -> - ok; -register_a_lot(N, Max) when N < Max -> - Topic = iolist_to_binary(["Topic", integer_to_list(N)]), - ?assertEqual(N, ?REGISTRY:register_topic(<<"ClientId">>, Topic)), - register_a_lot(N+1, Max). - diff --git a/apps/emqx_sn/vars b/apps/emqx_sn/vars deleted file mode 100644 index a170916f3..000000000 --- a/apps/emqx_sn/vars +++ /dev/null @@ -1,8 +0,0 @@ -%% vars here are for test only, not intended for release - -{platform_bin_dir, "bin"}. -{platform_data_dir, "data"}. -{platform_etc_dir, "etc"}. -{platform_lib_dir, "lib"}. -{platform_log_dir, "log"}. -{platform_plugins_dir, "plugins"}. diff --git a/apps/emqx_statsd/etc/emqx_statsd.conf b/apps/emqx_statsd/etc/emqx_statsd.conf index a2daa5521..2bb6014a4 100644 --- a/apps/emqx_statsd/etc/emqx_statsd.conf +++ b/apps/emqx_statsd/etc/emqx_statsd.conf @@ -1,13 +1,10 @@ ##-------------------------------------------------------------------- - ## Statsd for EMQ X - ##-------------------------------------------------------------------- +## Statsd for EMQ X +##-------------------------------------------------------------------- -emqx_statsd:{ - host: "127.0.0.1" - port: 8125 - batch_size: 10 - prefix: "emqx" - tags: {"from": "emqx"} +statsd:{ + enable: true + server: "127.0.0.1:8125" sample_time_interval: "10s" flush_time_interval: "10s" } diff --git a/apps/emqx_statsd/include/emqx_statsd.hrl b/apps/emqx_statsd/include/emqx_statsd.hrl index d88dbccbd..52f8774c0 100644 --- a/apps/emqx_statsd/include/emqx_statsd.hrl +++ b/apps/emqx_statsd/include/emqx_statsd.hrl @@ -1,9 +1,5 @@ -define(APP, emqx_statsd). - --define(DEFAULT_HOST, {127, 0, 0, 1}). +-define(DEFAULT_SAMPLE_TIME_INTERVAL, 10000). +-define(DEFAULT_FLUSH_TIME_INTERVAL, 10000). +-define(DEFAULT_HOST, "127.0.0.1"). -define(DEFAULT_PORT, 8125). --define(DEFAULT_PREFIX, undefined). --define(DEFAULT_TAGS, #{}). --define(DEFAULT_BATCH_SIZE, 10). --define(DEFAULT_SAMPLE_TIME_INTERVAL, 10). --define(DEFAULT_FLUSH_TIME_INTERVAL, 10). \ No newline at end of file diff --git a/apps/emqx_statsd/src/emqx_statsd.app.src b/apps/emqx_statsd/src/emqx_statsd.app.src index 04338fd62..44f5c6192 100644 --- a/apps/emqx_statsd/src/emqx_statsd.app.src +++ b/apps/emqx_statsd/src/emqx_statsd.app.src @@ -1,6 +1,6 @@ {application, emqx_statsd, [{description, "An OTP application"}, - {vsn, "0.1.0"}, + {vsn, "5.0.0"}, {registered, []}, {mod, {emqx_statsd_app, []}}, {applications, @@ -10,7 +10,6 @@ ]}, {env,[]}, {modules, []}, - {licenses, ["Apache 2.0"]}, {links, []} ]}. diff --git a/apps/emqx_statsd/src/emqx_statsd.erl b/apps/emqx_statsd/src/emqx_statsd.erl index 133c32321..892731a6c 100644 --- a/apps/emqx_statsd/src/emqx_statsd.erl +++ b/apps/emqx_statsd/src/emqx_statsd.erl @@ -1,101 +1,119 @@ %%-------------------------------------------------------------------- - %% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. - %% - %% Licensed under the Apache License, Version 2.0 (the "License"); - %% you may not use this file except in compliance with the License. - %% You may obtain a copy of the License at - %% - %% http://www.apache.org/licenses/LICENSE-2.0 - %% - %% Unless required by applicable law or agreed to in writing, software - %% distributed under the License is distributed on an "AS IS" BASIS, - %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - %% See the License for the specific language governing permissions and - %% limitations under the License. - %%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- - -module(emqx_statsd). +-module(emqx_statsd). - -behaviour(gen_server). +-behaviour(gen_server). - -ifdef(TEST). - -compile(export_all). - -compile(nowarn_export_all). - -endif. +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-endif. - -include_lib("emqx/include/logger.hrl"). +-include("emqx_statsd.hrl"). - %% Interface - -export([start_link/1]). +%% Interface +-export([start_link/1]). - %% Internal Exports - -export([ init/1 - , handle_call/3 - , handle_cast/2 - , handle_info/2 - , code_change/3 - , terminate/2 - ]). +%% Internal Exports +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , code_change/3 + , terminate/2 + ]). - -record(state, { - timer :: reference(), - sample_time_interval :: pos_integer(), - flush_time_interval :: pos_integer(), - estatsd_pid :: pid() - }). +-record(state, { + timer :: reference() | undefined, + sample_time_interval :: pos_integer(), + flush_time_interval :: pos_integer(), + estatsd_pid :: pid() +}). - start_link(Opts) -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [Opts], []). +start_link(Opts) -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [Opts], []). - init([Opts]) -> - SampleTimeInterval = proplists:get_value(sample_time_interval, Opts), - FlushTimeInterval = proplists:get_value(flush_time_interval, Opts), - Ref = erlang:start_timer(SampleTimeInterval, self(), sample_timeout), - Pid = proplists:get_value(estatsd_pid, Opts), - {ok, #state{timer = Ref, - sample_time_interval = SampleTimeInterval, - flush_time_interval = FlushTimeInterval, - estatsd_pid = Pid}}. +init([Opts]) -> + process_flag(trap_exit, true), + Tags = tags(maps:get(tags, Opts, #{})), + {Host, Port} = maps:get(server, Opts, {?DEFAULT_HOST, ?DEFAULT_PORT}), + Opts1 = maps:without([sample_time_interval, + flush_time_interval], Opts#{tags => Tags, + host => Host, + port => Port, + prefix => <<"emqx">>}), + {ok, Pid} = estatsd:start_link(maps:to_list(Opts1)), + SampleTimeInterval = maps:get(sample_time_interval, Opts, ?DEFAULT_FLUSH_TIME_INTERVAL), + FlushTimeInterval = maps:get(flush_time_interval, Opts, ?DEFAULT_FLUSH_TIME_INTERVAL), + {ok, ensure_timer(#state{sample_time_interval = SampleTimeInterval, + flush_time_interval = FlushTimeInterval, + estatsd_pid = Pid})}. - handle_call(_Req, _From, State) -> - {noreply, State}. +handle_call(_Req, _From, State) -> + {noreply, State}. - handle_cast(_Msg, State) -> - {noreply, State}. +handle_cast(_Msg, State) -> + {noreply, State}. - handle_info({timeout, Ref, sample_timeout}, State = #state{sample_time_interval = SampleTimeInterval, - flush_time_interval = FlushTimeInterval, - estatsd_pid = Pid, - timer = Ref}) -> - ?LOG(debug, "emqx statsd submit"), - Metrics = emqx_metrics:all() ++ emqx_stats:getstats() ++ emqx_vm_data(), - SampleRate = SampleTimeInterval / FlushTimeInterval, - StatsdMetrics = [{gauge, trans_metrics_name(Name), Value, SampleRate, []} || {Name, Value} <- Metrics], - estatsd:submit(Pid, StatsdMetrics), - {noreply, State#state{timer = erlang:start_timer(SampleTimeInterval, self(), sample_timeout)}}; +handle_info({timeout, Ref, sample_timeout}, + State = #state{sample_time_interval = SampleTimeInterval, + flush_time_interval = FlushTimeInterval, + estatsd_pid = Pid, + timer = Ref}) -> + Metrics = emqx_metrics:all() ++ emqx_stats:getstats() ++ emqx_vm_data(), + SampleRate = SampleTimeInterval / FlushTimeInterval, + StatsdMetrics = [{gauge, trans_metrics_name(Name), Value, SampleRate, []} || {Name, Value} <- Metrics], + estatsd:submit(Pid, StatsdMetrics), + {noreply, ensure_timer(State)}; - handle_info(_Msg, State) -> - {noreply, State}. +handle_info({'EXIT', Pid, Error}, State = #state{estatsd_pid = Pid}) -> + {stop, {shutdown, Error}, State}; - code_change(_OldVsn, State, _Extra) -> - {ok, State}. +handle_info(_Msg, State) -> + {noreply, State}. - terminate(_Reason, _State) -> - ok. +code_change(_OldVsn, State, _Extra) -> + {ok, State}. - %%------------------------------------------------------------------------------ - %% Internale function - %%------------------------------------------------------------------------------ - trans_metrics_name(Name) -> - Name0 = atom_to_binary(Name, utf8), - binary_to_atom(<<"emqx.", Name0/binary>>, utf8). +terminate(_Reason, #state{estatsd_pid = Pid}) -> + estatsd:stop(Pid), + ok. - emqx_vm_data() -> - Idle = case cpu_sup:util([detailed]) of - {_, 0, 0, _} -> 0; %% Not support for Windows - {_Num, _Use, IdleList, _} -> proplists:get_value(idle, IdleList, 0) - end, - RunQueue = erlang:statistics(run_queue), - [{run_queue, RunQueue}, - {cpu_idle, Idle}, - {cpu_use, 100 - Idle}] ++ emqx_vm:mem_info(). +%%------------------------------------------------------------------------------ +%% Internale function +%%------------------------------------------------------------------------------ +trans_metrics_name(Name) -> + Name0 = atom_to_binary(Name, utf8), + binary_to_atom(<<"emqx.", Name0/binary>>, utf8). + +emqx_vm_data() -> + Idle = case cpu_sup:util([detailed]) of + {_, 0, 0, _} -> 0; %% Not support for Windows + {_Num, _Use, IdleList, _} -> proplists:get_value(idle, IdleList, 0) + end, + RunQueue = erlang:statistics(run_queue), + [{run_queue, RunQueue}, + {cpu_idle, Idle}, + {cpu_use, 100 - Idle}] ++ emqx_vm:mem_info(). + +tags(Map) -> + Tags = maps:to_list(Map), + [{atom_to_binary(Key, utf8), Value} || {Key, Value} <- Tags]. + + +ensure_timer(State =#state{sample_time_interval = SampleTimeInterval}) -> + State#state{timer = emqx_misc:start_timer(SampleTimeInterval, sample_timeout)}. diff --git a/apps/emqx_statsd/src/emqx_statsd_api.erl b/apps/emqx_statsd/src/emqx_statsd_api.erl new file mode 100644 index 000000000..c16bc9a61 --- /dev/null +++ b/apps/emqx_statsd/src/emqx_statsd_api.erl @@ -0,0 +1,106 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_statsd_api). + +-behaviour(minirest_api). + +-include("emqx_statsd.hrl"). + +-import(emqx_mgmt_util, [ response_schema/1 + , response_schema/2 + , request_body_schema/1 + ]). + +-export([api_spec/0]). + +-export([ statsd/2 + ]). + +api_spec() -> + {statsd_api(), schemas()}. + +schemas() -> + [#{statsd => #{ + type => object, + properties => #{ + server => #{ + type => string, + description => <<"Statsd Server">>, + example => get_raw(<<"server">>, <<"127.0.0.1:8125">>)}, + enable => #{ + type => boolean, + description => <<"Statsd status">>, + example => get_raw(<<"enable">>, false)}, + sample_time_interval => #{ + type => string, + description => <<"Sample Time Interval">>, + example => get_raw(<<"sample_time_interval">>, <<"10s">>)}, + flush_time_interval => #{ + type => string, + description => <<"Flush Time Interval">>, + example => get_raw(<<"flush_time_interval">>, <<"10s">>)} + } + }}]. + +statsd_api() -> + Metadata = #{ + get => #{ + description => <<"Get statsd info">>, + responses => #{ + <<"200">> => response_schema(<<"statsd">>) + } + }, + put => #{ + description => <<"Update Statsd">>, + 'requestBody' => request_body_schema(<<"statsd">>), + responses => #{ + <<"200">> => + response_schema(<<"Update Statsd successfully">>), + <<"400">> => + response_schema(<<"Bad Request">>, #{ + type => object, + properties => #{ + message => #{type => string}, + code => #{type => string} + } + }) + } + } + }, + [{"/statsd", Metadata, statsd}]. + +statsd(get, _Request) -> + Response = emqx_config:get_raw([<<"statsd">>], #{}), + {200, Response}; + +statsd(put, Request) -> + {ok, Body, _} = cowboy_req:read_body(Request), + Params = emqx_json:decode(Body, [return_maps]), + Enable = maps:get(<<"enable">>, Params), + ok = emqx_config:update([statsd], Params), + enable_statsd(Enable). + +enable_statsd(true) -> + ok = emqx_statsd_sup:stop_child(?APP), + emqx_statsd_sup:start_child(?APP, emqx_config:get([statsd], #{})), + {200}; +enable_statsd(false) -> + _ = emqx_statsd_sup:stop_child(?APP), + {200}. + +get_raw(Key, Def) -> + emqx_config:get_raw([<<"statsd">>]++ [Key], Def). diff --git a/apps/emqx_statsd/src/emqx_statsd_app.erl b/apps/emqx_statsd/src/emqx_statsd_app.erl index cd998b158..6dd9dc5c7 100644 --- a/apps/emqx_statsd/src/emqx_statsd_app.erl +++ b/apps/emqx_statsd/src/emqx_statsd_app.erl @@ -18,9 +18,7 @@ -behaviour(application). --include_lib("emqx/include/logger.hrl"). - - -emqx_plugin(?MODULE). +-include("emqx_statsd.hrl"). -export([ start/2 , stop/1 @@ -28,11 +26,15 @@ start(_StartType, _StartArgs) -> {ok, Sup} = emqx_statsd_sup:start_link(), - {ok, _} = emqx_statsd_sup:start_statsd(), - ?LOG(info, "emqx statsd start: successfully"), + maybe_enable_statsd(), {ok, Sup}. - stop(_) -> - ok = emqx_statsd_sup:stop_statsd(), - ?LOG(info, "emqx statsd stop: successfully"), ok. + +maybe_enable_statsd() -> + case emqx_config:get([statsd, enable], false) of + true -> + emqx_statsd_sup:start_child(?APP, emqx_config:get([statsd], #{})); + false -> + ok + end. diff --git a/apps/emqx_statsd/src/emqx_statsd_schema.erl b/apps/emqx_statsd/src/emqx_statsd_schema.erl index 5600739e7..3af8a112c 100644 --- a/apps/emqx_statsd/src/emqx_statsd_schema.erl +++ b/apps/emqx_statsd/src/emqx_statsd_schema.erl @@ -4,41 +4,38 @@ -behaviour(hocon_schema). +-export([to_ip_port/1]). + -export([ structs/0 , fields/1]). -structs() -> ["emqx_statsd"]. +-typerefl_from_string({ip_port/0, emqx_statsd_schema, to_ip_port}). -fields("emqx_statsd") -> - [ {host, fun host/1} - , {port, fun port/1} - , {prefix, fun prefix/1} - , {tags, map()} - , {batch_size, fun batch_size/1} - , {sample_time_interval, fun duration_s/1} - , {flush_time_interval, fun duration_s/1}]. +structs() -> ["statsd"]. -host(type) -> string(); -host(default) -> "127.0.0.1"; -host(nullable) -> false; -host(_) -> undefined. +fields("statsd") -> + [ {enable, emqx_schema:t(boolean(), undefined, false)} + , {server, fun server/1} + , {sample_time_interval, fun duration_ms/1} + , {flush_time_interval, fun duration_ms/1} + ]. -port(type) -> integer(); -port(default) -> 8125; -port(nullable) -> true; -port(_) -> undefined. +server(type) -> emqx_schema:ip_port(); +server(default) -> "127.0.0.1:8125"; +server(nullable) -> false; +server(_) -> undefined. -prefix(type) -> string(); -prefix(default) -> "emqx"; -prefix(nullable) -> true; -prefix(_) -> undefined. +duration_ms(type) -> emqx_schema:duration_ms(); +duration_ms(nullable) -> false; +duration_ms(default) -> "10s"; +duration_ms(_) -> undefined. -batch_size(type) -> integer(); -batch_size(nullable) -> false; -batch_size(default) -> 10; -batch_size(_) -> undefined. - -duration_s(type) -> emqx_schema:duration_s(); -duration_s(nullable) -> false; -duration_s(default) -> "10s"; -duration_s(_) -> undefined. +to_ip_port(Str) -> + case string:tokens(Str, ":") of + [Ip, Port] -> + case inet:parse_address(Ip) of + {ok, R} -> {ok, {R, list_to_integer(Port)}}; + _ -> {error, Str} + end; + _ -> {error, Str} + end. diff --git a/apps/emqx_statsd/src/emqx_statsd_sup.erl b/apps/emqx_statsd/src/emqx_statsd_sup.erl index e33ea5493..02b50e3ca 100644 --- a/apps/emqx_statsd/src/emqx_statsd_sup.erl +++ b/apps/emqx_statsd/src/emqx_statsd_sup.erl @@ -7,63 +7,48 @@ -behaviour(supervisor). --include("emqx_statsd.hrl"). - --export([start_link/0]). - --export([start_statsd/0, stop_statsd/0]). +-export([ start_link/0 + , start_child/1 + , start_child/2 + , stop_child/1 + ]). -export([init/1]). --export([estatsd_options/0]). +%% Helper macro for declaring children of supervisor +-define(CHILD(Mod, Opts), #{id => Mod, + start => {Mod, start_link, [Opts]}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [Mod]}). - start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). - init([]) -> - {ok, {{one_for_one, 10, 100}, []}}. +-spec start_child(supervisor:child_spec()) -> ok. +start_child(ChildSpec) when is_map(ChildSpec) -> + assert_started(supervisor:start_child(?MODULE, ChildSpec)). -start_statsd() -> - {ok, Pid} = supervisor:start_child(?MODULE, estatsd_child_spec()), - {ok, _Pid1} = supervisor:start_child(?MODULE, emqx_statsd_child_spec(Pid)). +-spec start_child(atom(), map()) -> ok. +start_child(Mod, Opts) when is_atom(Mod) andalso is_map(Opts) -> + assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Opts))). -stop_statsd() -> - ok = supervisor:terminate_child(?MODULE, emqx_statsd), - ok = supervisor:terminate_child(?MODULE, estatsd). -%%============================================================================================== -%% internal -estatsd_child_spec() -> - #{id => estatsd - , start => {estatsd, start_link, [estatsd_options()]} - , restart => permanent - , shutdown => 5000 - , type => worker - , modules => [estatsd]}. +-spec(stop_child(any()) -> ok | {error, term()}). +stop_child(ChildId) -> + case supervisor:terminate_child(?MODULE, ChildId) of + ok -> supervisor:delete_child(?MODULE, ChildId); + Error -> Error + end. -estatsd_options() -> - Host = get_conf(host, ?DEFAULT_HOST), - Port = get_conf(port, ?DEFAULT_PORT), - Prefix = get_conf(prefix, ?DEFAULT_PREFIX), - Tags = tags(get_conf(tags, ?DEFAULT_TAGS)), - BatchSize = get_conf(batch_size, ?DEFAULT_BATCH_SIZE), - [{host, Host}, {port, Port}, {prefix, Prefix}, {tags, Tags}, {batch_size, BatchSize}]. +init([]) -> + {ok, {{one_for_one, 10, 3600}, []}}. -tags(Map) -> - Tags = maps:to_list(Map), - [{atom_to_binary(Key, utf8), Value} || {Key, Value} <- Tags]. +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- -emqx_statsd_child_spec(Pid) -> - #{id => emqx_statsd - , start => {emqx_statsd, start_link, [[{estatsd_pid, Pid} | emqx_statsd_options()]]} - , restart => permanent - , shutdown => 5000 - , type => worker - , modules => [emqx_statsd]}. - -emqx_statsd_options() -> - SampleTimeInterval = get_conf(sample_time_interval, ?DEFAULT_SAMPLE_TIME_INTERVAL) * 1000, - FlushTimeInterval = get_conf(flush_time_interval, ?DEFAULT_FLUSH_TIME_INTERVAL) * 1000, - [{sample_time_interval, SampleTimeInterval}, {flush_time_interval, FlushTimeInterval}]. - -get_conf(Key, Default) -> - emqx_config:get([?APP, Key], Default). +assert_started({ok, _Pid}) -> ok; +assert_started({ok, _Pid, _Info}) -> ok; +assert_started({error, {already_tarted, _Pid}}) -> ok; +assert_started({error, Reason}) -> erlang:error(Reason). diff --git a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl index 2855e8ee5..9d06ee351 100644 --- a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl +++ b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl @@ -13,7 +13,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([emqx_statsd]). -all() -> +all() -> emqx_ct:all(?MODULE). t_statsd(_) -> diff --git a/apps/emqx_stomp/.formatter.exs b/apps/emqx_stomp/.formatter.exs deleted file mode 100644 index d2cda26ed..000000000 --- a/apps/emqx_stomp/.formatter.exs +++ /dev/null @@ -1,4 +0,0 @@ -# Used by "mix format" -[ - inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] -] diff --git a/apps/emqx_stomp/.gitignore b/apps/emqx_stomp/.gitignore deleted file mode 100644 index 95654d437..000000000 --- a/apps/emqx_stomp/.gitignore +++ /dev/null @@ -1,25 +0,0 @@ -.eunit -deps -*.o -*.beam -*.plt -erl_crash.dump -ebin -rel/example_project -.concrete/DEV_MODE -.rebar -.erlang.mk/ -emq_stomp.d -ct.coverdata -logs/ -test/ct.cover.spec -data/ -.DS_Store -emqx_stomp.d -_build/ -rebar.lock -erlang.mk -rebar3.crashdump -etc/emqx_stomp.conf.rendered -.rebar3/ -*.swp diff --git a/apps/emqx_stomp/etc/emqx_stomp.conf b/apps/emqx_stomp/etc/emqx_stomp.conf deleted file mode 100644 index 832b50655..000000000 --- a/apps/emqx_stomp/etc/emqx_stomp.conf +++ /dev/null @@ -1,124 +0,0 @@ -##-------------------------------------------------------------------- -## Stomp Plugin -##-------------------------------------------------------------------- - -##-------------------------------------------------------------------- -## Stomp listener - -## The Port that stomp listener will bind. -## -## Value: Port -stomp.listener.port = 61613 - -## The acceptor pool for stomp listener. -## -## Value: Number -stomp.listener.acceptors = 4 - -## Maximum number of concurrent stomp connections. -## -## Value: Number -stomp.listener.max_connections = 512 - -## Whether to enable SSL. -## -## Value: on | off -## stomp.listener.ssl = off - -## Path to the file containing the user's private PEM-encoded key. -## -## Value: File -## stomp.listener.keyfile = "etc/certs/key.pem" - -## Path to a file containing the user certificate. -## -## Value: File -## stomp.listener.certfile = "etc/certs/cert.pem" - -## Path to the file containing PEM-encoded CA certificates. -## -## Value: File -## stomp.listener.cacertfile = "etc/certs/cacert.pem" - -## See: 'listener.ssl..dhfile' in emq.conf -## -## Value: File -## stomp.listener.dhfile = "etc/certs/dh-params.pem" - -## See: 'listener.ssl..verify' in emq.conf -## -## Value: verify_peer | verify_none -## stomp.listener.verify = verify_peer - -## See: 'listener.ssl..fail_if_no_peer_cert' in emq.conf -## -## Value: false | true -## stomp.listener.fail_if_no_peer_cert = true - -## TLS versions only to protect from POODLE attack. -## -## Value: String, seperated by ',' -## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier -## stomp.listener.tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" - -## SSL Handshake timeout. -## -## Value: Duration -## stomp.listener.handshake_timeout = 15s - -## See: 'listener.ssl..ciphers' in emq.conf -## -## Value: Ciphers -## stomp.listener.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" - -## See: 'listener.ssl..secure_renegotiate' in emq.conf -## -## Value: on | off -## stomp.listener.secure_renegotiate = off - -## See: 'listener.ssl..reuse_sessions' in emq.conf -## -## Value: on | off -## stomp.listener.reuse_sessions = on - -## See: 'listener.ssl..honor_cipher_order' in emq.conf -## -## Value: on | off -## stomp.listener.honor_cipher_order = on - -##-------------------------------------------------------------------- -## Stomp login user and password - -## Default login user -## -## Value: String -stomp.default_user.login = guest - -## Default login password -## -## Value: String -stomp.default_user.passcode = guest - -## Allow anonymous authentication. -## -## Value: true | false -stomp.allow_anonymous = true - -##-------------------------------------------------------------------- -## Stomp frame - -## Maximum numbers of frame headers. -## -## Value: Number -stomp.frame.max_headers = 10 - -## Maximum length of frame header. -## -## Value: Number -stomp.frame.max_header_length = 1024 - -## Maximum body length of frame. -## -## Value: Number -stomp.frame.max_body_length = 8192 - diff --git a/apps/emqx_stomp/include/emqx_stomp.hrl b/apps/emqx_stomp/include/emqx_stomp.hrl deleted file mode 100644 index a9cf2cf48..000000000 --- a/apps/emqx_stomp/include/emqx_stomp.hrl +++ /dev/null @@ -1,48 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc Stomp Frame Header. - --define(STOMP_VER, <<"1.2">>). - --define(STOMP_SERVER, <<"emqx-stomp/1.2">>). - -%%-------------------------------------------------------------------- -%% STOMP Frame -%%-------------------------------------------------------------------- - --record(stomp_frame, {command, headers = [], body = <<>> :: iodata()}). - --type(stomp_frame() :: #stomp_frame{}). - -%%-------------------------------------------------------------------- -%% Frame Size Limits -%% -%% To prevent malicious clients from exploiting memory allocation in a server, -%% servers MAY place maximum limits on: -%% -%% the number of frame headers allowed in a single frame -%% the maximum length of header lines -%% the maximum size of a frame body -%% -%% If these limits are exceeded the server SHOULD send the client an ERROR frame -%% and then close the connection. -%%-------------------------------------------------------------------- - --define(MAX_HEADER_NUM, 10). --define(MAX_HEADER_LENGTH, 1024). --define(MAX_BODY_LENGTH, 65536). - diff --git a/apps/emqx_stomp/mix.exs b/apps/emqx_stomp/mix.exs deleted file mode 100644 index 84cd4d395..000000000 --- a/apps/emqx_stomp/mix.exs +++ /dev/null @@ -1,32 +0,0 @@ -defmodule EMQXStomp.MixProject do - use Mix.Project - - def project do - [ - app: :emqx_stomp, - version: "4.4.0", - build_path: "../../_build", - config_path: "../../config/config.exs", - deps_path: "../../deps", - lockfile: "../../mix.lock", - elixir: "~> 1.12", - start_permanent: Mix.env() == :prod, - deps: deps(), - description: "EMQ X Stomp Protocol Plugin" - ] - end - - def application do - [ - registered: [:emqx_stomp_sup], - mod: {:emqx_stomp, []}, - extra_applications: [:logger] - ] - end - - defp deps do - [ - {:emqx, in_umbrella: true, runtime: false} - ] - end -end diff --git a/apps/emqx_stomp/priv/emqx_stomp.schema b/apps/emqx_stomp/priv/emqx_stomp.schema deleted file mode 100644 index 32a3c272b..000000000 --- a/apps/emqx_stomp/priv/emqx_stomp.schema +++ /dev/null @@ -1,149 +0,0 @@ -%%-*- mode: erlang -*- -%% emqx_stomp config mapping - -{mapping, "stomp.listener.port", "emqx_stomp.listener", [ - {default, 61613}, - {datatype, [integer, ip]} -]}. - -{mapping, "stomp.listener.acceptors", "emqx_stomp.listener", [ - {default, 4}, - {datatype, integer} -]}. - -{mapping, "stomp.listener.max_connections", "emqx_stomp.listener", [ - {default, 512}, - {datatype, integer} -]}. - -{mapping, "stomp.listener.ssl", "emqx_stomp.listener", [ - {datatype, flag}, - {default, off} -]}. - -{mapping, "stomp.listener.tls_versions", "emqx_stomp.listener", [ - {datatype, string} -]}. - -{mapping, "stomp.listener.handshake_timeout", "emqx_stomp.listener", [ - {default, "15s"}, - {datatype, {duration, ms}} -]}. - -{mapping, "stomp.listener.dhfile", "emqx_stomp.listener", [ - {datatype, string} -]}. - -{mapping, "stomp.listener.keyfile", "emqx_stomp.listener", [ - {datatype, string} -]}. - -{mapping, "stomp.listener.certfile", "emqx_stomp.listener", [ - {datatype, string} -]}. - -{mapping, "stomp.listener.cacertfile", "emqx_stomp.listener", [ - {datatype, string} -]}. - -{mapping, "stomp.listener.verify", "emqx_stomp.listener", [ - {datatype, string} -]}. - -{mapping, "stomp.listener.fail_if_no_peer_cert", "emqx_stomp.listener", [ - {datatype, {enum, [true, false]}} -]}. - -{mapping, "stomp.listener.ciphers", "emqx_stomp.listener", [ - {datatype, string} -]}. - -{mapping, "stomp.listener.secure_renegotiate", "emqx_stomp.listener", [ - {datatype, flag} -]}. - -{mapping, "stomp.listener.reuse_sessions", "emqx_stomp.listener", [ - {default, on}, - {datatype, flag} -]}. - -{mapping, "stomp.listener.honor_cipher_order", "emqx_stomp.listener", [ - {datatype, flag} -]}. - -{translation, "emqx_stomp.listener", fun(Conf) -> - Port = cuttlefish:conf_get("stomp.listener.port", Conf), - Acceptors = cuttlefish:conf_get("stomp.listener.acceptors", Conf), - MaxConnections = cuttlefish:conf_get("stomp.listener.max_connections", Conf), - Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, - SplitFun = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end, - SslOpts = fun(Prefix) -> - Versions = case SplitFun(cuttlefish:conf_get(Prefix ++ ".tls_versions", Conf, undefined)) of - undefined -> undefined; - L -> [list_to_atom(V) || V <- L] - end, - Filter([{versions, Versions}, - {ciphers, SplitFun(cuttlefish:conf_get(Prefix ++ ".ciphers", Conf, undefined))}, - {handshake_timeout, cuttlefish:conf_get(Prefix ++ ".handshake_timeout", Conf, undefined)}, - {dhfile, cuttlefish:conf_get(Prefix ++ ".dhfile", Conf, undefined)}, - {keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, - {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, - {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)}, - {verify, cuttlefish:conf_get(Prefix ++ ".verify", Conf, undefined)}, - {fail_if_no_peer_cert, cuttlefish:conf_get(Prefix ++ ".fail_if_no_peer_cert", Conf, undefined)}, - {secure_renegotiate, cuttlefish:conf_get(Prefix ++ ".secure_renegotiate", Conf, undefined)}, - {reuse_sessions, cuttlefish:conf_get(Prefix ++ ".reuse_sessions", Conf, undefined)}, - {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)}]) - end, - Opts = [{acceptors, Acceptors}, {max_connections, MaxConnections}], - {Port, case cuttlefish:conf_get("stomp.listener.ssl", Conf) of - true -> - [{sslopts, SslOpts("stomp.listener")} | Opts]; - false -> - Opts - end} -end}. - -{mapping, "stomp.default_user.login", "emqx_stomp.default_user", [ - {default, "guest"}, - {datatype, string} -]}. - -{mapping, "stomp.default_user.passcode", "emqx_stomp.default_user", [ - {default, "guest"}, - {datatype, string} -]}. - -{translation, "emqx_stomp.default_user", fun(Conf) -> - Login = cuttlefish:conf_get("stomp.default_user.login", Conf), - Passcode = cuttlefish:conf_get("stomp.default_user.passcode", Conf), - [{login, Login}, {passcode, Passcode}] -end}. - -{mapping, "stomp.allow_anonymous", "emqx_stomp.allow_anonymous", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "stomp.frame.max_headers", "emqx_stomp.frame", [ - {default, 10}, - {datatype, integer} -]}. - -{mapping, "stomp.frame.max_header_length", "emqx_stomp.frame", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "stomp.frame.max_body_length", "emqx_stomp.frame", [ - {default, 8192}, - {datatype, integer} -]}. - -{translation, "emqx_stomp.frame", fun(Conf) -> - MaxHeaders = cuttlefish:conf_get("stomp.frame.max_headers", Conf), - MaxHeaderLength = cuttlefish:conf_get("stomp.frame.max_header_length", Conf), - MaxBodyLength = cuttlefish:conf_get("stomp.frame.max_body_length", Conf), - [{max_headers, MaxHeaders}, {max_header_length, MaxHeaderLength}, {max_body_length, MaxBodyLength}] -end}. - diff --git a/apps/emqx_stomp/rebar.config b/apps/emqx_stomp/rebar.config deleted file mode 100644 index 7ac3b98c8..000000000 --- a/apps/emqx_stomp/rebar.config +++ /dev/null @@ -1,16 +0,0 @@ -{deps, []}. - -{edoc_opts, [{preprocess, true}]}. -{erl_opts, [warn_unused_vars, - warn_shadow_vars, - warn_unused_import, - warn_obsolete_guard, - debug_info, - {parse_transform}]}. - -{xref_checks, [undefined_function_calls, undefined_functions, - locals_not_used, deprecated_function_calls, - warnings_as_errors, deprecated_functions]}. -{cover_enabled, true}. -{cover_opts, [verbose]}. -{cover_export_enabled, true}. diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src deleted file mode 100644 index 2e66734ec..000000000 --- a/apps/emqx_stomp/src/emqx_stomp.app.src +++ /dev/null @@ -1,14 +0,0 @@ -{application, emqx_stomp, - [{description, "EMQ X Stomp Protocol Plugin"}, - {vsn, "4.4.0"}, % strict semver, bump manually! - {modules, []}, - {registered, [emqx_stomp_sup]}, - {applications, [kernel,stdlib]}, - {mod, {emqx_stomp,[]}}, - {env, []}, - {licenses, ["Apache-2.0"]}, - {maintainers, ["EMQ X Team "]}, - {links, [{"Homepage", "https://emqx.io/"}, - {"Github", "https://github.com/emqx/emqx-stomp"} - ]} - ]}. diff --git a/apps/emqx_stomp/src/emqx_stomp.erl b/apps/emqx_stomp/src/emqx_stomp.erl deleted file mode 100644 index 9eafe3cf7..000000000 --- a/apps/emqx_stomp/src/emqx_stomp.erl +++ /dev/null @@ -1,142 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_stomp). - --behaviour(application). --behaviour(supervisor). - --emqx_plugin(protocol). - --export([ start/2 - , stop/1 - ]). - --export([ start_listeners/0 - , start_listener/1 - , start_listener/3 - , stop_listeners/0 - , stop_listener/1 - , stop_listener/3 - ]). - --export([init/1]). - --define(APP, ?MODULE). --define(TCP_OPTS, [binary, {packet, raw}, {reuseaddr, true}, {nodelay, true}]). - --type(listener() :: {esockd:proto(), esockd:listen_on(), [esockd:option()]}). - -%%-------------------------------------------------------------------- -%% Application callbacks -%%-------------------------------------------------------------------- - -start(_StartType, _StartArgs) -> - {ok, Sup} = supervisor:start_link({local, emqx_stomp_sup}, ?MODULE, []), - start_listeners(), - {ok, Sup}. - -stop(_State) -> - stop_listeners(). - -%%-------------------------------------------------------------------- -%% Supervisor callbacks -%%-------------------------------------------------------------------- - -init([]) -> - {ok, {{one_for_all, 10, 100}, []}}. - -%%-------------------------------------------------------------------- -%% Start/Stop listeners -%%-------------------------------------------------------------------- - --spec(start_listeners() -> ok). -start_listeners() -> - lists:foreach(fun start_listener/1, listeners_confs()). - --spec(start_listener(listener()) -> ok). -start_listener({Proto, ListenOn, Options}) -> - case start_listener(Proto, ListenOn, Options) of - {ok, _} -> io:format("Start stomp:~s listener on ~s successfully.~n", - [Proto, format(ListenOn)]); - {error, Reason} -> - io:format(standard_error, "Failed to start stomp:~s listener on ~s: ~0p~n", - [Proto, format(ListenOn), Reason]), - error(Reason) - end. - --spec(start_listener(esockd:proto(), esockd:listen_on(), [esockd:option()]) - -> {ok, pid()} | {error, term()}). -start_listener(tcp, ListenOn, Options) -> - start_stomp_listener('stomp:tcp', ListenOn, Options); -start_listener(ssl, ListenOn, Options) -> - start_stomp_listener('stomp:ssl', ListenOn, Options). - -%% @private -start_stomp_listener(Name, ListenOn, Options) -> - SockOpts = esockd:parse_opt(Options), - esockd:open(Name, ListenOn, merge_default(SockOpts), - {emqx_stomp_connection, start_link, [Options -- SockOpts]}). - --spec(stop_listeners() -> ok). -stop_listeners() -> - lists:foreach(fun stop_listener/1, listeners_confs()). - --spec(stop_listener(listener()) -> ok | {error, term()}). -stop_listener({Proto, ListenOn, Opts}) -> - StopRet = stop_listener(Proto, ListenOn, Opts), - case StopRet of - ok -> io:format("Stop stomp:~s listener on ~s successfully.~n", - [Proto, format(ListenOn)]); - {error, Reason} -> - io:format(standard_error, "Failed to stop stomp:~s listener on ~s: ~0p~n", - [Proto, format(ListenOn), Reason]) - end, - StopRet. - --spec(stop_listener(esockd:proto(), esockd:listen_on(), [esockd:option()]) - -> ok | {error, term()}). -stop_listener(tcp, ListenOn, _Opts) -> - esockd:close('stomp:tcp', ListenOn); -stop_listener(ssl, ListenOn, _Opts) -> - esockd:close('stomp:ssl', ListenOn). - -%%-------------------------------------------------------------------- -%% Internal funcs -%%-------------------------------------------------------------------- - -listeners_confs() -> - {ok, {Port, Opts}} = application:get_env(?APP, listener), - Options = application:get_env(?APP, frame, []), - Anonymous = application:get_env(emqx_stomp, allow_anonymous, false), - {ok, DefaultUser} = application:get_env(emqx_stomp, default_user), - [{tcp, Port, [{allow_anonymous, Anonymous}, - {default_user, DefaultUser} | Options ++ Opts]}]. - -merge_default(Options) -> - case lists:keytake(tcp_options, 1, Options) of - {value, {tcp_options, TcpOpts}, Options1} -> - [{tcp_options, emqx_misc:merge_opts(?TCP_OPTS, TcpOpts)} | Options1]; - false -> - [{tcp_options, ?TCP_OPTS} | Options] - end. - -format(Port) when is_integer(Port) -> - io_lib:format("0.0.0.0:~w", [Port]); -format({Addr, Port}) when is_list(Addr) -> - io_lib:format("~s:~w", [Addr, Port]); -format({Addr, Port}) when is_tuple(Addr) -> - io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). diff --git a/apps/emqx_stomp/src/emqx_stomp_connection.erl b/apps/emqx_stomp/src/emqx_stomp_connection.erl deleted file mode 100644 index d4e7f6475..000000000 --- a/apps/emqx_stomp/src/emqx_stomp_connection.erl +++ /dev/null @@ -1,274 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_stomp_connection). - --behaviour(gen_server). - --include("emqx_stomp.hrl"). --include_lib("emqx/include/logger.hrl"). - --logger_header("[Stomp-Conn]"). - --export([ start_link/3 - , info/1 - ]). - -%% gen_server Function Exports --export([ init/1 - , handle_call/3 - , handle_cast/2 - , handle_info/2 - , code_change/3 - , terminate/2 - ]). - -%% for protocol --export([send/4, heartbeat/2]). - --record(state, {transport, socket, peername, conn_name, conn_state, - await_recv, rate_limit, parser, pstate, - proto_env, heartbeat}). - --define(INFO_KEYS, [peername, await_recv, conn_state]). --define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt]). - -start_link(Transport, Sock, ProtoEnv) -> - {ok, proc_lib:spawn_link(?MODULE, init, [[Transport, Sock, ProtoEnv]])}. - -info(CPid) -> - gen_server:call(CPid, info, infinity). - -init([Transport, Sock, ProtoEnv]) -> - process_flag(trap_exit, true), - case Transport:wait(Sock) of - {ok, NewSock} -> - {ok, Peername} = Transport:ensure_ok_or_exit(peername, [NewSock]), - ConnName = esockd:format(Peername), - SendFun = {fun ?MODULE:send/4, [Transport, Sock, self()]}, - HrtBtFun = {fun ?MODULE:heartbeat/2, [Transport, Sock]}, - Parser = emqx_stomp_frame:init_parer_state(ProtoEnv), - PState = emqx_stomp_protocol:init(#{peername => Peername, - sendfun => SendFun, - heartfun => HrtBtFun}, ProtoEnv), - RateLimit = init_rate_limit(proplists:get_value(rate_limit, ProtoEnv)), - State = run_socket(#state{transport = Transport, - socket = NewSock, - peername = Peername, - conn_name = ConnName, - conn_state = running, - await_recv = false, - rate_limit = RateLimit, - parser = Parser, - proto_env = ProtoEnv, - pstate = PState}), - emqx_logger:set_metadata_peername(esockd:format(Peername)), - gen_server:enter_loop(?MODULE, [{hibernate_after, 5000}], State, 20000); - {error, Reason} -> - {stop, Reason} - end. - -init_rate_limit(undefined) -> - undefined; -init_rate_limit({Rate, Burst}) -> - esockd_rate_limit:new(Rate, Burst). - -send(Data, Transport, Sock, ConnPid) -> - try Transport:async_send(Sock, Data) of - ok -> ok; - {error, Reason} -> ConnPid ! {shutdown, Reason} - catch - error:Error -> ConnPid ! {shutdown, Error} - end. - -heartbeat(Transport, Sock) -> - Transport:send(Sock, <<$\n>>). - -handle_call(info, _From, State = #state{transport = Transport, - socket = Sock, - peername = Peername, - await_recv = AwaitRecv, - conn_state = ConnState, - pstate = PState}) -> - ClientInfo = [{peername, Peername}, {await_recv, AwaitRecv}, - {conn_state, ConnState}], - ProtoInfo = emqx_stomp_protocol:info(PState), - case Transport:getstat(Sock, ?SOCK_STATS) of - {ok, SockStats} -> - {reply, lists:append([ClientInfo, ProtoInfo, SockStats]), State}; - {error, Reason} -> - {stop, Reason, lists:append([ClientInfo, ProtoInfo]), State} - end; - -handle_call(Req, _From, State) -> - ?LOG(error, "unexpected request: ~p", [Req]), - {reply, ignored, State}. - -handle_cast(Msg, State) -> - ?LOG(error, "unexpected msg: ~p", [Msg]), - noreply(State). - -handle_info(timeout, State) -> - shutdown(idle_timeout, State); - -handle_info({shutdown, Reason}, State) -> - shutdown(Reason, State); - -handle_info({timeout, TRef, TMsg}, State) when TMsg =:= incoming; - TMsg =:= outgoing -> - - Stat = case TMsg of - incoming -> recv_oct; - _ -> send_oct - end, - case getstat(Stat, State) of - {ok, Val} -> - with_proto(timeout, [TRef, {TMsg, Val}], State); - {error, Reason} -> - shutdown({sock_error, Reason}, State) - end; - -handle_info({timeout, TRef, TMsg}, State) -> - with_proto(timeout, [TRef, TMsg], State); - -handle_info({'EXIT', HbProc, Error}, State = #state{heartbeat = HbProc}) -> - stop(Error, State); - -handle_info(activate_sock, State) -> - noreply(run_socket(State#state{conn_state = running})); - -handle_info({inet_async, _Sock, _Ref, {ok, Bytes}}, State) -> - ?LOG(debug, "RECV ~p", [Bytes]), - received(Bytes, rate_limit(size(Bytes), State#state{await_recv = false})); - -handle_info({inet_async, _Sock, _Ref, {error, Reason}}, State) -> - shutdown(Reason, State); - -handle_info({inet_reply, _Ref, ok}, State) -> - noreply(State); - -handle_info({inet_reply, _Sock, {error, Reason}}, State) -> - shutdown(Reason, State); - -handle_info({deliver, _Topic, Msg}, State = #state{pstate = PState}) -> - noreply(State#state{pstate = case emqx_stomp_protocol:send(Msg, PState) of - {ok, PState1} -> - PState1; - {error, dropped, PState1} -> - PState1 - end}); - -handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), - noreply(State). - -terminate(Reason, #state{transport = Transport, - socket = Sock, - pstate = PState}) -> - ?LOG(info, "terminated for ~p", [Reason]), - Transport:fast_close(Sock), - case {PState, Reason} of - {undefined, _} -> ok; - {_, {shutdown, Error}} -> - emqx_stomp_protocol:shutdown(Error, PState); - {_, Reason} -> - emqx_stomp_protocol:shutdown(Reason, PState) - end. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Receive and Parse data -%%-------------------------------------------------------------------- - -with_proto(Fun, Args, State = #state{pstate = PState}) -> - case erlang:apply(emqx_stomp_protocol, Fun, Args ++ [PState]) of - {ok, NPState} -> - noreply(State#state{pstate = NPState}); - {F, Reason, NPState} when F == stop; - F == error; - F == shutdown -> - shutdown(Reason, State#state{pstate = NPState}) - end. - -received(<<>>, State) -> - noreply(State); - -received(Bytes, State = #state{parser = Parser, - pstate = PState}) -> - try emqx_stomp_frame:parse(Bytes, Parser) of - {more, NewParser} -> - noreply(State#state{parser = NewParser}); - {ok, Frame, Rest} -> - ?LOG(info, "RECV Frame: ~s", [emqx_stomp_frame:format(Frame)]), - case emqx_stomp_protocol:received(Frame, PState) of - {ok, PState1} -> - received(Rest, reset_parser(State#state{pstate = PState1})); - {error, Error, PState1} -> - shutdown(Error, State#state{pstate = PState1}); - {stop, Reason, PState1} -> - stop(Reason, State#state{pstate = PState1}) - end; - {error, Error} -> - ?LOG(error, "Framing error - ~s", [Error]), - ?LOG(error, "Bytes: ~p", [Bytes]), - shutdown(frame_error, State) - catch - _Error:Reason -> - ?LOG(error, "Parser failed for ~p", [Reason]), - ?LOG(error, "Error data: ~p", [Bytes]), - shutdown(parse_error, State) - end. - -reset_parser(State = #state{proto_env = ProtoEnv}) -> - State#state{parser = emqx_stomp_frame:init_parer_state(ProtoEnv)}. - -rate_limit(_Size, State = #state{rate_limit = undefined}) -> - run_socket(State); -rate_limit(Size, State = #state{rate_limit = Rl}) -> - case esockd_rate_limit:check(Size, Rl) of - {0, Rl1} -> - run_socket(State#state{conn_state = running, rate_limit = Rl1}); - {Pause, Rl1} -> - ?LOG(error, "Rate limiter pause for ~p", [Pause]), - erlang:send_after(Pause, self(), activate_sock), - State#state{conn_state = blocked, rate_limit = Rl1} - end. - -run_socket(State = #state{conn_state = blocked}) -> - State; -run_socket(State = #state{await_recv = true}) -> - State; -run_socket(State = #state{transport = Transport, socket = Sock}) -> - Transport:async_recv(Sock, 0, infinity), - State#state{await_recv = true}. - -getstat(Stat, #state{transport = Transport, socket = Sock}) -> - case Transport:getstat(Sock, [Stat]) of - {ok, [{Stat, Val}]} -> {ok, Val}; - {error, Error} -> {error, Error} - end. - -noreply(State) -> - {noreply, State}. - -stop(Reason, State) -> - {stop, Reason, State}. - -shutdown(Reason, State) -> - stop({shutdown, Reason}, State). - diff --git a/apps/emqx_stomp/src/emqx_stomp_protocol.erl b/apps/emqx_stomp/src/emqx_stomp_protocol.erl deleted file mode 100644 index cc5c28ce9..000000000 --- a/apps/emqx_stomp/src/emqx_stomp_protocol.erl +++ /dev/null @@ -1,468 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc Stomp Protocol Processor. --module(emqx_stomp_protocol). - --include("emqx_stomp.hrl"). - --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). - --logger_header("[Stomp-Proto]"). - --import(proplists, [get_value/2, get_value/3]). - -%% API --export([ init/2 - , info/1 - ]). - --export([ received/2 - , send/2 - , shutdown/2 - , timeout/3 - ]). - -%% for trans callback --export([ handle_recv_send_frame/2 - , handle_recv_ack_frame/2 - , handle_recv_nack_frame/2 - ]). - --record(pstate, { - peername, - heartfun, - sendfun, - connected = false, - proto_ver, - proto_name, - heart_beats, - login, - allow_anonymous, - default_user, - subscriptions = [], - timers :: #{atom() => disable | undefined | reference()}, - transaction :: #{binary() => list()} - }). - --define(TIMER_TABLE, #{ - incoming_timer => incoming, - outgoing_timer => outgoing, - clean_trans_timer => clean_trans - }). - --define(TRANS_TIMEOUT, 60000). - --type(pstate() :: #pstate{}). - -%% @doc Init protocol -init(#{peername := Peername, - sendfun := SendFun, - heartfun := HeartFun}, Env) -> - AllowAnonymous = get_value(allow_anonymous, Env, false), - DefaultUser = get_value(default_user, Env), - #pstate{peername = Peername, - heartfun = HeartFun, - sendfun = SendFun, - timers = #{}, - transaction = #{}, - allow_anonymous = AllowAnonymous, - default_user = DefaultUser}. - -info(#pstate{connected = Connected, - proto_ver = ProtoVer, - proto_name = ProtoName, - heart_beats = Heartbeats, - login = Login, - subscriptions = Subscriptions}) -> - [{connected, Connected}, - {proto_ver, ProtoVer}, - {proto_name, ProtoName}, - {heart_beats, Heartbeats}, - {login, Login}, - {subscriptions, Subscriptions}]. - --spec(received(stomp_frame(), pstate()) - -> {ok, pstate()} - | {error, any(), pstate()} - | {stop, any(), pstate()}). -received(Frame = #stomp_frame{command = <<"STOMP">>}, State) -> - received(Frame#stomp_frame{command = <<"CONNECT">>}, State); - -received(#stomp_frame{command = <<"CONNECT">>, headers = Headers}, - State = #pstate{connected = false, allow_anonymous = AllowAnonymous, default_user = DefaultUser}) -> - case negotiate_version(header(<<"accept-version">>, Headers)) of - {ok, Version} -> - Login = header(<<"login">>, Headers), - Passc = header(<<"passcode">>, Headers), - case check_login(Login, Passc, AllowAnonymous, DefaultUser) of - true -> - emqx_logger:set_metadata_clientid(Login), - - Heartbeats = parse_heartbeats(header(<<"heart-beat">>, Headers, <<"0,0">>)), - NState = start_heartbeart_timer(Heartbeats, State#pstate{connected = true, - proto_ver = Version, login = Login}), - send(connected_frame([{<<"version">>, Version}, - {<<"heart-beat">>, reverse_heartbeats(Heartbeats)}]), NState); - false -> - _ = send(error_frame(undefined, <<"Login or passcode error!">>), State), - {error, login_or_passcode_error, State} - end; - {error, Msg} -> - _ = send(error_frame([{<<"version">>, <<"1.0,1.1,1.2">>}, - {<<"content-type">>, <<"text/plain">>}], undefined, Msg), State), - {error, unsupported_version, State} - end; - -received(#stomp_frame{command = <<"CONNECT">>}, State = #pstate{connected = true}) -> - {error, unexpected_connect, State}; - -received(Frame = #stomp_frame{command = <<"SEND">>, headers = Headers}, State) -> - case header(<<"transaction">>, Headers) of - undefined -> {ok, handle_recv_send_frame(Frame, State)}; - TransactionId -> add_action(TransactionId, {fun ?MODULE:handle_recv_send_frame/2, [Frame]}, receipt_id(Headers), State) - end; - -received(#stomp_frame{command = <<"SUBSCRIBE">>, headers = Headers}, - State = #pstate{subscriptions = Subscriptions}) -> - Id = header(<<"id">>, Headers), - Topic = header(<<"destination">>, Headers), - Ack = header(<<"ack">>, Headers, <<"auto">>), - {ok, State1} = case lists:keyfind(Id, 1, Subscriptions) of - {Id, Topic, Ack} -> - {ok, State}; - false -> - emqx_broker:subscribe(Topic), - {ok, State#pstate{subscriptions = [{Id, Topic, Ack}|Subscriptions]}} - end, - maybe_send_receipt(receipt_id(Headers), State1); - -received(#stomp_frame{command = <<"UNSUBSCRIBE">>, headers = Headers}, - State = #pstate{subscriptions = Subscriptions}) -> - Id = header(<<"id">>, Headers), - - {ok, State1} = case lists:keyfind(Id, 1, Subscriptions) of - {Id, Topic, _Ack} -> - ok = emqx_broker:unsubscribe(Topic), - {ok, State#pstate{subscriptions = lists:keydelete(Id, 1, Subscriptions)}}; - false -> - {ok, State} - end, - maybe_send_receipt(receipt_id(Headers), State1); - -%% ACK -%% id:12345 -%% transaction:tx1 -%% -%% ^@ -received(Frame = #stomp_frame{command = <<"ACK">>, headers = Headers}, State) -> - case header(<<"transaction">>, Headers) of - undefined -> {ok, handle_recv_ack_frame(Frame, State)}; - TransactionId -> add_action(TransactionId, {fun ?MODULE:handle_recv_ack_frame/2, [Frame]}, receipt_id(Headers), State) - end; - -%% NACK -%% id:12345 -%% transaction:tx1 -%% -%% ^@ -received(Frame = #stomp_frame{command = <<"NACK">>, headers = Headers}, State) -> - case header(<<"transaction">>, Headers) of - undefined -> {ok, handle_recv_nack_frame(Frame, State)}; - TransactionId -> add_action(TransactionId, {fun ?MODULE:handle_recv_nack_frame/2, [Frame]}, receipt_id(Headers), State) - end; - -%% BEGIN -%% transaction:tx1 -%% -%% ^@ -received(#stomp_frame{command = <<"BEGIN">>, headers = Headers}, - State = #pstate{transaction = Trans}) -> - Id = header(<<"transaction">>, Headers), - case maps:get(Id, Trans, undefined) of - undefined -> - Ts = erlang:system_time(millisecond), - NState = ensure_clean_trans_timer(State#pstate{transaction = Trans#{Id => {Ts, []}}}), - maybe_send_receipt(receipt_id(Headers), NState); - _ -> - send(error_frame(receipt_id(Headers), ["Transaction ", Id, " already started"]), State) - end; - -%% COMMIT -%% transaction:tx1 -%% -%% ^@ -received(#stomp_frame{command = <<"COMMIT">>, headers = Headers}, - State = #pstate{transaction = Trans}) -> - Id = header(<<"transaction">>, Headers), - case maps:get(Id, Trans, undefined) of - {_, Actions} -> - NState = lists:foldr(fun({Func, Args}, S) -> - erlang:apply(Func, Args ++ [S]) - end, State#pstate{transaction = maps:remove(Id, Trans)}, Actions), - maybe_send_receipt(receipt_id(Headers), NState); - _ -> - send(error_frame(receipt_id(Headers), ["Transaction ", Id, " not found"]), State) - end; - -%% ABORT -%% transaction:tx1 -%% -%% ^@ -received(#stomp_frame{command = <<"ABORT">>, headers = Headers}, - State = #pstate{transaction = Trans}) -> - Id = header(<<"transaction">>, Headers), - case maps:get(Id, Trans, undefined) of - {_, _Actions} -> - NState = State#pstate{transaction = maps:remove(Id, Trans)}, - maybe_send_receipt(receipt_id(Headers), NState); - _ -> - send(error_frame(receipt_id(Headers), ["Transaction ", Id, " not found"]), State) - end; - -received(#stomp_frame{command = <<"DISCONNECT">>, headers = Headers}, State) -> - _ = maybe_send_receipt(receipt_id(Headers), State), - {stop, normal, State}. - -send(Msg = #message{topic = Topic, headers = Headers, payload = Payload}, - State = #pstate{subscriptions = Subscriptions}) -> - case lists:keyfind(Topic, 2, Subscriptions) of - {Id, Topic, Ack} -> - Headers0 = [{<<"subscription">>, Id}, - {<<"message-id">>, next_msgid()}, - {<<"destination">>, Topic}, - {<<"content-type">>, <<"text/plain">>}], - Headers1 = case Ack of - _ when Ack =:= <<"client">> orelse Ack =:= <<"client-individual">> -> - Headers0 ++ [{<<"ack">>, next_ackid()}]; - _ -> - Headers0 - end, - Frame = #stomp_frame{command = <<"MESSAGE">>, - headers = Headers1 ++ maps:get(stomp_headers, Headers, []), - body = Payload}, - send(Frame, State); - false -> - ?LOG(error, "Stomp dropped: ~p", [Msg]), - {error, dropped, State} - end; - -send(Frame, State = #pstate{sendfun = {Fun, Args}}) -> - ?LOG(info, "SEND Frame: ~s", [emqx_stomp_frame:format(Frame)]), - Data = emqx_stomp_frame:serialize(Frame), - ?LOG(debug, "SEND ~p", [Data]), - erlang:apply(Fun, [Data] ++ Args), - {ok, State}. - -shutdown(_Reason, _State) -> - ok. - -timeout(_TRef, {incoming, NewVal}, - State = #pstate{heart_beats = HrtBt}) -> - case emqx_stomp_heartbeat:check(incoming, NewVal, HrtBt) of - {error, timeout} -> - {shutdown, heartbeat_timeout, State}; - {ok, NHrtBt} -> - {ok, reset_timer(incoming_timer, State#pstate{heart_beats = NHrtBt})} - end; - -timeout(_TRef, {outgoing, NewVal}, - State = #pstate{heart_beats = HrtBt, - heartfun = {Fun, Args}}) -> - case emqx_stomp_heartbeat:check(outgoing, NewVal, HrtBt) of - {error, timeout} -> - _ = erlang:apply(Fun, Args), - {ok, State}; - {ok, NHrtBt} -> - {ok, reset_timer(outgoing_timer, State#pstate{heart_beats = NHrtBt})} - end; - -timeout(_TRef, clean_trans, State = #pstate{transaction = Trans}) -> - Now = erlang:system_time(millisecond), - NTrans = maps:filter(fun(_, {Ts, _}) -> Ts + ?TRANS_TIMEOUT < Now end, Trans), - {ok, ensure_clean_trans_timer(State#pstate{transaction = NTrans})}. - -negotiate_version(undefined) -> - {ok, <<"1.0">>}; -negotiate_version(Accepts) -> - negotiate_version(?STOMP_VER, - lists:reverse( - lists:sort( - binary:split(Accepts, <<",">>, [global])))). - -negotiate_version(Ver, []) -> - {error, <<"Supported protocol versions < ", Ver/binary>>}; -negotiate_version(Ver, [AcceptVer|_]) when Ver >= AcceptVer -> - {ok, AcceptVer}; -negotiate_version(Ver, [_|T]) -> - negotiate_version(Ver, T). - -check_login(undefined, _, AllowAnonymous, _) -> - AllowAnonymous; -check_login(_, _, _, undefined) -> - false; -check_login(Login, Passcode, _, DefaultUser) -> - case {list_to_binary(get_value(login, DefaultUser)), - list_to_binary(get_value(passcode, DefaultUser))} of - {Login, Passcode} -> true; - {_, _ } -> false - end. - -add_action(Id, Action, ReceiptId, State = #pstate{transaction = Trans}) -> - case maps:get(Id, Trans, undefined) of - {Ts, Actions} -> - NTrans = Trans#{Id => {Ts, [Action|Actions]}}, - {ok, State#pstate{transaction = NTrans}}; - _ -> - send(error_frame(ReceiptId, ["Transaction ", Id, " not found"]), State) - end. - -maybe_send_receipt(undefined, State) -> - {ok, State}; -maybe_send_receipt(ReceiptId, State) -> - send(receipt_frame(ReceiptId), State). - -ack(_Id, State) -> - State. - -nack(_Id, State) -> State. - -header(Name, Headers) -> - get_value(Name, Headers). -header(Name, Headers, Val) -> - get_value(Name, Headers, Val). - -connected_frame(Headers) -> - emqx_stomp_frame:make(<<"CONNECTED">>, Headers). - -receipt_frame(ReceiptId) -> - emqx_stomp_frame:make(<<"RECEIPT">>, [{<<"receipt-id">>, ReceiptId}]). - -error_frame(ReceiptId, Msg) -> - error_frame([{<<"content-type">>, <<"text/plain">>}], ReceiptId, Msg). - -error_frame(Headers, undefined, Msg) -> - emqx_stomp_frame:make(<<"ERROR">>, Headers, Msg); -error_frame(Headers, ReceiptId, Msg) -> - emqx_stomp_frame:make(<<"ERROR">>, [{<<"receipt-id">>, ReceiptId} | Headers], Msg). - -next_msgid() -> - MsgId = case get(msgid) of - undefined -> 1; - I -> I - end, - put(msgid, MsgId + 1), - MsgId. - -next_ackid() -> - AckId = case get(ackid) of - undefined -> 1; - I -> I - end, - put(ackid, AckId + 1), - AckId. - -make_mqtt_message(Topic, Headers, Body) -> - Msg = emqx_message:make(stomp, Topic, Body), - Headers1 = lists:foldl(fun(Key, Headers0) -> - proplists:delete(Key, Headers0) - end, Headers, [<<"destination">>, - <<"content-length">>, - <<"content-type">>, - <<"transaction">>, - <<"receipt">>]), - emqx_message:set_headers(#{stomp_headers => Headers1}, Msg). - -receipt_id(Headers) -> - header(<<"receipt">>, Headers). - -%%-------------------------------------------------------------------- -%% Transaction Handle - -handle_recv_send_frame(#stomp_frame{command = <<"SEND">>, headers = Headers, body = Body}, State) -> - Topic = header(<<"destination">>, Headers), - _ = maybe_send_receipt(receipt_id(Headers), State), - _ = emqx_broker:publish( - make_mqtt_message(Topic, Headers, iolist_to_binary(Body)) - ), - State. - -handle_recv_ack_frame(#stomp_frame{command = <<"ACK">>, headers = Headers}, State) -> - Id = header(<<"id">>, Headers), - _ = maybe_send_receipt(receipt_id(Headers), State), - ack(Id, State). - -handle_recv_nack_frame(#stomp_frame{command = <<"NACK">>, headers = Headers}, State) -> - Id = header(<<"id">>, Headers), - _ = maybe_send_receipt(receipt_id(Headers), State), - nack(Id, State). - -ensure_clean_trans_timer(State = #pstate{transaction = Trans}) -> - case maps:size(Trans) of - 0 -> State; - _ -> ensure_timer(clean_trans_timer, State) - end. - -%%-------------------------------------------------------------------- -%% Heartbeat - -parse_heartbeats(Heartbeats) -> - CxCy = re:split(Heartbeats, <<",">>, [{return, list}]), - list_to_tuple([list_to_integer(S) || S <- CxCy]). - -reverse_heartbeats({Cx, Cy}) -> - iolist_to_binary(io_lib:format("~w,~w", [Cy, Cx])). - -start_heartbeart_timer(Heartbeats, State) -> - ensure_timer( - [incoming_timer, outgoing_timer], - State#pstate{heart_beats = emqx_stomp_heartbeat:init(Heartbeats)}). - -%%-------------------------------------------------------------------- -%% Timer - -ensure_timer([Name], State) -> - ensure_timer(Name, State); -ensure_timer([Name | Rest], State) -> - ensure_timer(Rest, ensure_timer(Name, State)); - -ensure_timer(Name, State = #pstate{timers = Timers}) -> - TRef = maps:get(Name, Timers, undefined), - Time = interval(Name, State), - case TRef == undefined andalso is_integer(Time) andalso Time > 0 of - true -> ensure_timer(Name, Time, State); - false -> State %% Timer disabled or exists - end. - -ensure_timer(Name, Time, State = #pstate{timers = Timers}) -> - Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), - State#pstate{timers = Timers#{Name => TRef}}. - -reset_timer(Name, State) -> - ensure_timer(Name, clean_timer(Name, State)). - -clean_timer(Name, State = #pstate{timers = Timers}) -> - State#pstate{timers = maps:remove(Name, Timers)}. - -interval(incoming_timer, #pstate{heart_beats = HrtBt}) -> - emqx_stomp_heartbeat:interval(incoming, HrtBt); -interval(outgoing_timer, #pstate{heart_beats = HrtBt}) -> - emqx_stomp_heartbeat:interval(outgoing, HrtBt); -interval(clean_trans_timer, _) -> - ?TRANS_TIMEOUT. diff --git a/apps/emqx_stomp/test/client.py b/apps/emqx_stomp/test/client.py deleted file mode 100644 index f9f9e6577..000000000 --- a/apps/emqx_stomp/test/client.py +++ /dev/null @@ -1,19 +0,0 @@ -from stompest.config import StompConfig -from stompest.protocol import StompSpec -from stompest.sync import Stomp - -CONFIG = StompConfig('tcp://localhost:61613', version=StompSpec.VERSION_1_1) -QUEUE = '/queue/test' - -if __name__ == '__main__': - client = Stomp(CONFIG) - client.connect(heartBeats=(0, 10000)) - client.subscribe(QUEUE, {StompSpec.ID_HEADER: 1, StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}) - client.send(QUEUE, 'test message 1') - client.send(QUEUE, 'test message 2') - while True: - frame = client.receiveFrame() - print 'Got %s' % frame.info() - client.ack(frame) - client.disconnect() - diff --git a/apps/emqx_telemetry/.gitignore b/apps/emqx_telemetry/.gitignore deleted file mode 100644 index e1deda1b9..000000000 --- a/apps/emqx_telemetry/.gitignore +++ /dev/null @@ -1,26 +0,0 @@ -.eunit -deps -*.o -*.beam -*.plt -erl_crash.dump -ebin -rel/example_project -.concrete/DEV_MODE -.rebar -data/ -*.swp -*.swo -.erlang.mk/ -emqx_retainer.d -erlang.mk -rebar3.crashdump -_build -cover/ -ct.coverdata -eunit.coverdata -logs/ -rebar.lock -test/ct.cover.spec -etc/emqx_telemetry.conf.rendered -.rebar3/ \ No newline at end of file diff --git a/apps/emqx_telemetry/README.md b/apps/emqx_telemetry/README.md deleted file mode 100644 index 7c1bf3f43..000000000 --- a/apps/emqx_telemetry/README.md +++ /dev/null @@ -1 +0,0 @@ -# emqx-telemetry \ No newline at end of file diff --git a/apps/emqx_telemetry/etc/emqx_telemetry.conf b/apps/emqx_telemetry/etc/emqx_telemetry.conf deleted file mode 100644 index bbe4a2fd4..000000000 --- a/apps/emqx_telemetry/etc/emqx_telemetry.conf +++ /dev/null @@ -1,3 +0,0 @@ -emqx_telemetry:{ - enabled: true -} diff --git a/apps/emqx_telemetry/include/emqx_telemetry.hrl b/apps/emqx_telemetry/include/emqx_telemetry.hrl deleted file mode 100644 index 334173015..000000000 --- a/apps/emqx_telemetry/include/emqx_telemetry.hrl +++ /dev/null @@ -1,5 +0,0 @@ -%% The destination URL for the telemetry data report --define(TELEMETRY_URL, "https://telemetry.emqx.io/api/telemetry"). - -%% Interval for reporting telemetry data, Default: 7d --define(REPORT_INTERVAR, 604800). diff --git a/apps/emqx_telemetry/rebar.config b/apps/emqx_telemetry/rebar.config deleted file mode 100644 index 7b30a8fd8..000000000 --- a/apps/emqx_telemetry/rebar.config +++ /dev/null @@ -1 +0,0 @@ -{deps, []}. diff --git a/apps/emqx_telemetry/src/emqx_telemetry.app.src b/apps/emqx_telemetry/src/emqx_telemetry.app.src deleted file mode 100644 index 0f6bee95c..000000000 --- a/apps/emqx_telemetry/src/emqx_telemetry.app.src +++ /dev/null @@ -1,14 +0,0 @@ -{application, emqx_telemetry, - [{description, "EMQ X Telemetry"}, - {vsn, "5.0.0"}, % strict semver, bump manually! - {modules, []}, - {registered, [emqx_telemetry_sup]}, - {applications, [kernel,stdlib]}, - {mod, {emqx_telemetry_app,[]}}, - {env, []}, - {licenses, ["Apache-2.0"]}, - {maintainers, ["EMQ X Team "]}, - {links, [{"Homepage", "https://emqx.io/"}, - {"Github", "https://github.com/emqx/emqx-telemetry"} - ]} - ]}. diff --git a/apps/emqx_telemetry/src/emqx_telemetry_api.erl b/apps/emqx_telemetry/src/emqx_telemetry_api.erl deleted file mode 100644 index 8bb97086e..000000000 --- a/apps/emqx_telemetry/src/emqx_telemetry_api.erl +++ /dev/null @@ -1,131 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_telemetry_api). - --rest_api(#{name => enable_telemetry, - method => 'PUT', - path => "/telemetry/status", - func => enable, - descr => "Enable or disbale telemetry"}). - --rest_api(#{name => get_telemetry_status, - method => 'GET', - path => "/telemetry/status", - func => get_status, - descr => "Get telemetry status"}). - --rest_api(#{name => get_telemetry_data, - method => 'GET', - path => "/telemetry/data", - func => get_data, - descr => "Get reported telemetry data"}). - --export([ cli/1 - , enable/2 - , get_status/2 - , get_data/2 - , enable_telemetry/1 - , disable_telemetry/1 - , get_telemetry_status/0 - , get_telemetry_data/0 - ]). - --import(minirest, [return/1]). - -%%-------------------------------------------------------------------- -%% CLI -%%-------------------------------------------------------------------- - -cli(["enable"]) -> - enable_telemetry(), - emqx_ctl:print("Enable telemetry successfully~n"); - -cli(["disable"]) -> - disable_telemetry(), - emqx_ctl:print("Disable telemetry successfully~n"); - -cli(["get", "status"]) -> - case get_telemetry_status() of - [{enabled, true}] -> - emqx_ctl:print("Telemetry is enabled~n"); - [{enabled, false}] -> - emqx_ctl:print("Telemetry is disabled~n") - end; - -cli(["get", "data"]) -> - {ok, TelemetryData} = get_telemetry_data(), - case emqx_json:safe_encode(TelemetryData, [pretty]) of - {ok, Bin} -> - emqx_ctl:print("~s~n", [Bin]); - {error, _Reason} -> - emqx_ctl:print("Failed to get telemetry data") - end; - -cli(_) -> - emqx_ctl:usage([{"telemetry enable", "Enable telemetry"}, - {"telemetry disable", "Disable telemetry"}, - {"telemetry get data", "Get reported telemetry data"}]). - -%%-------------------------------------------------------------------- -%% HTTP API -%%-------------------------------------------------------------------- - -enable(_Bindings, Params) -> - case proplists:get_value(<<"enabled">>, Params) of - true -> - enable_telemetry(), - return(ok); - false -> - disable_telemetry(), - return(ok); - undefined -> - return({error, missing_required_params}) - end. - -get_status(_Bindings, _Params) -> - return({ok, get_telemetry_status()}). - -get_data(_Bindings, _Params) -> - return(get_telemetry_data()). - -enable_telemetry() -> - lists:foreach(fun enable_telemetry/1, ekka_mnesia:running_nodes()). - -enable_telemetry(Node) when Node =:= node() -> - emqx_telemetry:enable(); -enable_telemetry(Node) -> - rpc_call(Node, ?MODULE, enable_telemetry, [Node]). - -disable_telemetry() -> - lists:foreach(fun disable_telemetry/1, ekka_mnesia:running_nodes()). - -disable_telemetry(Node) when Node =:= node() -> - emqx_telemetry:disable(); -disable_telemetry(Node) -> - rpc_call(Node, ?MODULE, disable_telemetry, [Node]). - -get_telemetry_status() -> - [{enabled, emqx_telemetry:is_enabled()}]. - -get_telemetry_data() -> - emqx_telemetry:get_telemetry(). - -rpc_call(Node, Module, Fun, Args) -> - case rpc:call(Node, Module, Fun, Args) of - {badrpc, Reason} -> {error, Reason}; - Result -> Result - end. diff --git a/apps/emqx_telemetry/src/emqx_telemetry_schema.erl b/apps/emqx_telemetry/src/emqx_telemetry_schema.erl deleted file mode 100644 index 4d5cab684..000000000 --- a/apps/emqx_telemetry/src/emqx_telemetry_schema.erl +++ /dev/null @@ -1,13 +0,0 @@ --module(emqx_telemetry_schema). - --include_lib("typerefl/include/types.hrl"). - --behaviour(hocon_schema). - --export([ structs/0 - , fields/1]). - -structs() -> ["emqx_telemetry"]. - -fields("emqx_telemetry") -> - [{enabled, emqx_schema:t(boolean(), undefined, false)}]. \ No newline at end of file diff --git a/apps/emqx_telemetry/src/emqx_telemetry_sup.erl b/apps/emqx_telemetry/src/emqx_telemetry_sup.erl deleted file mode 100644 index da17c7a67..000000000 --- a/apps/emqx_telemetry/src/emqx_telemetry_sup.erl +++ /dev/null @@ -1,35 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_telemetry_sup). - --behaviour(supervisor). - --export([start_link/1]). - --export([init/1]). - -start_link(Env) -> - supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]). - -init([Env]) -> - {ok, {{one_for_one, 10, 3600}, - [#{id => telemetry, - start => {emqx_telemetry, start_link, [Env]}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_telemetry]}]}}. \ No newline at end of file diff --git a/apps/emqx_web_hook/.gitignore b/apps/emqx_web_hook/.gitignore deleted file mode 100644 index e6348348a..000000000 --- a/apps/emqx_web_hook/.gitignore +++ /dev/null @@ -1,31 +0,0 @@ -.rebar3 -_* -.eunit -*.o -*.beam -*.plt -*.swp -*.swo -.erlang.cookie -ebin -log -erl_crash.dump -.rebar -logs -_build -.idea -data -.DS_Store -.erlang.mk/ -cover -ct.coverdata -deps -eunit.coverdata -test/ct.cover.spec -emqx_web_hook.d -emq_web_hook.d -rebar.lock -erlang.mk -rebar3.crashdump -etc/emqx_web_hook.conf.rendered -Mnesia.nonode@nohost diff --git a/apps/emqx_web_hook/README.md b/apps/emqx_web_hook/README.md deleted file mode 100644 index c76c2936d..000000000 --- a/apps/emqx_web_hook/README.md +++ /dev/null @@ -1,194 +0,0 @@ - -# emqx-web-hook - -EMQ X WebHook plugin. - -Please see: [EMQ X - WebHook](https://docs.emqx.io/broker/latest/en/advanced/webhook.html) - -## emqx_web_hook.conf - -```properties -## The web services URL for Hook request -## -## Value: String -web.hook.url = http://127.0.0.1:8080 - -## Encode message payload field -## -## Value: base64 | base62 -## web.hook.encode_payload = base64 - -##-------------------------------------------------------------------- -## Hook Rules - -## These configuration items represent a list of events should be forwarded -## -## Format: -## web.hook.rule.. = -web.hook.rule.client.connect.1 = {"action": "on_client_connect"} -web.hook.rule.client.connack.1 = {"action": "on_client_connack"} -web.hook.rule.client.connected.1 = {"action": "on_client_connected"} -web.hook.rule.client.disconnected.1 = {"action": "on_client_disconnected"} -web.hook.rule.client.subscribe.1 = {"action": "on_client_subscribe"} -web.hook.rule.client.unsubscribe.1 = {"action": "on_client_unsubscribe"} -web.hook.rule.session.subscribed.1 = {"action": "on_session_subscribed"} -web.hook.rule.session.unsubscribed.1 = {"action": "on_session_unsubscribed"} -web.hook.rule.session.terminated.1 = {"action": "on_session_terminated"} -web.hook.rule.message.publish.1 = {"action": "on_message_publish"} -web.hook.rule.message.delivered.1 = {"action": "on_message_delivered"} -web.hook.rule.message.acked.1 = {"action": "on_message_acked"} -``` - -## API - -The HTTP request parameter format: - -* client.connected -```json -{ - "action":"client_connected", - "clientid":"C_1492410235117", - "username":"C_1492410235117", - "keepalive": 60, - "ipaddress": "127.0.0.1", - "proto_ver": 4, - "connected_at": 1556176748, - "conn_ack":0 -} -``` - -* client.disconnected -```json -{ - "action":"client_disconnected", - "clientid":"C_1492410235117", - "username":"C_1492410235117", - "reason":"normal" -} -``` - -* client.subscribe -```json -{ - "action":"client_subscribe", - "clientid":"C_1492410235117", - "username":"C_1492410235117", - "topic":"world", - "opts":{ - "qos":0 - } -} -``` - -* client.unsubscribe -```json -{ - "action":"client_unsubscribe", - "clientid":"C_1492410235117", - "username":"C_1492410235117", - "topic":"world" -} -``` - -* session.created -```json -{ - "action":"session_created", - "clientid":"C_1492410235117", - "username":"C_1492410235117" -} -``` - -* session.subscribed -```json -{ - "action":"session_subscribed", - "clientid":"C_1492410235117", - "username":"C_1492410235117", - "topic":"world", - "opts":{ - "qos":0 - } -} -``` - -* session.unsubscribed -```json -{ - "action":"session_unsubscribed", - "clientid":"C_1492410235117", - "username":"C_1492410235117", - "topic":"world" -} -``` - -* session.terminated -```json -{ - "action":"session_terminated", - "clientid":"C_1492410235117", - "username":"C_1492410235117", - "reason":"normal" -} -``` - -* message.publish -```json -{ - "action":"message_publish", - "from_client_id":"C_1492410235117", - "from_username":"C_1492410235117", - "topic":"world", - "qos":0, - "retain":true, - "payload":"Hello world!", - "ts":1492412774 -} -``` - -* message.delivered -```json -{ - "action":"message_delivered", - "clientid":"C_1492410235117", - "username":"C_1492410235117", - "from_client_id":"C_1492410235117", - "from_username":"C_1492410235117", - "topic":"world", - "qos":0, - "retain":true, - "payload":"Hello world!", - "ts":1492412826 -} -``` - -* message.acked -```json -{ - "action":"message_acked", - "clientid":"C_1492410235117", - "username":"C_1492410235117", - "from_client_id":"C_1492410235117", - "from_username":"C_1492410235117", - "topic":"world", - "qos":1, - "retain":true, - "payload":"Hello world!", - "ts":1492412914 -} -``` - -## License - -Apache License Version 2.0 - -## Author - -* [Sakib Sami](https://github.com/s4kibs4mi) - -## Contributors - -* [Deng](https://github.com/turtleDeng) -* [vishr](https://github.com/vishr) -* [emqplus](https://github.com/emqplus) -* [huangdan](https://github.com/huangdan) diff --git a/apps/emqx_web_hook/TODO b/apps/emqx_web_hook/TODO deleted file mode 100644 index 31bf5a2ad..000000000 --- a/apps/emqx_web_hook/TODO +++ /dev/null @@ -1,3 +0,0 @@ -1. HTTPS -2. More HTTP Headers and Options -3. MQTT 5.0 diff --git a/apps/emqx_web_hook/etc/emqx_web_hook.conf b/apps/emqx_web_hook/etc/emqx_web_hook.conf deleted file mode 100644 index 6707e4673..000000000 --- a/apps/emqx_web_hook/etc/emqx_web_hook.conf +++ /dev/null @@ -1,77 +0,0 @@ -##==================================================================== -## WebHook -##==================================================================== - -## Webhook URL -## -## Value: String -web.hook.url = "http://127.0.0.1:80" - -## HTTP Headers -## -## Example: -## 1. web.hook.headers.content-type = "application/json" -## 2. web.hook.headers.accept = "*" -## -## Value: String -web.hook.headers.content-type = "application/json" - -## The encoding format of the payload field in the HTTP body -## The payload field only appears in the on_message_publish and on_message_delivered actions -## -## Value: plain | base64 | base62 -web.hook.body.encoding_of_payload_field = plain - -##-------------------------------------------------------------------- -## PEM format file of CA's -## -## Value: File -## web.hook.ssl.cacertfile = - -## Certificate file to use, PEM format assumed -## -## Value: File -## web.hook.ssl.certfile = - -## Private key file to use, PEM format assumed -## -## Value: File -## web.hook.ssl.keyfile = - -## Turn on peer certificate verification -## -## Value: true | false -## web.hook.ssl.verify = false - -## If not specified, the server's names returned in server's certificate is validated against -## what's provided `web.hook.url` config's host part. -## Setting to 'disable' will make EMQ X ignore unmatched server names. -## If set with a host name, the server's names returned in server's certificate is validated -## against this value. -## -## Value: String | disable -## web.hook.ssl.server_name_indication = disable - -## Connection process pool size -## -## Value: Number -web.hook.pool_size = 32 - -##-------------------------------------------------------------------- -## Hook Rules -## These configuration items represent a list of events should be forwarded -## -## Format: -## web.hook.rule.. = -#web.hook.rule.client.connect.1 = "{"action": "on_client_connect"}" -#web.hook.rule.client.connack.1 = "{"action": "on_client_connack"}" -#web.hook.rule.client.connected.1 = "{"action": "on_client_connected"}" -#web.hook.rule.client.disconnected.1 = "{"action": "on_client_disconnected"}" -#web.hook.rule.client.subscribe.1 = "{"action": "on_client_subscribe"}" -#web.hook.rule.client.unsubscribe.1 = "{"action": "on_client_unsubscribe"}" -#web.hook.rule.session.subscribed.1 = "{"action": "on_session_subscribed"}" -#web.hook.rule.session.unsubscribed.1 = "{"action": "on_session_unsubscribed"}" -#web.hook.rule.session.terminated.1 = "{"action": "on_session_terminated"}" -#web.hook.rule.message.publish.1 = "{"action": "on_message_publish"}" -#web.hook.rule.message.delivered.1 = "{"action": "on_message_delivered"}" -#web.hook.rule.message.acked.1 = ""{"action": "on_message_acked"}" diff --git a/apps/emqx_web_hook/include/emqx_web_hook.hrl b/apps/emqx_web_hook/include/emqx_web_hook.hrl deleted file mode 100644 index 73019ec8c..000000000 --- a/apps/emqx_web_hook/include/emqx_web_hook.hrl +++ /dev/null @@ -1 +0,0 @@ --define(APP, emqx_web_hook). diff --git a/apps/emqx_web_hook/priv/emqx_web_hook.schema b/apps/emqx_web_hook/priv/emqx_web_hook.schema deleted file mode 100644 index 8ba1cc0fd..000000000 --- a/apps/emqx_web_hook/priv/emqx_web_hook.schema +++ /dev/null @@ -1,105 +0,0 @@ -%%-*- mode: erlang -*- -%% EMQ X R3.0 config mapping - -{mapping, "web.hook.url", "emqx_web_hook.url", [ - {datatype, string} -]}. - -{mapping, "web.hook.headers.$name", "emqx_web_hook.headers", [ - {datatype, string} -]}. - -{mapping, "web.hook.body.encoding_of_payload_field", "emqx_web_hook.encoding_of_payload_field", [ - {default, plain}, - {datatype, {enum, [plain, base62, base64]}} -]}. - -{mapping, "web.hook.ssl.cacertfile", "emqx_web_hook.cacertfile", [ - {default, ""}, - {datatype, string} -]}. - -{mapping, "web.hook.ssl.certfile", "emqx_web_hook.certfile", [ - {default, ""}, - {datatype, string} -]}. - -{mapping, "web.hook.ssl.keyfile", "emqx_web_hook.keyfile", [ - {default, ""}, - {datatype, string} -]}. - -{mapping, "web.hook.ssl.verify", "emqx_web_hook.verify", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "web.hook.ssl.server_name_indication", "emqx_web_hook.server_name_indication", [ - {datatype, string} -]}. - -{mapping, "web.hook.pool_size", "emqx_web_hook.pool_size", [ - {default, 32}, - {datatype, integer} -]}. - -{mapping, "web.hook.rule.client.connect.$name", "emqx_web_hook.rules", [ - {datatype, string} -]}. - -{mapping, "web.hook.rule.client.connack.$name", "emqx_web_hook.rules", [ - {datatype, string} -]}. - -{mapping, "web.hook.rule.client.connected.$name", "emqx_web_hook.rules", [ - {datatype, string} -]}. - -{mapping, "web.hook.rule.client.disconnected.$name", "emqx_web_hook.rules", [ - {datatype, string} -]}. - -{mapping, "web.hook.rule.client.subscribe.$name", "emqx_web_hook.rules", [ - {datatype, string} -]}. - -{mapping, "web.hook.rule.client.unsubscribe.$name", "emqx_web_hook.rules", [ - {datatype, string} -]}. - -{mapping, "web.hook.rule.session.subscribed.$name", "emqx_web_hook.rules", [ - {datatype, string} -]}. - -{mapping, "web.hook.rule.session.unsubscribed.$name", "emqx_web_hook.rules", [ - {datatype, string} -]}. - -{mapping, "web.hook.rule.session.terminated.$name", "emqx_web_hook.rules", [ - {datatype, string} -]}. - -{mapping, "web.hook.rule.message.publish.$name", "emqx_web_hook.rules", [ - {datatype, string} -]}. - -{mapping, "web.hook.rule.message.acked.$name", "emqx_web_hook.rules", [ - {datatype, string} -]}. - -{mapping, "web.hook.rule.message.delivered.$name", "emqx_web_hook.rules", [ - {datatype, string} -]}. - -{translation, "emqx_web_hook.headers", fun(Conf) -> - Headers = cuttlefish_variable:filter_by_prefix("web.hook.headers", Conf), - [{K, V} || {[_, _, _, K], V} <- Headers] -end}. - -{translation, "emqx_web_hook.rules", fun(Conf) -> - Hooks = cuttlefish_variable:filter_by_prefix("web.hook.rule", Conf), - lists:map( - fun({[_, _, _,Name1,Name2, _], Val}) -> - {lists:concat([Name1,".",Name2]), Val} - end, Hooks) -end}. diff --git a/apps/emqx_web_hook/rebar.config b/apps/emqx_web_hook/rebar.config deleted file mode 100644 index 387972c9f..000000000 --- a/apps/emqx_web_hook/rebar.config +++ /dev/null @@ -1,18 +0,0 @@ -{plugins, [rebar3_proper]}. - -{deps, []}. - -{edoc_opts, [{preprocess, true}]}. -{erl_opts, [warn_unused_vars, - warn_shadow_vars, - warn_unused_import, - warn_obsolete_guard, - debug_info, - {parse_transform}]}. - -{xref_checks, [undefined_function_calls, undefined_functions, - locals_not_used, deprecated_function_calls, - warnings_as_errors, deprecated_functions]}. -{cover_enabled, true}. -{cover_opts, [verbose]}. -{cover_export_enabled, true}. \ No newline at end of file diff --git a/apps/emqx_web_hook/src/emqx_web_hook.app.src b/apps/emqx_web_hook/src/emqx_web_hook.app.src deleted file mode 100644 index e1cfda173..000000000 --- a/apps/emqx_web_hook/src/emqx_web_hook.app.src +++ /dev/null @@ -1,14 +0,0 @@ -{application, emqx_web_hook, - [{description, "EMQ X WebHook Plugin"}, - {vsn, "4.3.2"}, % strict semver, bump manually! - {modules, []}, - {registered, [emqx_web_hook_sup]}, - {applications, [kernel,stdlib,ehttpc]}, - {mod, {emqx_web_hook_app,[]}}, - {env, []}, - {licenses, ["Apache-2.0"]}, - {maintainers, ["EMQ X Team "]}, - {links, [{"Homepage", "https://emqx.io/"}, - {"Github", "https://github.com/emqx/emqx-web-hook"} - ]} - ]}. diff --git a/apps/emqx_web_hook/src/emqx_web_hook.appup.src b/apps/emqx_web_hook/src/emqx_web_hook.appup.src deleted file mode 100644 index ae6e9e1ae..000000000 --- a/apps/emqx_web_hook/src/emqx_web_hook.appup.src +++ /dev/null @@ -1,18 +0,0 @@ -%% -*-: erlang -*- - -{VSN, - [ - {<<"4.3.[0-1]">>, [ - {restart_application, emqx_web_hook}, - {apply,{emqx_rule_engine,refresh_resource,[web_hook]}} - ]}, - {<<".*">>, []} - ], - [ - {<<"4.3.[0-1]">>, [ - {restart_application, emqx_web_hook}, - {apply,{emqx_rule_engine,refresh_resource,[web_hook]}} - ]}, - {<<".*">>, []} - ] -}. diff --git a/apps/emqx_web_hook/src/emqx_web_hook.erl b/apps/emqx_web_hook/src/emqx_web_hook.erl deleted file mode 100644 index 7af83d749..000000000 --- a/apps/emqx_web_hook/src/emqx_web_hook.erl +++ /dev/null @@ -1,390 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_web_hook). - --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/logger.hrl"). - --define(APP, emqx_web_hook). - --logger_header("[WebHook]"). - --import(inet, [ntoa/1]). - -%% APIs --export([ register_metrics/0 - , load/0 - , unload/0 - ]). - -%% Hooks callback --export([ on_client_connect/3 - , on_client_connack/4 - , on_client_connected/3 - , on_client_disconnected/4 - , on_client_subscribe/4 - , on_client_unsubscribe/4 - ]). - --export([ on_session_subscribed/4 - , on_session_unsubscribed/4 - , on_session_terminated/4 - ]). --export([ on_message_publish/2 - , on_message_delivered/3 - , on_message_acked/3 - ]). - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, - ['webhook.client_connect', - 'webhook.client_connack', - 'webhook.client_connected', - 'webhook.client_disconnected', - 'webhook.client_subscribe', - 'webhook.client_unsubscribe', - 'webhook.session_subscribed', - 'webhook.session_unsubscribed', - 'webhook.session_terminated', - 'webhook.message_publish', - 'webhook.message_delivered', - 'webhook.message_acked']). - -load() -> - lists:foreach( - fun({Hook, Fun, Filter}) -> - emqx:hook(Hook, {?MODULE, Fun, [{Filter}]}) - end, parse_rule(application:get_env(?APP, rules, []))). - -unload() -> - lists:foreach( - fun({Hook, Fun, _Filter}) -> - emqx:unhook(Hook, {?MODULE, Fun}) - end, parse_rule(application:get_env(?APP, rules, []))). - -%%-------------------------------------------------------------------- -%% Client connect -%%-------------------------------------------------------------------- - -on_client_connect(ConnInfo = #{clientid := ClientId, username := Username, peername := {Peerhost, _}}, _ConnProp, _Env) -> - emqx_metrics:inc('webhook.client_connect'), - Params = #{ action => client_connect - , node => node() - , clientid => ClientId - , username => maybe(Username) - , ipaddress => iolist_to_binary(ntoa(Peerhost)) - , keepalive => maps:get(keepalive, ConnInfo) - , proto_ver => maps:get(proto_ver, ConnInfo) - }, - send_http_request(ClientId, Params). - -%%-------------------------------------------------------------------- -%% Client connack -%%-------------------------------------------------------------------- - -on_client_connack(ConnInfo = #{clientid := ClientId, username := Username, peername := {Peerhost, _}}, Rc, _AckProp, _Env) -> - emqx_metrics:inc('webhook.client_connack'), - Params = #{ action => client_connack - , node => node() - , clientid => ClientId - , username => maybe(Username) - , ipaddress => iolist_to_binary(ntoa(Peerhost)) - , keepalive => maps:get(keepalive, ConnInfo) - , proto_ver => maps:get(proto_ver, ConnInfo) - , conn_ack => Rc - }, - send_http_request(ClientId, Params). - -%%-------------------------------------------------------------------- -%% Client connected -%%-------------------------------------------------------------------- - -on_client_connected(#{clientid := ClientId, username := Username, peerhost := Peerhost}, ConnInfo, _Env) -> - emqx_metrics:inc('webhook.client_connected'), - Params = #{ action => client_connected - , node => node() - , clientid => ClientId - , username => maybe(Username) - , ipaddress => iolist_to_binary(ntoa(Peerhost)) - , keepalive => maps:get(keepalive, ConnInfo) - , proto_ver => maps:get(proto_ver, ConnInfo) - , connected_at => maps:get(connected_at, ConnInfo) - }, - send_http_request(ClientId, Params). - -%%-------------------------------------------------------------------- -%% Client disconnected -%%-------------------------------------------------------------------- - -on_client_disconnected(ClientInfo, {shutdown, Reason}, ConnInfo, Env) when is_atom(Reason) -> - on_client_disconnected(ClientInfo, Reason, ConnInfo, Env); -on_client_disconnected(#{clientid := ClientId, username := Username}, Reason, ConnInfo, _Env) -> - emqx_metrics:inc('webhook.client_disconnected'), - Params = #{ action => client_disconnected - , node => node() - , clientid => ClientId - , username => maybe(Username) - , reason => stringfy(maybe(Reason)) - , disconnected_at => maps:get(disconnected_at, ConnInfo, erlang:system_time(millisecond)) - }, - send_http_request(ClientId, Params). - -%%-------------------------------------------------------------------- -%% Client subscribe -%%-------------------------------------------------------------------- - -on_client_subscribe(#{clientid := ClientId, username := Username}, _Properties, TopicTable, {Filter}) -> - lists:foreach(fun({Topic, Opts}) -> - with_filter( - fun() -> - emqx_metrics:inc('webhook.client_subscribe'), - Params = #{ action => client_subscribe - , node => node() - , clientid => ClientId - , username => maybe(Username) - , topic => Topic - , opts => Opts - }, - send_http_request(ClientId, Params) - end, Topic, Filter) - end, TopicTable). - -%%-------------------------------------------------------------------- -%% Client unsubscribe -%%-------------------------------------------------------------------- - -on_client_unsubscribe(#{clientid := ClientId, username := Username}, _Properties, TopicTable, {Filter}) -> - lists:foreach(fun({Topic, Opts}) -> - with_filter( - fun() -> - emqx_metrics:inc('webhook.client_unsubscribe'), - Params = #{ action => client_unsubscribe - , node => node() - , clientid => ClientId - , username => maybe(Username) - , topic => Topic - , opts => Opts - }, - send_http_request(ClientId, Params) - end, Topic, Filter) - end, TopicTable). - -%%-------------------------------------------------------------------- -%% Session subscribed -%%-------------------------------------------------------------------- - -on_session_subscribed(#{clientid := ClientId, username := Username}, Topic, Opts, {Filter}) -> - with_filter( - fun() -> - emqx_metrics:inc('webhook.session_subscribed'), - Params = #{ action => session_subscribed - , node => node() - , clientid => ClientId - , username => maybe(Username) - , topic => Topic - , opts => Opts - }, - send_http_request(ClientId, Params) - end, Topic, Filter). - -%%-------------------------------------------------------------------- -%% Session unsubscribed -%%-------------------------------------------------------------------- - -on_session_unsubscribed(#{clientid := ClientId, username := Username}, Topic, _Opts, {Filter}) -> - with_filter( - fun() -> - emqx_metrics:inc('webhook.session_unsubscribed'), - Params = #{ action => session_unsubscribed - , node => node() - , clientid => ClientId - , username => maybe(Username) - , topic => Topic - }, - send_http_request(ClientId, Params) - end, Topic, Filter). - -%%-------------------------------------------------------------------- -%% Session terminated -%%-------------------------------------------------------------------- - -on_session_terminated(Info, {shutdown, Reason}, SessInfo, Env) when is_atom(Reason) -> - on_session_terminated(Info, Reason, SessInfo, Env); -on_session_terminated(#{clientid := ClientId, username := Username}, Reason, _SessInfo, _Env) -> - emqx_metrics:inc('webhook.session_terminated'), - Params = #{ action => session_terminated - , node => node() - , clientid => ClientId - , username => maybe(Username) - , reason => stringfy(maybe(Reason)) - }, - send_http_request(ClientId, Params). - -%%-------------------------------------------------------------------- -%% Message publish -%%-------------------------------------------------------------------- - -on_message_publish(Message = #message{topic = <<"$SYS/", _/binary>>}, _Env) -> - {ok, Message}; -on_message_publish(Message = #message{topic = Topic}, {Filter}) -> - with_filter( - fun() -> - emqx_metrics:inc('webhook.message_publish'), - {FromClientId, FromUsername} = parse_from(Message), - Params = #{ action => message_publish - , node => node() - , from_client_id => FromClientId - , from_username => FromUsername - , topic => Message#message.topic - , qos => Message#message.qos - , retain => emqx_message:get_flag(retain, Message) - , payload => encode_payload(Message#message.payload) - , ts => Message#message.timestamp - }, - send_http_request(FromClientId, Params), - {ok, Message} - end, Message, Topic, Filter). - -%%-------------------------------------------------------------------- -%% Message deliver -%%-------------------------------------------------------------------- - -on_message_delivered(_ClientInfo,#message{topic = <<"$SYS/", _/binary>>}, _Env) -> - ok; -on_message_delivered(#{clientid := ClientId, username := Username}, - Message = #message{topic = Topic}, {Filter}) -> - with_filter( - fun() -> - emqx_metrics:inc('webhook.message_delivered'), - {FromClientId, FromUsername} = parse_from(Message), - Params = #{ action => message_delivered - , node => node() - , clientid => ClientId - , username => maybe(Username) - , from_client_id => FromClientId - , from_username => FromUsername - , topic => Message#message.topic - , qos => Message#message.qos - , retain => emqx_message:get_flag(retain, Message) - , payload => encode_payload(Message#message.payload) - , ts => Message#message.timestamp - }, - send_http_request(ClientId, Params) - end, Topic, Filter). - -%%-------------------------------------------------------------------- -%% Message acked -%%-------------------------------------------------------------------- - -on_message_acked(_ClientInfo, #message{topic = <<"$SYS/", _/binary>>}, _Env) -> - ok; -on_message_acked(#{clientid := ClientId, username := Username}, - Message = #message{topic = Topic}, {Filter}) -> - with_filter( - fun() -> - emqx_metrics:inc('webhook.message_acked'), - {FromClientId, FromUsername} = parse_from(Message), - Params = #{ action => message_acked - , node => node() - , clientid => ClientId - , username => maybe(Username) - , from_client_id => FromClientId - , from_username => FromUsername - , topic => Message#message.topic - , qos => Message#message.qos - , retain => emqx_message:get_flag(retain, Message) - , payload => encode_payload(Message#message.payload) - , ts => Message#message.timestamp - }, - send_http_request(ClientId, Params) - end, Topic, Filter). - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -send_http_request(ClientID, Params) -> - {ok, Path} = application:get_env(?APP, path), - Headers = application:get_env(?APP, headers, []), - Body = emqx_json:encode(Params), - ?LOG(debug, "Send to: ~0p, params: ~s", [Path, Body]), - case ehttpc:request(ehttpc_pool:pick_worker(?APP, ClientID), post, {Path, Headers, Body}) of - {ok, StatusCode, _} when StatusCode >= 200 andalso StatusCode < 300 -> - ok; - {ok, StatusCode, _, _} when StatusCode >= 200 andalso StatusCode < 300 -> - ok; - {ok, StatusCode, _} -> - ?LOG(warning, "HTTP request failed with status code: ~p", [StatusCode]), - ok; - {ok, StatusCode, _, _} -> - ?LOG(warning, "HTTP request failed with status code: ~p", [StatusCode]), - ok; - {error, Reason} -> - ?LOG(error, "HTTP request error: ~p", [Reason]), - ok - end. - -parse_rule(Rules) -> - parse_rule(Rules, []). -parse_rule([], Acc) -> - lists:reverse(Acc); -parse_rule([{Rule, Conf} | Rules], Acc) -> - Params = emqx_json:decode(iolist_to_binary(Conf)), - Action = proplists:get_value(<<"action">>, Params), - Filter = proplists:get_value(<<"topic">>, Params), - parse_rule(Rules, [{list_to_atom(Rule), binary_to_existing_atom(Action, utf8), Filter} | Acc]). - -with_filter(Fun, _, undefined) -> - Fun(), ok; -with_filter(Fun, Topic, Filter) -> - case emqx_topic:match(Topic, Filter) of - true -> Fun(), ok; - false -> ok - end. - -with_filter(Fun, _, _, undefined) -> - Fun(); -with_filter(Fun, Msg, Topic, Filter) -> - case emqx_topic:match(Topic, Filter) of - true -> Fun(); - false -> {ok, Msg} - end. - -parse_from(Message) -> - {emqx_message:from(Message), maybe(emqx_message:get_header(username, Message))}. - -encode_payload(Payload) -> - encode_payload(Payload, application:get_env(?APP, encoding_of_payload_field, plain)). - -encode_payload(Payload, base62) -> emqx_base62:encode(Payload); -encode_payload(Payload, base64) -> base64:encode(Payload); -encode_payload(Payload, plain) -> Payload. - -stringfy(Term) when is_binary(Term) -> - Term; -stringfy(Term) when is_atom(Term) -> - atom_to_binary(Term, utf8); -stringfy(Term) -> - unicode:characters_to_binary((io_lib:format("~0p", [Term]))). - -maybe(undefined) -> null; -maybe(Str) -> Str. - diff --git a/apps/emqx_web_hook/src/emqx_web_hook_app.erl b/apps/emqx_web_hook/src/emqx_web_hook_app.erl deleted file mode 100644 index 580742c47..000000000 --- a/apps/emqx_web_hook/src/emqx_web_hook_app.erl +++ /dev/null @@ -1,102 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_web_hook_app). - --behaviour(application). - --emqx_plugin(?MODULE). - --include("emqx_web_hook.hrl"). - --export([ start/2 - , stop/1 - ]). - -start(_StartType, _StartArgs) -> - translate_env(), - {ok, Sup} = emqx_web_hook_sup:start_link(), - {ok, PoolOpts} = application:get_env(?APP, pool_opts), - {ok, _Pid} = ehttpc_sup:start_pool(?APP, PoolOpts), - emqx_web_hook:register_metrics(), - emqx_web_hook:load(), - {ok, Sup}. - -stop(_State) -> - emqx_web_hook:unload(), - ehttpc_sup:stop_pool(?APP). - -translate_env() -> - {ok, URL} = application:get_env(?APP, url), - {ok, #{host := Host, - port := Port, - scheme := Scheme} = URIMap} = emqx_http_lib:uri_parse(URL), - Path = path(URIMap), - PoolSize = application:get_env(?APP, pool_size, 32), - MoreOpts = case Scheme of - http -> - [{transport_opts, emqx_misc:ipv6_probe([])}]; - https -> - CACertFile = application:get_env(?APP, cacertfile, undefined), - CertFile = application:get_env(?APP, certfile, undefined), - KeyFile = application:get_env(?APP, keyfile, undefined), - {ok, Verify} = application:get_env(?APP, verify), - VerifyType = case Verify of - true -> verify_peer; - false -> verify_none - end, - SNI = case application:get_env(?APP, server_name_indication, undefined) of - "disable" -> disable; - SNI0 -> SNI0 - end, - TLSOpts = lists:filter(fun({_K, V}) -> - V /= <<>> andalso V /= undefined andalso V /= "" andalso true - end, [{keyfile, KeyFile}, - {certfile, CertFile}, - {cacertfile, CACertFile}, - {verify, VerifyType}, - {server_name_indication, SNI}]), - NTLSOpts = [ {versions, emqx_tls_lib:default_versions()} - , {ciphers, emqx_tls_lib:default_ciphers()} - | TLSOpts - ], - [{transport, ssl}, {transport_opts, emqx_misc:ipv6_probe(NTLSOpts)}] - end, - PoolOpts = [{host, Host}, - {port, Port}, - {pool_size, PoolSize}, - {pool_type, hash}, - {connect_timeout, 5000}, - {retry, 5}, - {retry_timeout, 1000}] ++ MoreOpts, - application:set_env(?APP, path, Path), - application:set_env(?APP, pool_opts, PoolOpts), - Headers = application:get_env(?APP, headers, []), - NHeaders = set_content_type(Headers), - application:set_env(?APP, headers, NHeaders). - -path(#{path := "", 'query' := Query}) -> - "?" ++ Query; -path(#{path := Path, 'query' := Query}) -> - Path ++ "?" ++ Query; -path(#{path := ""}) -> - "/"; -path(#{path := Path}) -> - Path. - -set_content_type(Headers) -> - NHeaders = proplists:delete(<<"Content-Type">>, proplists:delete(<<"content-type">>, Headers)), - [{<<"content-type">>, <<"application/json">>} | NHeaders]. diff --git a/apps/emqx_web_hook/test/emqx_web_hook_SUITE.erl b/apps/emqx_web_hook/test/emqx_web_hook_SUITE.erl deleted file mode 100644 index 864e1b150..000000000 --- a/apps/emqx_web_hook/test/emqx_web_hook_SUITE.erl +++ /dev/null @@ -1,284 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_web_hook_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("emqx/include/emqx.hrl"). --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - --define(HOOK_LOOKUP(H), emqx_hooks:lookup(list_to_atom(H))). --define(ACTION(Name), #{<<"action">> := Name}). - -%%-------------------------------------------------------------------- -%% Setups -%%-------------------------------------------------------------------- - -all() -> - [ {group, http} - , {group, https} - , {group, ipv6http} - , {group, ipv6https} - , {group, all} - ]. - -groups() -> - Cases = [test_full_flow], - [ {http, [sequence], Cases} - , {https, [sequence], Cases} - , {ipv6http, [sequence], Cases} - , {ipv6https, [sequence], Cases} - , {all, [sequence], emqx_ct:all(?MODULE)} - ]. - -start_apps(F) -> emqx_ct_helpers:start_apps(apps(), F). - -init_per_group(Name, Config) -> - application:ensure_all_started(emqx_management), - set_special_cfgs(), - BasePort = - case Name of - all -> 8801; - http -> 8811; - https -> 8821; - ipv6http -> 8831; - ipv6https -> 8841 - end, - CF = case Name of - all -> fun set_special_configs_http/1; - http -> fun set_special_configs_http/1; - https -> fun set_special_configs_https/1; - ipv6http -> fun set_special_configs_ipv6_http/1; - ipv6https -> fun set_special_configs_ipv6_https/1 - end, - start_apps(fun(_) -> CF(BasePort) end), - Opts = case atom_to_list(Name) of - "ipv6" ++ _ -> [{ip, {0,0,0,0,0,0,0,1}}, inet6]; - _ -> [inet] - end, - [{base_port, BasePort}, {transport_opts, Opts} | Config]. - -end_per_group(_Name, Config) -> - emqx_ct_helpers:stop_apps(apps()), - Config. - -set_special_configs_http(Port) -> - application:set_env(emqx_web_hook, url, "http://127.0.0.1:" ++ integer_to_list(Port)). - -set_special_configs_https(Port) -> - set_ssl_configs(), - application:set_env(emqx_web_hook, url, "https://127.0.0.1:" ++ integer_to_list(Port+1)). - -set_special_configs_ipv6_http(Port) -> - application:set_env(emqx_web_hook, url, "http://[::1]:" ++ integer_to_list(Port)). - -set_special_configs_ipv6_https(Port) -> - set_ssl_configs(), - application:set_env(emqx_web_hook, url, "https://[::1]:" ++ integer_to_list(Port+1)). - -set_ssl_configs() -> - Path = emqx_ct_helpers:deps_path(emqx_web_hook, "test/emqx_web_hook_SUITE_data/"), - SslOpts = [{keyfile, Path ++ "/client-key.pem"}, - {certfile, Path ++ "/client-cert.pem"}, - {cacertfile, Path ++ "/ca.pem"}], - application:set_env(emqx_web_hook, ssl, true), - application:set_env(emqx_web_hook, ssloptions, SslOpts). - -set_special_cfgs() -> - AllRules = [{"message.acked", "{\"action\": \"on_message_acked\"}"}, - {"message.delivered", "{\"action\": \"on_message_delivered\"}"}, - {"message.publish", "{\"action\": \"on_message_publish\"}"}, - {"session.terminated", "{\"action\": \"on_session_terminated\"}"}, - {"session.unsubscribed", "{\"action\": \"on_session_unsubscribed\"}"}, - {"session.subscribed", "{\"action\": \"on_session_subscribed\"}"}, - {"client.unsubscribe", "{\"action\": \"on_client_unsubscribe\"}"}, - {"client.subscribe", "{\"action\": \"on_client_subscribe\"}"}, - {"client.disconnected", "{\"action\": \"on_client_disconnected\"}"}, - {"client.connected", "{\"action\": \"on_client_connected\"}"}, - {"client.connack", "{\"action\": \"on_client_connack\"}"}, - {"client.connect", "{\"action\": \"on_client_connect\"}"}], - application:set_env(emqx_web_hook, rules, AllRules). - -%%-------------------------------------------------------------------- -%% Test cases -%%-------------------------------------------------------------------- - -test_full_flow(Config) -> - [_|_] = Opts = proplists:get_value(transport_opts, Config), - BasePort = proplists:get_value(base_port, Config), - Tester = self(), - {ok, ServerPid} = http_server:start_link(Tester, BasePort, Opts), - receive {ServerPid, ready} -> ok - after 1000 -> error(timeout) - end, - application:set_env(emqx_web_hook, headers, [{"k1","K1"}, {"k2", "K2"}]), - ClientId = iolist_to_binary(["client-", integer_to_list(erlang:system_time())]), - {ok, C} = emqtt:start_link([ {clientid, ClientId} - , {proto_ver, v5} - , {keepalive, 60} - ]), - try - do_test_full_flow(C, ClientId) - after - Ref = erlang:monitor(process, ServerPid), - http_server:stop(ServerPid), - receive {'DOWN', Ref, _, _, _} -> ok - after 5000 -> error(timeout) - end - end. - -do_test_full_flow(C, ClientId) -> - {ok, _} = emqtt:connect(C), - {ok, _, _} = emqtt:subscribe(C, <<"TopicA">>, qos2), - {ok, _} = emqtt:publish(C, <<"TopicA">>, <<"Payload...">>, qos2), - {ok, _, _} = emqtt:unsubscribe(C, <<"TopicA">>), - emqtt:disconnect(C), - validate_params_and_headers(undefined, ClientId). - -validate_params_and_headers(ClientState, ClientId) -> - receive - {http_server, {Params0, _Bool}, Headers} -> - Params = emqx_json:decode(Params0, [return_maps]), - try - validate_hook_resp(ClientId, Params), - validate_hook_headers(Headers), - case maps:get(<<"action">>, Params) of - <<"session_terminated">> -> - ok; - <<"client_connect">> -> - validate_params_and_headers(connected, ClientId); - _ -> - validate_params_and_headers(ClientState, ClientId) %% continue looping - end - catch - throw : {unknown_client, Other} -> - ct:pal("ignored_event_from_other_client ~p~nexpecting:~p~n~p~n~p", - [Other, ClientId, Params, Headers]), - validate_params_and_headers(ClientState, ClientId) %% continue looping - end - after - 5000 -> - case ClientState =:= undefined of - true -> error("client_was_never_connected"); - false -> error("terminate_action_is_not_received_in_time") - end - end. - -t_check_hooked(_) -> - {ok, Rules} = application:get_env(emqx_web_hook, rules), - lists:foreach(fun({HookName, _Action}) -> - Hooks = ?HOOK_LOOKUP(HookName), - ?assertEqual(true, length(Hooks) > 0) - end, Rules). - -t_change_config(_) -> - {ok, Rules} = application:get_env(emqx_web_hook, rules), - emqx_web_hook:unload(), - HookRules = lists:keydelete("message.delivered", 1, Rules), - application:set_env(emqx_web_hook, rules, HookRules), - emqx_web_hook:load(), - ?assertEqual([], ?HOOK_LOOKUP("message.delivered")), - emqx_web_hook:unload(), - application:set_env(emqx_web_hook, rules, Rules), - emqx_web_hook:load(). - -%%-------------------------------------------------------------------- -%% Utils -%%-------------------------------------------------------------------- - -validate_hook_headers(Headers) -> - ?assertEqual(<<"K1">>, maps:get(<<"k1">>, Headers)), - ?assertEqual(<<"K2">>, maps:get(<<"k2">>, Headers)). - -validate_hook_resp(ClientId, Body = ?ACTION(<<"client_connect">>)) -> - assert_username_clientid(ClientId, Body), - ?assertEqual(5, maps:get(<<"proto_ver">>, Body)), - ?assertEqual(60, maps:get(<<"keepalive">>, Body)), - ?assertEqual(<<"127.0.0.1">>, maps:get(<<"ipaddress">>, Body)), - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)), - ok; -validate_hook_resp(ClientId, Body = ?ACTION(<<"client_connack">>)) -> - assert_username_clientid(ClientId, Body), - ?assertEqual(5, maps:get(<<"proto_ver">>, Body)), - ?assertEqual(60, maps:get(<<"keepalive">>, Body)), - ?assertEqual(<<"success">>, maps:get(<<"conn_ack">>, Body)), - ?assertEqual(<<"127.0.0.1">>, maps:get(<<"ipaddress">>, Body)), - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)), - ok; -validate_hook_resp(ClientId, Body = ?ACTION(<<"client_connected">>)) -> - assert_username_clientid(ClientId, Body), - _ = maps:get(<<"connected_at">>, Body), - ?assertEqual(5, maps:get(<<"proto_ver">>, Body)), - ?assertEqual(60, maps:get(<<"keepalive">>, Body)), - ?assertEqual(<<"127.0.0.1">>, maps:get(<<"ipaddress">>, Body)), - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)); -validate_hook_resp(ClientId, Body = ?ACTION(<<"client_disconnected">>)) -> - assert_username_clientid(ClientId, Body), - ?assertEqual(<<"normal">>, maps:get(<<"reason">>, Body)), - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)); -validate_hook_resp(ClientId, Body = ?ACTION(<<"client_subscribe">>)) -> - assert_username_clientid(ClientId, Body), - _ = maps:get(<<"opts">>, Body), - ?assertEqual(<<"TopicA">>, maps:get(<<"topic">>, Body)), - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)); -validate_hook_resp(ClientId, Body = ?ACTION(<<"client_unsubscribe">>)) -> - assert_username_clientid(ClientId, Body), - _ = maps:get(<<"opts">>, Body), - ?assertEqual(<<"TopicA">>, maps:get(<<"topic">>, Body)), - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)); -validate_hook_resp(ClientId, Body = ?ACTION(<<"session_subscribed">>)) -> - assert_username_clientid(ClientId, Body), - _ = maps:get(<<"opts">>, Body), - ?assertEqual(<<"TopicA">>, maps:get(<<"topic">>, Body)), - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)); -validate_hook_resp(ClientId, Body = ?ACTION(<<"session_unsubscribed">>)) -> - assert_username_clientid(ClientId, Body), - ?assertEqual(<<"TopicA">>, maps:get(<<"topic">>, Body)), - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)); -validate_hook_resp(ClientId, Body = ?ACTION(<<"session_terminated">>)) -> - assert_username_clientid(ClientId, Body), - ?assertEqual(<<"normal">>, maps:get(<<"reason">>, Body)), - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)); -validate_hook_resp(_ClientId, Body = ?ACTION(<<"message_publish">>)) -> - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)), - assert_messages_attrs(Body); -validate_hook_resp(_ClientId, Body = ?ACTION(<<"message_delivered">>)) -> - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)), - assert_messages_attrs(Body); -validate_hook_resp(_ClientId, Body = ?ACTION(<<"message_acked">>)) -> - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)), - assert_messages_attrs(Body). - -assert_username_clientid(ClientId, #{<<"clientid">> := ClientId, <<"username">> := Username}) -> - ?assertEqual(null, Username); -assert_username_clientid(_ClientId, #{<<"clientid">> := Other}) -> - throw({unknown_client, Other}). - -assert_messages_attrs(#{ <<"ts">> := _ - , <<"qos">> := _ - , <<"topic">> := _ - , <<"retain">> := _ - , <<"payload">> := _ - , <<"from_username">> := _ - , <<"from_client_id">> := _ - }) -> - ok. - -apps() -> - [emqx_web_hook, emqx_modules, emqx_management, emqx_rule_engine]. diff --git a/apps/emqx_web_hook/test/emqx_web_hook_SUITE_data/ca.pem b/apps/emqx_web_hook/test/emqx_web_hook_SUITE_data/ca.pem deleted file mode 100644 index 00b31d8a4..000000000 --- a/apps/emqx_web_hook/test/emqx_web_hook_SUITE_data/ca.pem +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDAzCCAeugAwIBAgIBATANBgkqhkiG9w0BAQsFADA8MTowOAYDVQQDDDFNeVNR -TF9TZXJ2ZXJfOC4wLjE5X0F1dG9fR2VuZXJhdGVkX0NBX0NlcnRpZmljYXRlMB4X -DTIwMDYxMTAzMzg0NloXDTMwMDYwOTAzMzg0NlowPDE6MDgGA1UEAwwxTXlTUUxf -U2VydmVyXzguMC4xOV9BdXRvX0dlbmVyYXRlZF9DQV9DZXJ0aWZpY2F0ZTCCASIw -DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANJBlAYvTQ6euY4HcSn4syH7kq9s -KcG+OMjPUrj+KFEElCzgNuIhaS0f3ORQGB1PNcvVcfdXUI3WX332gWbr9s1b7Xl1 -JKJfDXs+26Cm6NhONTE3sPHnbTSmQEFb52hwAtjQmcY3IQs1AgxKFFHJfnCBEWfE -ePBQaiuYk1XDESMdWpMLrPnYQaj9MpAOUxjlmZCayzPWlF0j0IWvfsF5TqZL7tFK -9p5F/DzyZ4n1mqPVEoUmq5ZdSKj2TQkpWTMHBWHEDQQqXbyE1FGJR7zEUFeuG1KT -sVBg7iZEC93SygZTbgUZSQXIwQCsO6xZ8MB2XDJkPbWp/3Wc6c8I6P09F48CAwEA -AaMQMA4wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEADKz6bIpP5anp -GgLB0jkclRWuMlS4qqIt4itSsMXPJ/ezpHwECixmgW2TIQl6S1woRkUeMxhT2/Ay -Sn/7aKxuzRagyE5NEGOvrOuAP5RO2ZdNJ/X3/Rh533fK1sOTEEbSsWUvW6iSkZef -rsfZBVP32xBhRWkKRdLeLB4W99ADMa0IrTmZPCXHSSE2V4e1o6zWLXcOZeH1Qh8N -SkelBweR+8r1Fbvy1r3s7eH7DCbYoGEDVLQGOLvzHKBisQHmoDnnF5E9g1eeNRdg -o+vhOKfYCOzeNREJIqS42PHcGhdNRk90ycigPmfUJclz1mDHoMjKR2S5oosTpr65 -tNPx3CL7GA== ------END CERTIFICATE----- diff --git a/apps/emqx_web_hook/test/emqx_web_hook_SUITE_data/client-cert.pem b/apps/emqx_web_hook/test/emqx_web_hook_SUITE_data/client-cert.pem deleted file mode 100644 index aad1404ca..000000000 --- a/apps/emqx_web_hook/test/emqx_web_hook_SUITE_data/client-cert.pem +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDBDCCAeygAwIBAgIBAzANBgkqhkiG9w0BAQsFADA8MTowOAYDVQQDDDFNeVNR -TF9TZXJ2ZXJfOC4wLjE5X0F1dG9fR2VuZXJhdGVkX0NBX0NlcnRpZmljYXRlMB4X -DTIwMDYxMTAzMzg0N1oXDTMwMDYwOTAzMzg0N1owQDE+MDwGA1UEAww1TXlTUUxf -U2VydmVyXzguMC4xOV9BdXRvX0dlbmVyYXRlZF9DbGllbnRfQ2VydGlmaWNhdGUw -ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDVYSWpOvCTupz82fc85Opv -EQ7rkB8X2oOMyBCpkyHKBIr1ZQgRDWBp9UVOASq3GnSElm6+T3Kb1QbOffa8GIlw -sjAueKdq5L2eSkmPIEQ7eoO5kEW+4V866hE1LeL/PmHg2lGP0iqZiJYtElhHNQO8 -3y9I7cm3xWMAA3SSWikVtpJRn3qIp2QSrH+tK+/HHbE5QwtPxdir4ULSCSOaM5Yh -Wi5Oto88TZqe1v7SXC864JVvO4LuS7TuSreCdWZyPXTJFBFeCEWSAxonKZrqHbBe -CwKML6/0NuzjaQ51c2tzmVI6xpHj3nnu4cSRx6Jf9WBm+35vm0wk4pohX3ptdzeV -AgMBAAGjDTALMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEBAByQ5zSNeFUH -Aw7JlpZHtHaSEeiiyBHke20ziQ07BK1yi/ms2HAWwQkpZv149sjNuIRH8pkTmkZn -g8PDzSefjLbC9AsWpWV0XNV22T/cdobqLqMBDDZ2+5bsV+jTrOigWd9/AHVZ93PP -IJN8HJn6rtvo2l1bh/CdsX14uVSdofXnuWGabNTydqtMvmCerZsdf6qKqLL+PYwm -RDpgWiRUY7KPBSSlKm/9lJzA+bOe4dHeJzxWFVCJcbpoiTFs1je1V8kKQaHtuW39 -ifX6LTKUMlwEECCbDKM8Yq2tm8NjkjCcnFDtKg8zKGPUu+jrFMN5otiC3wnKcP7r -O9EkaPcgYH8= ------END CERTIFICATE----- diff --git a/apps/emqx_web_hook/test/emqx_web_hook_SUITE_data/client-key.pem b/apps/emqx_web_hook/test/emqx_web_hook_SUITE_data/client-key.pem deleted file mode 100644 index 6789d0291..000000000 --- a/apps/emqx_web_hook/test/emqx_web_hook_SUITE_data/client-key.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEA1WElqTrwk7qc/Nn3POTqbxEO65AfF9qDjMgQqZMhygSK9WUI -EQ1gafVFTgEqtxp0hJZuvk9ym9UGzn32vBiJcLIwLninauS9nkpJjyBEO3qDuZBF -vuFfOuoRNS3i/z5h4NpRj9IqmYiWLRJYRzUDvN8vSO3Jt8VjAAN0klopFbaSUZ96 -iKdkEqx/rSvvxx2xOUMLT8XYq+FC0gkjmjOWIVouTraPPE2antb+0lwvOuCVbzuC -7ku07kq3gnVmcj10yRQRXghFkgMaJyma6h2wXgsCjC+v9Dbs42kOdXNrc5lSOsaR -49557uHEkceiX/VgZvt+b5tMJOKaIV96bXc3lQIDAQABAoIBAF7yjXmSOn7h6P0y -WCuGiTLG2mbDiLJqj2LTm2Z5i+2Cu/qZ7E76Ls63TxF4v3MemH5vGfQhEhR5ZD/6 -GRJ1sKKvB3WGRqjwA9gtojHH39S/nWGy6vYW/vMOOH37XyjIr3EIdIaUtFQBTSHd -Kd71niYrAbVn6fyWHolhADwnVmTMOl5OOAhCdEF4GN3b5aIhIu8BJ7EUzTtHBJIj -CAEfjZFjDs1y1cIgGFJkuIQxMfCpq5recU2qwip7YO6fk//WEjOPu7kSf5IEswL8 -jg1dea9rGBV6KaD2xsgsC6Ll6Sb4BbsrHMfflG3K2Lk3RdVqqTFp1Fn1PTLQE/1S -S/SZPYECgYEA9qYcHKHd0+Q5Ty5wgpxKGa4UCWkpwvfvyv4bh8qlmxueB+l2AIdo -ZvkM8gTPagPQ3WypAyC2b9iQu70uOJo1NizTtKnpjDdN1YpDjISJuS/P0x73gZwy -gmoM5AzMtN4D6IbxXtXnPaYICvwLKU80ouEN5ZPM4/ODLUu6gsp0v2UCgYEA3Xgi -zMC4JF0vEKEaK0H6QstaoXUmw/lToZGH3TEojBIkb/2LrHUclygtONh9kJSFb89/ -jbmRRLAOrx3HZKCNGUmF4H9k5OQyAIv6OGBinvLGqcbqnyNlI+Le8zxySYwKMlEj -EMrBCLmSyi0CGFrbZ3mlj/oCET/ql9rNvcK+DHECgYAEx5dH3sMjtgp+RFId1dWB -xePRgt4yTwewkVgLO5wV82UOljGZNQaK6Eyd7AXw8f38LHzh+KJQbIvxd2sL4cEi -OaAoohpKg0/Y0YMZl//rPMf0OWdmdZZs/I0fZjgZUSwWN3c59T8z7KG/RL8an9RP -S7kvN7wCttdV61/D5RR6GQKBgDxCe/WKWpBKaovzydMLWLTj7/0Oi0W3iXHkzzr4 -LTgvl4qBSofaNbVLUUKuZTv5rXUG2IYPf99YqCYtzBstNDc1MiAriaBeFtzfOW4t -i6gEFtoLLbuvPc3N5Sv5vn8Ug5G9UfU3td5R4AbyyCcoUZqOFuZd+EIJSiOXfXOs -kVmBAoGBAIU9aPAqhU5LX902oq8KsrpdySONqv5mtoStvl3wo95WIqXNEsFY60wO -q02jKQmJJ2MqhkJm2EoF2Mq8+40EZ5sz8LdgeQ/M0yQ9lAhPi4rftwhpe55Ma9dk -SE9X1c/DMCBEaIjJqVXdy0/EeArwpb8sHkguVVAZUWxzD+phm1gs ------END RSA PRIVATE KEY----- diff --git a/apps/emqx_web_hook/test/emqx_web_hook_SUITE_data/server-cert.pem b/apps/emqx_web_hook/test/emqx_web_hook_SUITE_data/server-cert.pem deleted file mode 100644 index a2f9688df..000000000 --- a/apps/emqx_web_hook/test/emqx_web_hook_SUITE_data/server-cert.pem +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDBDCCAeygAwIBAgIBAjANBgkqhkiG9w0BAQsFADA8MTowOAYDVQQDDDFNeVNR -TF9TZXJ2ZXJfOC4wLjE5X0F1dG9fR2VuZXJhdGVkX0NBX0NlcnRpZmljYXRlMB4X -DTIwMDYxMTAzMzg0NloXDTMwMDYwOTAzMzg0NlowQDE+MDwGA1UEAww1TXlTUUxf -U2VydmVyXzguMC4xOV9BdXRvX0dlbmVyYXRlZF9TZXJ2ZXJfQ2VydGlmaWNhdGUw -ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCcEnEm5hqP1EbEJycOz8Ua -NWp29QdpFUzTWhkKGhVXk+0msmNTw4NBAFB42moY44OU8wvDideOlJNhPRWveD8z -G2lxzJA91p0UK4et8ia9MmeuCGhdC9jxJ8X69WNlUiPyy0hI/ZsqRq9Z0C2eW0iL -JPXsy4X8Xpw3SFwoXf5pR9RFY5Pb2tuyxqmSestu2VXT/NQjJg4CVDR3mFcHPXZB -4elRzH0WshExEGkgy0bg20MJeRc2Qdb5Xx+EakbmwroDWaCn3NSGqQ7jv6Vw0doy -TGvS6h6RHBxnyqRfRgKGlCoOMG9/5+rFJC00QpCUG2vHXHWGoWlMlJ3foN7rj5v9 -AgMBAAGjDTALMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEBAJ5zt2rj4Ag6 -zpN59AWC1Fur8g8l41ksHkSpKPp+PtyO/ngvbMqBpfmK1e7JCKZv/68QXfMyWWAI -hwalqZkXXWHKjuz3wE7dE25PXFXtGJtcZAaj10xt98fzdqt8lQSwh2kbfNwZIz1F -sgAStgE7+ZTcqTgvNB76Os1UK0to+/P0VBWktaVFdyub4Nc2SdPVnZNvrRBXBwOD -3V8ViwywDOFoE7DvCvwx/SVsvoC0Z4j3AMMovO6oHicP7uU83qsQgm1Qru3YeoLR -+DoVi7IPHbWvN7MqFYn3YjNlByO2geblY7MR0BlqbFlmFrqLsUfjsh2ys7/U/knC -dN/klu446fI= ------END CERTIFICATE----- diff --git a/apps/emqx_web_hook/test/emqx_web_hook_SUITE_data/server-key.pem b/apps/emqx_web_hook/test/emqx_web_hook_SUITE_data/server-key.pem deleted file mode 100644 index a1dfd5f78..000000000 --- a/apps/emqx_web_hook/test/emqx_web_hook_SUITE_data/server-key.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAnBJxJuYaj9RGxCcnDs/FGjVqdvUHaRVM01oZChoVV5PtJrJj -U8ODQQBQeNpqGOODlPMLw4nXjpSTYT0Vr3g/MxtpccyQPdadFCuHrfImvTJnrgho -XQvY8SfF+vVjZVIj8stISP2bKkavWdAtnltIiyT17MuF/F6cN0hcKF3+aUfURWOT -29rbssapknrLbtlV0/zUIyYOAlQ0d5hXBz12QeHpUcx9FrIRMRBpIMtG4NtDCXkX -NkHW+V8fhGpG5sK6A1mgp9zUhqkO47+lcNHaMkxr0uoekRwcZ8qkX0YChpQqDjBv -f+fqxSQtNEKQlBtrx1x1hqFpTJSd36De64+b/QIDAQABAoIBAFiah66Dt9SruLkn -WR8piUaFyLlcBib8Nq9OWSTJBhDAJERxxb4KIvvGB+l0ZgNXNp5bFPSfzsZdRwZP -PX5uj8Kd71Dxx3mz211WESMJdEC42u+MSmN4lGLkJ5t/sDwXU91E1vbJM0ve8THV -4/Ag9qA4DX2vVZOeyqT/6YHpSsPNZplqzrbAiwrfHwkctHfgqwOf3QLfhmVQgfCS -VwidBldEUv2whSIiIxh4Rv5St4kA68IBCbJxdpOpyuQBkk6CkxZ7VN9FqOuSd4Pk -Wm7iWyBMZsCmELZh5XAXld4BEt87C5R4CvbPBDZxAv3THk1DNNvpy3PFQfwARRFb -SAToYMECgYEAyL7U8yxpzHDYWd3oCx6vTi9p9N/z0FfAkWrRF6dm4UcSklNiT1Aq -EOnTA+SaW8tV3E64gCWcY23gNP8so/ZseWj6L+peHwtchaP9+KB7yGw2A+05+lOx -VetLTjAOmfpiUXFe5w1q4C1RGhLjZjjzW+GvwdAuchQgUEFaomrV+PUCgYEAxwfH -cmVGFbAktcjU4HSRjKSfawCrut+3YUOLybyku3Q/hP9amG8qkVTFe95CTLjLe2D0 -ccaTTpofFEJ32COeck0g0Ujn/qQ+KXRoauOYs4FB1DtqMpqB78wufWEUpDpbd9/h -J+gJdC/IADd4tJW9zA92g8IA7ZtFmqDtiSpQ0ekCgYAQGkaorvJZpN+l7cf0RGTZ -h7IfI2vCVZer0n6tQA9fmLzjoe6r4AlPzAHSOR8sp9XeUy43kUzHKQQoHCPvjw/K -eWJAP7OHF/k2+x2fOPhU7mEy1W+mJdp+wt4Kio5RSaVjVQ3AyPG+w8PSrJszEvRq -dWMMz+851WV2KpfjmWBKlQKBgQC++4j4DZQV5aMkSKV1CIZOBf3vaIJhXKEUFQPD -PmB4fBEjpwCg+zNGp6iktt65zi17o8qMjrb1mtCt2SY04eD932LZUHNFlwcLMmes -Ad+aiDLJ24WJL1f16eDGcOyktlblDZB5gZ/ovJzXEGOkLXglosTfo77OQculmDy2 -/UL2WQKBgGeKasmGNfiYAcWio+KXgFkHXWtAXB9B91B1OFnCa40wx+qnl71MIWQH -PQ/CZFNWOfGiNEJIZjrHsfNJoeXkhq48oKcT0AVCDYyLV0VxDO4ejT95mGW6njNd -JpvmhwwAjOvuWVr0tn4iXlSK8irjlJHmwcRjLTJq97vE9fsA2MjI ------END RSA PRIVATE KEY----- diff --git a/apps/emqx_web_hook/test/http_server.erl b/apps/emqx_web_hook/test/http_server.erl deleted file mode 100644 index 791f725d1..000000000 --- a/apps/emqx_web_hook/test/http_server.erl +++ /dev/null @@ -1,102 +0,0 @@ -%%-------------------------------------------------------------------- -%% A Simple HTTP Server based cowboy -%% -%% It will deliver the http-request params to initialer process -%%-------------------------------------------------------------------- -%% -%% Author:wwhai -%% --module(http_server). --behaviour(gen_server). - --export([start_link/3]). --export([stop/1]). --export([code_change/3, handle_call/3, handle_cast/2, handle_info/2, init/1, init/2, terminate/2]). --record(state, {parent :: pid()}). -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -start_link(Parent, BasePort, Opts) -> - stop_http(), - stop_https(), - timer:sleep(100), - gen_server:start_link(?MODULE, {Parent, BasePort, Opts}, []). - -init({Parent, BasePort, Opts}) -> - ok = start_http(Parent, [{port, BasePort} | Opts]), - ok = start_https(Parent, [{port, BasePort + 1} | Opts]), - Parent ! {self(), ready}, - {ok, #state{parent = Parent}}. - -handle_call(_Request, _From, State) -> - {reply, ignored, State}. - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, _State) -> - stop_http(), - stop_https(). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -stop(Pid) -> - ok = gen_server:stop(Pid). - -%%-------------------------------------------------------------------- -%% Callbacks -%%-------------------------------------------------------------------- - -start_http(Parent, Opts) -> - {ok, _Pid1} = cowboy:start_clear(http, Opts, #{ - env => #{dispatch => compile_router(Parent)} - }), - Port = proplists:get_value(port, Opts), - io:format(standard_error, "[TEST LOG] Start http server on ~p successfully!~n", [Port]). - -start_https(Parent, Opts) -> - Path = emqx_ct_helpers:deps_path(emqx_web_hook, "test/emqx_web_hook_SUITE_data/"), - SslOpts = [{keyfile, Path ++ "/server-key.pem"}, - {cacertfile, Path ++ "/ca.pem"}, - {certfile, Path ++ "/server-cert.pem"}], - - {ok, _Pid2} = cowboy:start_tls(https, Opts ++ SslOpts, - #{env => #{dispatch => compile_router(Parent)}}), - Port = proplists:get_value(port, Opts), - io:format(standard_error, "[TEST LOG] Start https server on ~p successfully!~n", [Port]). - -stop_http() -> - cowboy:stop_listener(http), - io:format("[TEST LOG] Stopped http server"). - -stop_https() -> - cowboy:stop_listener(https), - io:format("[TEST LOG] Stopped https server"). - -compile_router(Parent) -> - {ok, _} = application:ensure_all_started(cowboy), - cowboy_router:compile([ - {'_', [{"/", ?MODULE, #{parent => Parent}}]} - ]). - -init(Req, #{parent := Parent} = State) -> - Method = cowboy_req:method(Req), - Headers = cowboy_req:headers(Req), - [Params] = case Method of - <<"GET">> -> cowboy_req:parse_qs(Req); - <<"POST">> -> - {ok, PostVals, _} = cowboy_req:read_urlencoded_body(Req), - PostVals - end, - Parent ! {?MODULE, Params, Headers}, - {ok, reply(Req, ok), State}. - -reply(Req, ok) -> - cowboy_req:reply(200, #{<<"content-type">> => <<"text/plain">>}, <<"ok">>, Req); -reply(Req, error) -> - cowboy_req:reply(404, #{<<"content-type">> => <<"text/plain">>}, <<"deny">>, Req). diff --git a/apps/emqx_web_hook/test/props/prop_webhook_confs.erl b/apps/emqx_web_hook/test/props/prop_webhook_confs.erl deleted file mode 100644 index 8946ce1d2..000000000 --- a/apps/emqx_web_hook/test/props/prop_webhook_confs.erl +++ /dev/null @@ -1,146 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(prop_webhook_confs). --include_lib("proper/include/proper.hrl"). - --import(emqx_ct_proper_types, - [ url/0 - , nof/1 - ]). - --define(ALL(Vars, Types, Exprs), - ?SETUP(fun() -> - State = do_setup(), - fun() -> do_teardown(State) end - end, ?FORALL(Vars, Types, Exprs))). - -%%-------------------------------------------------------------------- -%% Properties -%%-------------------------------------------------------------------- - -prop_confs() -> - Schema = cuttlefish_schema:files(filelib:wildcard(code:priv_dir(emqx_web_hook) ++ "/*.schema")), - ?ALL({Url, Confs0}, {url(), confs()}, - begin - Confs = [{"web.hook.url", Url}|Confs0], - Envs = cuttlefish_generator:map(Schema, cuttlefish_conf_file(Confs)), - - assert_confs(Confs, Envs), - - set_application_envs(Envs), - {ok, _} = application:ensure_all_started(emqx_web_hook), - application:stop(emqx_web_hook), - unset_application_envs(Envs), - true - end). - -%%-------------------------------------------------------------------- -%% Helpers -%%-------------------------------------------------------------------- - -do_setup() -> - logger:set_primary_config(#{level => warning}), - emqx_ct_helpers:start_apps([], fun set_special_cfgs/1), - ok. - -do_teardown(_) -> - emqx_ct_helpers:stop_apps([]), - logger:set_primary_config(#{level => info}), - ok. - -set_special_cfgs(_) -> - application:set_env(emqx, plugins_loaded_file, undefined), - application:set_env(emqx, modules_loaded_file, undefined), - ok. - -assert_confs([{"web.hook.url", Url}|More], Envs) -> - %% Assert! - Url = deep_get_env("emqx_web_hook.url", Envs), - assert_confs(More, Envs); - -assert_confs([{"web.hook.rule." ++ HookName0, Spec}|More], Envs) -> - HookName = re:replace(HookName0, "\\.[0-9]", "", [{return, list}]), - Rules = deep_get_env("emqx_web_hook.rules", Envs), - - %% Assert! - Spec = proplists:get_value(HookName, Rules), - - assert_confs(More, Envs); - -assert_confs([_|More], Envs) -> - assert_confs(More, Envs); - -assert_confs([], _) -> - true. - -deep_get_env(Path, Envs) -> - lists:foldl( - fun(_K, undefiend) -> undefiend; - (K, Acc) -> proplists:get_value(binary_to_atom(K, utf8), Acc) - end, Envs, re:split(Path, "\\.")). - -set_application_envs(Envs) -> - application:set_env(Envs). - -unset_application_envs(Envs) -> - lists:foreach(fun({App, Es}) -> - lists:foreach(fun({K, _}) -> - application:unset_env(App, K) - end, Es) end, Envs). - -cuttlefish_conf_file(Ls) when is_list(Ls) -> - [cuttlefish_conf_option(K,V) || {K, V} <- Ls]. - -cuttlefish_conf_option(K, V) - when is_list(K) -> - {re:split(K, "[.]", [{return, list}]), V}. - -%%-------------------------------------------------------------------- -%% Generators -%%-------------------------------------------------------------------- - -confs() -> - nof([{"web.hook.headers.content-type", - oneof(["application/json"])}, - {"web.hook.body.encoding_of_payload_field", - oneof(["plain", "base64", "base62"])}, - {"web.hook.rule.client.connect.1", rule_spec()}, - {"web.hook.rule.client.connack.1", rule_spec()}, - {"web.hook.rule.client.connected.1", rule_spec()}, - {"web.hook.rule.client.disconnected.1", rule_spec()}, - {"web.hook.rule.client.subscribe.1", rule_spec()}, - {"web.hook.rule.client.unsubscribe.1", rule_spec()}, - {"web.hook.rule.session.subscribed.1", rule_spec()}, - {"web.hook.rule.session.unsubscribed.1", rule_spec()}, - {"web.hook.rule.session.terminated.1", rule_spec()}, - {"web.hook.rule.message.publish.1", rule_spec()}, - {"web.hook.rule.message.delivered.1", rule_spec()}, - {"web.hook.rule.message.acked.1", rule_spec()} - ]). - -rule_spec() -> - ?LET(Action, action_names(), - begin - binary_to_list(emqx_json:encode(#{action => Action})) - end). - -action_names() -> - oneof([on_client_connect, on_client_connack, on_client_connected, - on_client_connected, on_client_disconnected, on_client_subscribe, on_client_unsubscribe, - on_session_subscribed, on_session_unsubscribed, on_session_terminated, - on_message_publish, on_message_delivered, on_message_acked]). - diff --git a/apps/emqx_web_hook/test/props/prop_webhook_hooks.erl b/apps/emqx_web_hook/test/props/prop_webhook_hooks.erl deleted file mode 100644 index 311585287..000000000 --- a/apps/emqx_web_hook/test/props/prop_webhook_hooks.erl +++ /dev/null @@ -1,397 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(prop_webhook_hooks). - --include_lib("proper/include/proper.hrl"). - --import(emqx_ct_proper_types, - [ conninfo/0 - , clientinfo/0 - , sessioninfo/0 - , message/0 - , connack_return_code/0 - , topictab/0 - , topic/0 - , subopts/0 - ]). - --define(ALL(Vars, Types, Exprs), - ?SETUP(fun() -> - State = do_setup(), - fun() -> do_teardown(State) end - end, ?FORALL(Vars, Types, Exprs))). - -%%-------------------------------------------------------------------- -%% Properties -%%-------------------------------------------------------------------- - -prop_client_connect() -> - ?ALL({ConnInfo, ConnProps, Env}, - {conninfo(), conn_properties(), empty_env()}, - begin - ok = emqx_web_hook:on_client_connect(ConnInfo, ConnProps, Env), - Body = receive_http_request_body(), - Body = emqx_json:encode( - #{action => client_connect, - node => stringfy(node()), - clientid => maps:get(clientid, ConnInfo), - username => maybe(maps:get(username, ConnInfo)), - ipaddress => peer2addr(maps:get(peername, ConnInfo)), - keepalive => maps:get(keepalive, ConnInfo), - proto_ver => maps:get(proto_ver, ConnInfo) - }), - true - end). - -prop_client_connack() -> - ?ALL({ConnInfo, Rc, AckProps, Env}, - {conninfo(), connack_return_code(), ack_properties(), empty_env()}, - begin - ok = emqx_web_hook:on_client_connack(ConnInfo, Rc, AckProps, Env), - Body = receive_http_request_body(), - Body = emqx_json:encode( - #{action => client_connack, - node => stringfy(node()), - clientid => maps:get(clientid, ConnInfo), - username => maybe(maps:get(username, ConnInfo)), - ipaddress => peer2addr(maps:get(peername, ConnInfo)), - keepalive => maps:get(keepalive, ConnInfo), - proto_ver => maps:get(proto_ver, ConnInfo), - conn_ack => Rc - }), - true - end). - -prop_client_connected() -> - ?ALL({ClientInfo, ConnInfo, Env}, - {clientinfo(), conninfo(), empty_env()}, - begin - ok = emqx_web_hook:on_client_connected(ClientInfo, ConnInfo, Env), - Body = receive_http_request_body(), - Body = emqx_json:encode( - #{action => client_connected, - node => stringfy(node()), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo)), - ipaddress => peer2addr(maps:get(peerhost, ClientInfo)), - keepalive => maps:get(keepalive, ConnInfo), - proto_ver => maps:get(proto_ver, ConnInfo), - connected_at => maps:get(connected_at, ConnInfo) - }), - true - end). - -prop_client_disconnected() -> - ?ALL({ClientInfo, Reason, ConnInfo, Env}, - {clientinfo(), shutdown_reason(), disconnected_conninfo(), empty_env()}, - begin - ok = emqx_web_hook:on_client_disconnected(ClientInfo, Reason, ConnInfo, Env), - Body = receive_http_request_body(), - Body = emqx_json:encode( - #{action => client_disconnected, - node => stringfy(node()), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo)), - disconnected_at => maps:get(disconnected_at, ConnInfo), - reason => stringfy(Reason) - }), - true - end). - -prop_client_subscribe() -> - ?ALL({ClientInfo, SubProps, TopicTab, Env}, - {clientinfo(), sub_properties(), topictab(), topic_filter_env()}, - begin - ok = emqx_web_hook:on_client_subscribe(ClientInfo, SubProps, TopicTab, Env), - - Matched = filter_topictab(TopicTab, Env), - - lists:foreach(fun({Topic, Opts}) -> - Body = receive_http_request_body(), - Body = emqx_json:encode( - #{action => client_subscribe, - node => stringfy(node()), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo)), - topic => Topic, - opts => Opts}) - end, Matched), - true - end). - -prop_client_unsubscribe() -> - ?ALL({ClientInfo, SubProps, TopicTab, Env}, - {clientinfo(), unsub_properties(), topictab(), topic_filter_env()}, - begin - ok = emqx_web_hook:on_client_unsubscribe(ClientInfo, SubProps, TopicTab, Env), - - Matched = filter_topictab(TopicTab, Env), - - lists:foreach(fun({Topic, Opts}) -> - Body = receive_http_request_body(), - Body = emqx_json:encode( - #{action => client_unsubscribe, - node => stringfy(node()), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo)), - topic => Topic, - opts => Opts}) - end, Matched), - true - end). - -prop_session_subscribed() -> - ?ALL({ClientInfo, Topic, SubOpts, Env}, - {clientinfo(), topic(), subopts(), topic_filter_env()}, - begin - ok = emqx_web_hook:on_session_subscribed(ClientInfo, Topic, SubOpts, Env), - filter_topic_match(Topic, Env) andalso begin - Body = receive_http_request_body(), - Body1 = emqx_json:encode( - #{action => session_subscribed, - node => stringfy(node()), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo)), - topic => Topic, - opts => SubOpts - }), - Body = Body1 - end, - true - end). - -prop_session_unsubscribed() -> - ?ALL({ClientInfo, Topic, SubOpts, Env}, - {clientinfo(), topic(), subopts(), empty_env()}, - begin - ok = emqx_web_hook:on_session_unsubscribed(ClientInfo, Topic, SubOpts, Env), - filter_topic_match(Topic, Env) andalso begin - Body = receive_http_request_body(), - Body = emqx_json:encode( - #{action => session_unsubscribed, - node => stringfy(node()), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo)), - topic => Topic - }) - end, - true - end). - -prop_session_terminated() -> - ?ALL({ClientInfo, Reason, SessInfo, Env}, - {clientinfo(), shutdown_reason(), sessioninfo(), empty_env()}, - begin - ok = emqx_web_hook:on_session_terminated(ClientInfo, Reason, SessInfo, Env), - Body = receive_http_request_body(), - Body = emqx_json:encode( - #{action => session_terminated, - node => stringfy(node()), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo)), - reason => stringfy(Reason) - }), - true - end). - -prop_message_publish() -> - ?ALL({Msg, Env, Encode}, {message(), topic_filter_env(), payload_encode()}, - begin - application:set_env(emqx_web_hook, encoding_of_payload_field, Encode), - {ok, Msg} = emqx_web_hook:on_message_publish(Msg, Env), - application:unset_env(emqx_web_hook, encoding_of_payload_field), - - (not emqx_message:is_sys(Msg)) - andalso filter_topic_match(emqx_message:topic(Msg), Env) - andalso begin - Body = receive_http_request_body(), - Body = emqx_json:encode( - #{action => message_publish, - node => stringfy(node()), - from_client_id => emqx_message:from(Msg), - from_username => maybe(emqx_message:get_header(username, Msg)), - topic => emqx_message:topic(Msg), - qos => emqx_message:qos(Msg), - retain => emqx_message:get_flag(retain, Msg), - payload => encode(emqx_message:payload(Msg), Encode), - ts => emqx_message:timestamp(Msg) - }) - end, - true - end). - -prop_message_delivered() -> - ?ALL({ClientInfo, Msg, Env, Encode}, {clientinfo(), message(), topic_filter_env(), payload_encode()}, - begin - application:set_env(emqx_web_hook, encoding_of_payload_field, Encode), - ok = emqx_web_hook:on_message_delivered(ClientInfo, Msg, Env), - application:unset_env(emqx_web_hook, encoding_of_payload_field), - - (not emqx_message:is_sys(Msg)) - andalso filter_topic_match(emqx_message:topic(Msg), Env) - andalso begin - Body = receive_http_request_body(), - Body = emqx_json:encode( - #{action => message_delivered, - node => stringfy(node()), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo)), - from_client_id => emqx_message:from(Msg), - from_username => maybe(emqx_message:get_header(username, Msg)), - topic => emqx_message:topic(Msg), - qos => emqx_message:qos(Msg), - retain => emqx_message:get_flag(retain, Msg), - payload => encode(emqx_message:payload(Msg), Encode), - ts => emqx_message:timestamp(Msg) - }) - end, - true - end). - -prop_message_acked() -> - ?ALL({ClientInfo, Msg, Env, Encode}, {clientinfo(), message(), empty_env(), payload_encode()}, - begin - application:set_env(emqx_web_hook, encoding_of_payload_field, Encode), - ok = emqx_web_hook:on_message_acked(ClientInfo, Msg, Env), - application:unset_env(emqx_web_hook, encoding_of_payload_field), - - (not emqx_message:is_sys(Msg)) - andalso filter_topic_match(emqx_message:topic(Msg), Env) - andalso begin - Body = receive_http_request_body(), - Body = emqx_json:encode( - #{action => message_acked, - node => stringfy(node()), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo)), - from_client_id => emqx_message:from(Msg), - from_username => maybe(emqx_message:get_header(username, Msg)), - topic => emqx_message:topic(Msg), - qos => emqx_message:qos(Msg), - retain => emqx_message:get_flag(retain, Msg), - payload => encode(emqx_message:payload(Msg), Encode), - ts => emqx_message:timestamp(Msg) - }) - end, - true - end). - -%%-------------------------------------------------------------------- -%% Helper -%%-------------------------------------------------------------------- -do_setup() -> - %% Pre-defined envs - application:set_env(emqx_web_hook, path, "path"), - application:set_env(emqx_web_hook, headers, []), - - meck:new(ehttpc_pool, [passthrough, no_history]), - meck:expect(ehttpc_pool, pick_worker, fun(_, _) -> ok end), - - Self = self(), - meck:new(ehttpc, [passthrough, no_history]), - meck:expect(ehttpc, request, - fun(_ClientId, Method, {Path, Headers, Body}) -> - Self ! {Method, Path, Headers, Body}, {ok, 200, ok} - end), - - meck:new(emqx_metrics, [passthrough, no_history]), - meck:expect(emqx_metrics, inc, fun(_) -> ok end), - ok. - -do_teardown(_) -> - meck:unload(ehttpc_pool), - meck:unload(ehttpc), - meck:unload(emqx_metrics). - -maybe(undefined) -> null; -maybe(T) -> T. - -peer2addr({Host, _}) -> - list_to_binary(inet:ntoa(Host)); -peer2addr(Host) -> - list_to_binary(inet:ntoa(Host)). - -stringfy({shutdown, Reason}) -> - stringfy(Reason); -stringfy(Term) when is_binary(Term) -> - Term; -stringfy(Term) when is_atom(Term) -> - atom_to_binary(Term, utf8); -stringfy(Term) -> - unicode:characters_to_binary(io_lib:format("~0p", [Term])). - -receive_http_request_body() -> - receive - {post, _, _, Body} -> - Body - after 100 -> - exit(waiting_message_timeout) - end. - -filter_topictab(TopicTab, {undefined}) -> - TopicTab; -filter_topictab(TopicTab, {TopicFilter}) -> - lists:filter(fun({Topic, _}) -> emqx_topic:match(Topic, TopicFilter) end, TopicTab). - -filter_topic_match(_Topic, {undefined}) -> - true; -filter_topic_match(Topic, {TopicFilter}) -> - emqx_topic:match(Topic, TopicFilter). - -encode(Bin, base64) -> - base64:encode(Bin); -encode(Bin, base62) -> - emqx_base62:encode(Bin); -encode(Bin, _) -> - Bin. - -%%-------------------------------------------------------------------- -%% Generators -%%-------------------------------------------------------------------- - -conn_properties() -> - #{}. - -ack_properties() -> - #{}. - -sub_properties() -> - #{}. - -unsub_properties() -> - #{}. - -shutdown_reason() -> - oneof([disconnected, not_autherised, - "list_reason", <<"binary_reason">>, - {tuple, reason}, - {shutdown, emqx_ct_proper_types:limited_atom()}]). - -empty_env() -> - {undefined}. - -topic_filter_env() -> - oneof([{<<"#">>}, {undefined}, {topic()}]). - -payload_encode() -> - oneof([base62, base64, plain]). - -disconnected_conninfo() -> - ?LET(Info, conninfo(), - begin - Info#{disconnected_at => erlang:system_time(millisecond)} - end). diff --git a/bin/emqx b/bin/emqx index a5f76ac72..bc1d00e35 100755 --- a/bin/emqx +++ b/bin/emqx @@ -3,6 +3,11 @@ # ex: ts=4 sw=4 et set -e +set -o pipefail + +if [ -n "$DEBUG" ]; then + set -x +fi ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" # shellcheck disable=SC1090 @@ -36,6 +41,12 @@ export LD_LIBRARY_PATH="$ERTS_DIR/lib:$LD_LIBRARY_PATH" export ERTS_LIB_DIR="$ERTS_DIR/../lib" MNESIA_DATA_DIR="$RUNNER_DATA_DIR/mnesia/$NAME" +die() { + echo >&2 "$1" + errno=${2:-1} + exit "$errno" +} + relx_usage() { command="$1" @@ -156,11 +167,6 @@ relx_get_pid() { fi } -relx_get_nodename() { - id="longname$(relx_gen_id)-${NAME}" - "$BINDIR/erl" -boot "$REL_DIR/start_clean" -eval '[Host] = tl(string:tokens(atom_to_list(node()),"@")), io:format("~s~n", [Host]), halt()' -noshell "${NAME_TYPE}" "$id" -} - # Connect to a remote node relx_rem_sh() { # Generate a unique id used to allow multiple remsh to the same node @@ -194,8 +200,10 @@ relx_nodetool() { call_hocon() { export RUNNER_ROOT_DIR + export RUNNER_ETC_DIR export REL_VSN - "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" hocon "$@" + "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" hocon "$@" \ + || die "ERROR: call_hocon failed: $*" $? } # Run an escript in the node's environment @@ -214,6 +222,8 @@ relx_start_command() { # Function to generate app.config and vm.args generate_config() { + local name_type="$1" + local node_name="$2" ## Delete the *.siz files first or it cann't start after ## changing the config 'log.rotation.size' rm -rf "${RUNNER_LOG_DIR}"/*.siz @@ -251,26 +261,26 @@ generate_config() { ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}') ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}') ## use the key to look up in vm.args file for the value - TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}') + TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" || true | awk '{print $NF}') ## compare generated (to override) value to original (to be overriden) value if [ "$ARG_VALUE" != "$TMP_ARG_VALUE" ] ; then ## if they are different if [ -n "$TMP_ARG_VALUE" ]; then ## if the old value is present, replace it with generated value - sh -c "$SED_REPLACE 's/^$ARG_KEY.*$/$ARG_LINE/' $TMP_ARG_FILE" + sh -c "$SED_REPLACE 's|^$ARG_KEY.*$|$ARG_LINE|' $TMP_ARG_FILE" else ## otherwise append generated value to the end echo "$ARG_LINE" >> "$TMP_ARG_FILE" fi fi done + echo "$name_type $node_name" >> "$TMP_ARG_FILE" ## rename the generated vm.