diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 7c0a5dc87..64fc6d5b0 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -25,7 +25,7 @@ jobs: prepare: runs-on: ubuntu-22.04 # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-24.3.4.2-3-ubuntu22.04" outputs: PROFILE: ${{ steps.get_profile.outputs.PROFILE }} @@ -121,7 +121,7 @@ jobs: # NOTE: 'otp' and 'elixir' are to configure emqx-builder image # only support latest otp and elixir, not a matrix builder: - - 5.0-34 # update to latest + - 5.0-35 # update to latest otp: - 24.3.4.2-3 # switch to 25 once ready to release 5.1 elixir: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index eab9dc115..80a87662a 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -21,7 +21,7 @@ on: jobs: prepare: runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-24.3.4.2-3-ubuntu22.04 outputs: BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} @@ -102,9 +102,16 @@ jobs: - name: run emqx timeout-minutes: 5 run: | + $ErrorActionPreference = "Stop" ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start - Start-Sleep -s 5 - echo "EMQX started" + Start-Sleep -s 10 + $pingOutput = ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx ping + if ($pingOutput = 'pong') { + echo "EMQX started OK" + } else { + echo "Failed to ping EMQX $pingOutput" + Exit 1 + } ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop echo "EMQX stopped" ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install @@ -184,7 +191,7 @@ jobs: - aws-arm64 - ubuntu-22.04 builder: - - 5.0-34 + - 5.0-35 elixir: - 1.13.4 exclude: @@ -198,7 +205,7 @@ jobs: arch: amd64 os: ubuntu22.04 build_machine: ubuntu-22.04 - builder: 5.0-34 + builder: 5.0-35 elixir: 1.13.4 release_with: elixir - profile: emqx @@ -206,7 +213,7 @@ jobs: arch: amd64 os: amzn2 build_machine: ubuntu-22.04 - builder: 5.0-34 + builder: 5.0-35 elixir: 1.13.4 release_with: elixir @@ -306,35 +313,3 @@ jobs: fi aws s3 cp --recursive packages/$PROFILE s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$s3dir/${{ github.ref_name }}/*" - - name: Push to packagecloud.io - env: - PROFILE: ${{ matrix.profile }} - VERSION: ${{ needs.prepare.outputs.VERSION }} - PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - run: | - set -eu - REPO=$PROFILE - if [ $PROFILE = 'emqx-enterprise' ]; then - REPO='emqx-enterprise5' - fi - function push() { - docker run -t --rm -e PACKAGECLOUD_TOKEN=$PACKAGECLOUD_TOKEN -v $(pwd)/$2:/w/$2 -w /w ghcr.io/emqx/package_cloud push emqx/$REPO/$1 $2 - } - push "debian/buster" "packages/$PROFILE/$PROFILE-$VERSION-debian10-amd64.deb" - push "debian/buster" "packages/$PROFILE/$PROFILE-$VERSION-debian10-arm64.deb" - push "debian/bullseye" "packages/$PROFILE/$PROFILE-$VERSION-debian11-amd64.deb" - push "debian/bullseye" "packages/$PROFILE/$PROFILE-$VERSION-debian11-arm64.deb" - push "ubuntu/bionic" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu18.04-amd64.deb" - push "ubuntu/bionic" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu18.04-arm64.deb" - push "ubuntu/focal" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu20.04-amd64.deb" - push "ubuntu/focal" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu20.04-arm64.deb" - push "ubuntu/jammy" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu22.04-amd64.deb" - push "ubuntu/jammy" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu22.04-arm64.deb" - push "el/6" "packages/$PROFILE/$PROFILE-$VERSION-amzn2-amd64.rpm" - push "el/6" "packages/$PROFILE/$PROFILE-$VERSION-amzn2-arm64.rpm" - push "el/7" "packages/$PROFILE/$PROFILE-$VERSION-el7-amd64.rpm" - push "el/7" "packages/$PROFILE/$PROFILE-$VERSION-el7-arm64.rpm" - push "el/8" "packages/$PROFILE/$PROFILE-$VERSION-el8-amd64.rpm" - push "el/8" "packages/$PROFILE/$PROFILE-$VERSION-el8-arm64.rpm" - push "el/9" "packages/$PROFILE/$PROFILE-$VERSION-el9-amd64.rpm" - push "el/9" "packages/$PROFILE/$PROFILE-$VERSION-el9-arm64.rpm" diff --git a/.github/workflows/build_packages_cron.yaml b/.github/workflows/build_packages_cron.yaml index 7f6773f4a..8498b30ba 100644 --- a/.github/workflows/build_packages_cron.yaml +++ b/.github/workflows/build_packages_cron.yaml @@ -24,9 +24,6 @@ jobs: profile: - ['emqx', 'master'] - ['emqx-enterprise', 'release-50'] - branch: - - master - - release-50 otp: - 24.3.4.2-3 arch: @@ -35,7 +32,7 @@ jobs: - debian10 - amzn2 builder: - - 5.0-34 + - 5.0-35 elixir: - 1.13.4 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 06bcb98a2..7e664c1c7 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -35,7 +35,7 @@ jobs: - ["emqx-enterprise", "24.3.4.2-3", "amzn2", "erlang"] - ["emqx-enterprise", "25.1.2-3", "ubuntu20.04", "erlang"] builder: - - 5.0-34 + - 5.0-35 elixir: - '1.13.4' @@ -111,8 +111,14 @@ jobs: timeout-minutes: 5 run: | ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start - Start-Sleep -s 5 - echo "EMQX started" + Start-Sleep -s 10 + $pingOutput = ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx ping + if ($pingOutput = 'pong') { + echo "EMQX started OK" + } else { + echo "Failed to ping EMQX $pingOutput" + Exit 1 + } ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop echo "EMQX stopped" ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 52ebf9efc..83e328d21 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -6,7 +6,7 @@ on: jobs: check_deps_integrity: runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-25.1.2-3-ubuntu22.04 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml index 1508cdd6e..8baa7e721 100644 --- a/.github/workflows/code_style_check.yaml +++ b/.github/workflows/code_style_check.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: code_style_check: runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-25.1.2-3-ubuntu22.04" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml index 7e942f3f3..123b79566 100644 --- a/.github/workflows/elixir_apps_check.yaml +++ b/.github/workflows/elixir_apps_check.yaml @@ -9,7 +9,7 @@ jobs: elixir_apps_check: runs-on: ubuntu-22.04 # just use the latest builder - container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-25.1.2-3-ubuntu22.04" strategy: fail-fast: false diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml index e967c186b..99afd109e 100644 --- a/.github/workflows/elixir_deps_check.yaml +++ b/.github/workflows/elixir_deps_check.yaml @@ -8,7 +8,7 @@ on: jobs: elixir_deps_check: runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-25.1.2-3-ubuntu22.04 steps: - name: Checkout diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml index 9a916d332..14b6f941f 100644 --- a/.github/workflows/elixir_release.yml +++ b/.github/workflows/elixir_release.yml @@ -17,7 +17,7 @@ jobs: profile: - emqx - emqx-enterprise - container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-25.1.2-3-ubuntu22.04 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/performance_test.yaml b/.github/workflows/performance_test.yaml index 1d474f7b2..5283de51a 100644 --- a/.github/workflows/performance_test.yaml +++ b/.github/workflows/performance_test.yaml @@ -15,7 +15,7 @@ jobs: prepare: runs-on: ubuntu-latest if: github.repository_owner == 'emqx' - container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-25.1.2-3-ubuntu20.04 outputs: BENCH_ID: ${{ steps.prepare.outputs.BENCH_ID }} PACKAGE_FILE: ${{ steps.package_file.outputs.PACKAGE_FILE }} @@ -51,11 +51,10 @@ jobs: needs: - prepare env: - TF_VAR_bench_id: ${{ needs.prepare.outputs.BENCH_ID }} TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }} - TF_VAR_test_duration: 300 TF_VAR_grafana_api_key: ${{ secrets.TF_EMQX_PERF_TEST_GRAFANA_API_KEY }} TF_AWS_REGION: eu-north-1 + TF_VAR_test_duration: 1800 steps: - name: Configure AWS Credentials @@ -77,38 +76,37 @@ jobs: uses: hashicorp/setup-terraform@v2 with: terraform_wrapper: false - - name: terraform init + - name: 1on1 scenario + id: scenario_1on1 working-directory: ./tf-emqx-performance-test + timeout-minutes: 60 + env: + TF_VAR_bench_id: "${{ needs.prepare.outputs.BENCH_ID }}/1on1" + TF_VAR_use_emqttb: 1 + TF_VAR_use_emqtt_bench: 0 + TF_VAR_emqttb_instance_count: 2 + TF_VAR_emqttb_instance_type: "c5.large" + TF_VAR_emqttb_scenario: "@pub --topic 't/%n' --pubinterval 10ms --qos 1 --publatency 50ms --size 16 --num-clients 25000 @sub --topic 't/%n' --num-clients 25000" + TF_VAR_emqx_instance_type: "c5.xlarge" + TF_VAR_emqx_instance_count: 3 run: | terraform init - - name: terraform apply - working-directory: ./tf-emqx-performance-test - run: | terraform apply -auto-approve - - name: Wait for test results - timeout-minutes: 30 - working-directory: ./tf-emqx-performance-test - id: test-results - run: | - sleep $TF_VAR_test_duration - until aws s3api head-object --bucket tf-emqx-performance-test --key "$TF_VAR_bench_id/DONE" > /dev/null 2>&1 - do - printf '.' - sleep 10 - done - echo - aws s3 cp "s3://tf-emqx-performance-test/$TF_VAR_bench_id/metrics.json" ./ - aws s3 cp "s3://tf-emqx-performance-test/$TF_VAR_bench_id/stats.json" ./ - echo MESSAGES_DELIVERED=$(cat metrics.json | jq '[.[]."messages.delivered"] | add') >> $GITHUB_OUTPUT - echo MESSAGES_DROPPED=$(cat metrics.json | jq '[.[]."messages.dropped"] | add') >> $GITHUB_OUTPUT + ./wait-emqttb.sh + ./fetch-metrics.sh + MESSAGES_RECEIVED=$(cat metrics.json | jq '[.[]."messages.received"] | add') + MESSAGES_SENT=$(cat metrics.json | jq '[.[]."messages.sent"] | add') + echo MESSAGES_DROPPED=$(cat metrics.json | jq '[.[]."messages.dropped"] | add') >> $GITHUB_OUTPUT + echo PUB_MSG_RATE=$(($MESSAGES_RECEIVED / $TF_VAR_test_duration)) >> $GITHUB_OUTPUT + echo SUB_MSG_RATE=$(($MESSAGES_SENT / $TF_VAR_test_duration)) >> $GITHUB_OUTPUT + terraform destroy -auto-approve - name: Send notification to Slack - if: success() uses: slackapi/slack-github-action@v1.23.0 env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} with: payload: | - {"text": "EMQX performance test completed.\nMessages delivered: ${{ steps.test-results.outputs.MESSAGES_DELIVERED }}.\nMessages dropped: ${{ steps.test-results.outputs.MESSAGES_DROPPED }}.\nhttps://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} + {"text": "Performance test result for 1on1 scenario (50k pub, 50k sub): ${{ job.status }}\nhttps://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}\n*Pub message rate*: ${{ steps.scenario_1on1.outputs.PUB_MSG_RATE }}\n*Sub message rate*: ${{ steps.scenario_1on1.outputs.SUB_MSG_RATE }}\nDropped messages: ${{ steps.scenario_1on1.outputs.MESSAGES_DROPPED }}"} - name: terraform destroy if: always() working-directory: ./tf-emqx-performance-test @@ -117,10 +115,10 @@ jobs: - uses: actions/upload-artifact@v3 if: success() with: - name: test-results - path: "./tf-emqx-performance-test/*.json" + name: metrics + path: "./tf-emqx-performance-test/metrics.json" - uses: actions/upload-artifact@v3 - if: always() + if: failure() with: name: terraform path: | diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 32a45bd51..30de6f3b1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -15,7 +15,7 @@ on: jobs: upload: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false steps: @@ -53,16 +53,6 @@ jobs: BUCKET=${{ secrets.AWS_S3_BUCKET }} OUTPUT_DIR=${{ steps.profile.outputs.s3dir }} aws s3 cp --recursive s3://$BUCKET/$OUTPUT_DIR/${{ github.ref_name }} packages - cd packages - DEFAULT_BEAM_PLATFORM='otp24.3.4.2-3' - # all packages including full-name and default-name are uploaded to s3 - # but we only upload default-name packages (and elixir) as github artifacts - # so we rename (overwrite) non-default packages before uploading - while read -r fname; do - default_fname=$(echo "$fname" | sed "s/-${DEFAULT_BEAM_PLATFORM}//g") - echo "$fname -> $default_fname" - mv -f "$fname" "$default_fname" - done < <(find . -maxdepth 1 -type f | grep -E "emqx(-enterprise)?-5\.[0-9]+\.[0-9]+.*-${DEFAULT_BEAM_PLATFORM}" | grep -v elixir) - uses: alexellis/upload-assets@0.4.0 env: GITHUB_TOKEN: ${{ github.token }} @@ -79,3 +69,35 @@ jobs: -X POST \ -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} + - name: Push to packagecloud.io + env: + PROFILE: ${{ steps.profile.outputs.profile }} + VERSION: ${{ steps.profile.outputs.version }} + PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} + run: | + set -eu + REPO=$PROFILE + if [ $PROFILE = 'emqx-enterprise' ]; then + REPO='emqx-enterprise5' + fi + function push() { + docker run -t --rm -e PACKAGECLOUD_TOKEN=$PACKAGECLOUD_TOKEN -v $(pwd)/$2:/w/$2 -w /w ghcr.io/emqx/package_cloud push emqx/$REPO/$1 $2 + } + push "debian/buster" "packages/$PROFILE-$VERSION-debian10-amd64.deb" + push "debian/buster" "packages/$PROFILE-$VERSION-debian10-arm64.deb" + push "debian/bullseye" "packages/$PROFILE-$VERSION-debian11-amd64.deb" + push "debian/bullseye" "packages/$PROFILE-$VERSION-debian11-arm64.deb" + push "ubuntu/bionic" "packages/$PROFILE-$VERSION-ubuntu18.04-amd64.deb" + push "ubuntu/bionic" "packages/$PROFILE-$VERSION-ubuntu18.04-arm64.deb" + push "ubuntu/focal" "packages/$PROFILE-$VERSION-ubuntu20.04-amd64.deb" + push "ubuntu/focal" "packages/$PROFILE-$VERSION-ubuntu20.04-arm64.deb" + push "ubuntu/jammy" "packages/$PROFILE-$VERSION-ubuntu22.04-amd64.deb" + push "ubuntu/jammy" "packages/$PROFILE-$VERSION-ubuntu22.04-arm64.deb" + push "el/6" "packages/$PROFILE-$VERSION-amzn2-amd64.rpm" + push "el/6" "packages/$PROFILE-$VERSION-amzn2-arm64.rpm" + push "el/7" "packages/$PROFILE-$VERSION-el7-amd64.rpm" + push "el/7" "packages/$PROFILE-$VERSION-el7-arm64.rpm" + push "el/8" "packages/$PROFILE-$VERSION-el8-amd64.rpm" + push "el/8" "packages/$PROFILE-$VERSION-el8-arm64.rpm" + push "el/9" "packages/$PROFILE-$VERSION-el9-amd64.rpm" + push "el/9" "packages/$PROFILE-$VERSION-el9-arm64.rpm" diff --git a/.github/workflows/run_conf_tests.yaml b/.github/workflows/run_conf_tests.yaml index 93d9fee95..7e153b386 100644 --- a/.github/workflows/run_conf_tests.yaml +++ b/.github/workflows/run_conf_tests.yaml @@ -1,7 +1,7 @@ name: Run Configuration tests concurrency: - group: test-${{ github.event_name }}-${{ github.ref }} + group: conftest-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: true on: diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index 551d0d9e6..ffed8d0c8 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -12,9 +12,8 @@ jobs: strategy: matrix: builder: - - 5.0-34 + - 5.0-35 otp: - - 24.3.4.2-3 - 25.1.2-3 # no need to use more than 1 version of Elixir, since tests # run using only Erlang code. This is needed just to specify diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 185c76be1..b1246f745 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -17,7 +17,7 @@ jobs: prepare: runs-on: ubuntu-22.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-debian11 + container: ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-24.3.4.2-3-debian11 steps: - uses: actions/checkout@v3 @@ -50,7 +50,7 @@ jobs: os: - ["debian11", "debian:11-slim"] builder: - - 5.0-34 + - 5.0-35 otp: - 24.3.4.2-3 elixir: @@ -123,7 +123,7 @@ jobs: os: - ["debian11", "debian:11-slim"] builder: - - 5.0-34 + - 5.0-35 otp: - 24.3.4.2-3 elixir: diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 2ad7c3345..8f5d9f6ae 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -15,7 +15,7 @@ concurrency: jobs: relup_test_plan: runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-24.3.4.2-3-ubuntu22.04" outputs: CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }} OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }} diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index c28ebc0bc..f96798493 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -34,12 +34,12 @@ jobs: MATRIX="$(echo "${APPS}" | jq -c ' [ (.[] | select(.profile == "emqx") | . + { - builder: "5.0-34", + builder: "5.0-35", otp: "25.1.2-3", elixir: "1.13.4" }), (.[] | select(.profile == "emqx-enterprise") | . + { - builder: "5.0-34", + builder: "5.0-35", otp: ["24.3.4.2-3", "25.1.2-3"][], elixir: "1.13.4" }) @@ -109,7 +109,9 @@ jobs: - uses: actions/cache@v3 with: path: "source/emqx_dialyzer_${{ matrix.otp }}_plt" - key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }} + key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-${{ hashFiles('source/rebar.*', 'source/apps/*/rebar.*', 'source/lib-ee/*/rebar.*') }} + restore-keys: | + rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}- - name: run static checks env: PROFILE: ${{ matrix.profile }} @@ -256,7 +258,7 @@ jobs: - ct - ct_docker runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-24.3.4.2-3-ubuntu22.04" steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 diff --git a/apps/emqx/include/asserts.hrl b/apps/emqx/include/asserts.hrl index 1be725d2d..4936da1f9 100644 --- a/apps/emqx/include/asserts.hrl +++ b/apps/emqx/include/asserts.hrl @@ -30,16 +30,32 @@ ) ). --define(assertInclude(PATTERN, LIST), - ?assert( - lists:any( - fun(X__Elem_) -> - case X__Elem_ of - PATTERN -> true; - _ -> false - end - end, - LIST - ) - ) +-define(drainMailbox(), + (fun F__Flush_() -> + receive + X__Msg_ -> [X__Msg_ | F__Flush_()] + after 0 -> [] + end + end)() +). + +-define(assertReceive(PATTERN), + ?assertReceive(PATTERN, 1000) +). + +-define(assertReceive(PATTERN, TIMEOUT), + (fun() -> + receive + X__V = PATTERN -> X__V + after TIMEOUT -> + erlang:error( + {assertReceive, [ + {module, ?MODULE}, + {line, ?LINE}, + {expression, (??PATTERN)}, + {mailbox, ?drainMailbox()} + ]} + ) + end + end)() ). diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index ec78091b7..f9154f987 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Community edition --define(EMQX_RELEASE_CE, "5.0.25"). +-define(EMQX_RELEASE_CE, "5.0.26-alpha.1"). %% Enterprise edition -define(EMQX_RELEASE_EE, "5.0.4"). diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index aabe80b7d..55fe1bf42 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -45,6 +45,5 @@ {emqx_rule_engine,1}. {emqx_shared_sub,1}. {emqx_slow_subs,1}. -{emqx_statsd,1}. {emqx_telemetry,1}. {emqx_topic_metrics,1}. diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 425c49fb3..fd7855004 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -27,9 +27,9 @@ {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.1"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.2"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.4"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.7"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/apps/emqx/src/config/emqx_config_logger.erl b/apps/emqx/src/config/emqx_config_logger.erl index b0fc1ca67..a7bf54aee 100644 --- a/apps/emqx/src/config/emqx_config_logger.erl +++ b/apps/emqx/src/config/emqx_config_logger.erl @@ -112,8 +112,8 @@ update_log_handler({Action, {handler, Id, Mod, Conf}}) -> end, ok. -id_for_log(console) -> "log.console_handler"; -id_for_log(Other) -> "log.file_handlers." ++ atom_to_list(Other). +id_for_log(console) -> "log.console"; +id_for_log(Other) -> "log.file." ++ atom_to_list(Other). atom(Id) when is_binary(Id) -> binary_to_atom(Id, utf8); atom(Id) when is_atom(Id) -> Id. @@ -126,12 +126,12 @@ tr_handlers(Conf) -> %% For the default logger that outputs to console tr_console_handler(Conf) -> - case conf_get("log.console_handler.enable", Conf) of + case conf_get("log.console.enable", Conf) of true -> - ConsoleConf = conf_get("log.console_handler", Conf), + ConsoleConf = conf_get("log.console", Conf), [ {handler, console, logger_std_h, #{ - level => conf_get("log.console_handler.level", Conf), + level => conf_get("log.console.level", Conf), config => (log_handler_conf(ConsoleConf))#{type => standard_io}, formatter => log_formatter(ConsoleConf), filters => log_filter(ConsoleConf) @@ -150,14 +150,10 @@ tr_file_handler({HandlerName, SubConf}) -> {handler, atom(HandlerName), logger_disk_log_h, #{ level => conf_get("level", SubConf), config => (log_handler_conf(SubConf))#{ - type => - case conf_get("rotation.enable", SubConf) of - true -> wrap; - _ -> halt - end, - file => conf_get("file", SubConf), - max_no_files => conf_get("rotation.count", SubConf), - max_no_bytes => conf_get("max_size", SubConf) + type => wrap, + file => conf_get("to", SubConf), + max_no_files => conf_get("rotation_count", SubConf), + max_no_bytes => conf_get("rotation_size", SubConf) }, formatter => log_formatter(SubConf), filters => log_filter(SubConf), @@ -165,14 +161,11 @@ tr_file_handler({HandlerName, SubConf}) -> }}. logger_file_handlers(Conf) -> - Handlers = maps:to_list(conf_get("log.file_handlers", Conf, #{})), lists:filter( - fun({_Name, Opts}) -> - B = conf_get("enable", Opts), - true = is_boolean(B), - B + fun({_Name, Handler}) -> + conf_get("enable", Handler, false) end, - Handlers + maps:to_list(conf_get("log.file", Conf, #{})) ). conf_get(Key, Conf) -> emqx_schema:conf_get(Key, Conf). @@ -237,12 +230,8 @@ log_filter(Conf) -> end. tr_level(Conf) -> - ConsoleLevel = conf_get("log.console_handler.level", Conf, undefined), - FileLevels = [ - conf_get("level", SubConf) - || {_, SubConf} <- - logger_file_handlers(Conf) - ], + ConsoleLevel = conf_get("log.console.level", Conf, undefined), + FileLevels = [conf_get("level", SubConf) || {_, SubConf} <- logger_file_handlers(Conf)], case FileLevels ++ [ConsoleLevel || ConsoleLevel =/= undefined] of %% warning is the default level we should use [] -> warning; diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index be68b438f..2a2b3c323 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -3,7 +3,7 @@ {id, "emqx"}, {description, "EMQX Core"}, % strict semver, bump manually! - {vsn, "5.0.26"}, + {vsn, "5.0.27"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl index ffee5fba7..1cdb563aa 100644 --- a/apps/emqx/src/emqx.erl +++ b/apps/emqx/src/emqx.erl @@ -184,11 +184,18 @@ run_fold_hook(HookPoint, Args, Acc) -> -spec get_config(emqx_utils_maps:config_key_path()) -> term(). get_config(KeyPath) -> - emqx_config:get(KeyPath). + KeyPath1 = emqx_config:ensure_atom_conf_path(KeyPath, {raise_error, config_not_found}), + emqx_config:get(KeyPath1). -spec get_config(emqx_utils_maps:config_key_path(), term()) -> term(). get_config(KeyPath, Default) -> - emqx_config:get(KeyPath, Default). + try + KeyPath1 = emqx_config:ensure_atom_conf_path(KeyPath, {raise_error, config_not_found}), + emqx_config:get(KeyPath1, Default) + catch + error:config_not_found -> + Default + end. -spec get_raw_config(emqx_utils_maps:config_key_path()) -> term(). get_raw_config(KeyPath) -> diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl index be3b35f57..98c0a19f8 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -29,9 +29,13 @@ authn_type/1 ]). --ifdef(TEST). --export([convert_certs/2, convert_certs/3, clear_certs/2]). --endif. +%% Used in emqx_gateway +-export([ + certs_dir/2, + convert_certs/2, + convert_certs/3, + clear_certs/2 +]). -export_type([config/0]). diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 88e8669cd..5637bb171 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -256,9 +256,7 @@ init( ), {NClientInfo, NConnInfo} = take_ws_cookie(ClientInfo, ConnInfo), #channel{ - %% We remove the peercert because it duplicates to what's stored in the socket, - %% Saving a copy here causes unnecessary wast of memory (about 1KB per connection). - conninfo = maps:put(peercert, undefined, NConnInfo), + conninfo = NConnInfo, clientinfo = NClientInfo, topic_aliases = #{ inbound => #{}, @@ -1217,7 +1215,7 @@ handle_call( } ) -> ClientId = info(clientid, Channel), - NKeepalive = emqx_keepalive:set(interval, Interval * 1000, KeepAlive), + NKeepalive = emqx_keepalive:update(timer:seconds(Interval), KeepAlive), NConnInfo = maps:put(keepalive, Interval, ConnInfo), NChannel = Channel#channel{keepalive = NKeepalive, conninfo = NConnInfo}, SockInfo = maps:get(sockinfo, emqx_cm:get_chan_info(ClientId), #{}), @@ -2004,10 +2002,21 @@ ensure_connected( NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, ok = run_hooks('client.connected', [ClientInfo, NConnInfo]), Channel#channel{ - conninfo = NConnInfo, + conninfo = trim_conninfo(NConnInfo), conn_state = connected }. +trim_conninfo(ConnInfo) -> + maps:without( + [ + %% NOTE + %% We remove the peercert because it duplicates what's stored in the socket, + %% otherwise it wastes about 1KB per connection. + peercert + ], + ConnInfo + ). + %%-------------------------------------------------------------------- %% Init Alias Maximum @@ -2040,9 +2049,9 @@ ensure_keepalive_timer(0, Channel) -> ensure_keepalive_timer(disabled, Channel) -> Channel; ensure_keepalive_timer(Interval, Channel = #channel{clientinfo = #{zone := Zone}}) -> - Backoff = get_mqtt_conf(Zone, keepalive_backoff), - RecvOct = emqx_pd:get_counter(incoming_bytes), - Keepalive = emqx_keepalive:init(RecvOct, round(timer:seconds(Interval) * Backoff)), + Multiplier = get_mqtt_conf(Zone, keepalive_multiplier), + RecvCnt = emqx_pd:get_counter(recv_pkt), + Keepalive = emqx_keepalive:init(RecvCnt, round(timer:seconds(Interval) * Multiplier)), ensure_timer(alive_timer, Channel#channel{keepalive = Keepalive}). clear_keepalive(Channel = #channel{timers = Timers}) -> @@ -2151,7 +2160,8 @@ publish_will_msg( ok; false -> NMsg = emqx_mountpoint:mount(MountPoint, Msg), - _ = emqx_broker:publish(NMsg), + NMsg2 = NMsg#message{timestamp = erlang:system_time(millisecond)}, + _ = emqx_broker:publish(NMsg2), ok end. diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 630952166..91809134c 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -88,6 +88,8 @@ remove_handlers/0 ]). +-export([ensure_atom_conf_path/2]). + -ifdef(TEST). -export([erase_all/0]). -endif. @@ -113,7 +115,8 @@ update_cmd/0, update_args/0, update_error/0, - update_result/0 + update_result/0, + runtime_config_key_path/0 ]). -type update_request() :: term(). @@ -144,6 +147,8 @@ -type config() :: #{atom() => term()} | list() | undefined. -type app_envs() :: [proplists:property()]. +-type runtime_config_key_path() :: [atom()]. + %% @doc For the given path, get root value enclosed in a single-key map. -spec get_root(emqx_utils_maps:config_key_path()) -> map(). get_root([RootName | _]) -> @@ -156,25 +161,21 @@ get_root_raw([RootName | _]) -> %% @doc Get a config value for the given path. %% The path should at least include root config name. --spec get(emqx_utils_maps:config_key_path()) -> term(). +-spec get(runtime_config_key_path()) -> term(). get(KeyPath) -> do_get(?CONF, KeyPath). --spec get(emqx_utils_maps:config_key_path(), term()) -> term(). +-spec get(runtime_config_key_path(), term()) -> term(). get(KeyPath, Default) -> do_get(?CONF, KeyPath, Default). --spec find(emqx_utils_maps:config_key_path()) -> +-spec find(runtime_config_key_path()) -> {ok, term()} | {not_found, emqx_utils_maps:config_key_path(), term()}. find([]) -> case do_get(?CONF, [], ?CONFIG_NOT_FOUND_MAGIC) of ?CONFIG_NOT_FOUND_MAGIC -> {not_found, []}; Res -> {ok, Res} end; -find(KeyPath) -> - atom_conf_path( - KeyPath, - fun(AtomKeyPath) -> emqx_utils_maps:deep_find(AtomKeyPath, get_root(KeyPath)) end, - {return, {not_found, KeyPath}} - ). +find(AtomKeyPath) -> + emqx_utils_maps:deep_find(AtomKeyPath, get_root(AtomKeyPath)). -spec find_raw(emqx_utils_maps:config_key_path()) -> {ok, term()} | {not_found, emqx_utils_maps:config_key_path(), term()}. @@ -712,21 +713,14 @@ do_put(Type, Putter, [RootName | KeyPath], DeepValue) -> NewValue = do_deep_put(Type, Putter, KeyPath, OldValue, DeepValue), persistent_term:put(?PERSIS_KEY(Type, RootName), NewValue). -do_deep_get(?CONF, KeyPath, Map, Default) -> - atom_conf_path( - KeyPath, - fun(AtomKeyPath) -> emqx_utils_maps:deep_get(AtomKeyPath, Map, Default) end, - {return, Default} - ); +do_deep_get(?CONF, AtomKeyPath, Map, Default) -> + emqx_utils_maps:deep_get(AtomKeyPath, Map, Default); do_deep_get(?RAW_CONF, KeyPath, Map, Default) -> emqx_utils_maps:deep_get([bin(Key) || Key <- KeyPath], Map, Default). do_deep_put(?CONF, Putter, KeyPath, Map, Value) -> - atom_conf_path( - KeyPath, - fun(AtomKeyPath) -> Putter(AtomKeyPath, Map, Value) end, - {raise_error, {not_found, KeyPath}} - ); + AtomKeyPath = ensure_atom_conf_path(KeyPath, {raise_error, {not_found, KeyPath}}), + Putter(AtomKeyPath, Map, Value); do_deep_put(?RAW_CONF, Putter, KeyPath, Map, Value) -> Putter([bin(Key) || Key <- KeyPath], Map, Value). @@ -773,15 +767,24 @@ conf_key(?CONF, RootName) -> conf_key(?RAW_CONF, RootName) -> bin(RootName). -atom_conf_path(Path, ExpFun, OnFail) -> - try [atom(Key) || Key <- Path] of - AtomKeyPath -> ExpFun(AtomKeyPath) +ensure_atom_conf_path(Path, OnFail) -> + case lists:all(fun erlang:is_atom/1, Path) of + true -> + %% Do not try to build new atom PATH if it already is. + Path; + _ -> + to_atom_conf_path(Path, OnFail) + end. + +to_atom_conf_path(Path, OnFail) -> + try + [atom(Key) || Key <- Path] catch error:badarg -> case OnFail of - {return, Val} -> - Val; {raise_error, Err} -> - error(Err) + error(Err); + {return, V} -> + V end end. diff --git a/apps/emqx/src/emqx_keepalive.erl b/apps/emqx/src/emqx_keepalive.erl index 9ba11e23f..c0a1c7657 100644 --- a/apps/emqx/src/emqx_keepalive.erl +++ b/apps/emqx/src/emqx_keepalive.erl @@ -22,7 +22,7 @@ info/1, info/2, check/2, - set/3 + update/2 ]). -elvis([{elvis_style, no_if_expression, disable}]). @@ -31,66 +31,16 @@ -record(keepalive, { interval :: pos_integer(), - statval :: non_neg_integer(), - repeat :: non_neg_integer() + statval :: non_neg_integer() }). -opaque keepalive() :: #keepalive{}. +-define(MAX_INTERVAL, 65535000). %% @doc Init keepalive. -spec init(Interval :: non_neg_integer()) -> keepalive(). init(Interval) -> init(0, Interval). -%% @doc Init keepalive. --spec init(StatVal :: non_neg_integer(), Interval :: non_neg_integer()) -> keepalive(). -init(StatVal, Interval) when Interval > 0 -> - #keepalive{ - interval = Interval, - statval = StatVal, - repeat = 0 - }. - -%% @doc Get Info of the keepalive. --spec info(keepalive()) -> emqx_types:infos(). -info(#keepalive{ - interval = Interval, - statval = StatVal, - repeat = Repeat -}) -> - #{ - interval => Interval, - statval => StatVal, - repeat => Repeat - }. - --spec info(interval | statval | repeat, keepalive()) -> - non_neg_integer(). -info(interval, #keepalive{interval = Interval}) -> - Interval; -info(statval, #keepalive{statval = StatVal}) -> - StatVal; -info(repeat, #keepalive{repeat = Repeat}) -> - Repeat. - -%% @doc Check keepalive. --spec check(non_neg_integer(), keepalive()) -> - {ok, keepalive()} | {error, timeout}. -check( - NewVal, - KeepAlive = #keepalive{ - statval = OldVal, - repeat = Repeat - } -) -> - if - NewVal =/= OldVal -> - {ok, KeepAlive#keepalive{statval = NewVal, repeat = 0}}; - Repeat < 1 -> - {ok, KeepAlive#keepalive{repeat = Repeat + 1}}; - true -> - {error, timeout} - end. - %% from mqtt-v3.1.1 specific %% A Keep Alive value of zero (0) has the effect of turning off the keep alive mechanism. %% This means that, in this case, the Server is not required @@ -102,7 +52,43 @@ check( %%The actual value of the Keep Alive is application specific; %% typically this is a few minutes. %% The maximum value is (65535s) 18 hours 12 minutes and 15 seconds. -%% @doc Update keepalive's interval --spec set(interval, non_neg_integer(), keepalive()) -> keepalive(). -set(interval, Interval, KeepAlive) when Interval >= 0 andalso Interval =< 65535000 -> - KeepAlive#keepalive{interval = Interval}. +%% @doc Init keepalive. +-spec init(StatVal :: non_neg_integer(), Interval :: non_neg_integer()) -> keepalive() | undefined. +init(StatVal, Interval) when Interval > 0 andalso Interval =< ?MAX_INTERVAL -> + #keepalive{interval = Interval, statval = StatVal}; +init(_, 0) -> + undefined; +init(StatVal, Interval) when Interval > ?MAX_INTERVAL -> init(StatVal, ?MAX_INTERVAL). + +%% @doc Get Info of the keepalive. +-spec info(keepalive()) -> emqx_types:infos(). +info(#keepalive{ + interval = Interval, + statval = StatVal +}) -> + #{ + interval => Interval, + statval => StatVal + }. + +-spec info(interval | statval, keepalive()) -> + non_neg_integer(). +info(interval, #keepalive{interval = Interval}) -> + Interval; +info(statval, #keepalive{statval = StatVal}) -> + StatVal; +info(interval, undefined) -> + 0. + +%% @doc Check keepalive. +-spec check(non_neg_integer(), keepalive()) -> + {ok, keepalive()} | {error, timeout}. +check(Val, #keepalive{statval = Val}) -> {error, timeout}; +check(Val, KeepAlive) -> {ok, KeepAlive#keepalive{statval = Val}}. + +%% @doc Update keepalive. +%% The statval of the previous keepalive will be used, +%% and normal checks will begin from the next cycle. +-spec update(non_neg_integer(), keepalive() | undefined) -> keepalive() | undefined. +update(Interval, undefined) -> init(0, Interval); +update(Interval, #keepalive{statval = StatVal}) -> init(StatVal, Interval). diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 2b80000dc..acdb9ff96 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -277,9 +277,8 @@ restart_listener(Type, ListenerName, Conf) -> restart_listener(Type, ListenerName, Conf, Conf). restart_listener(Type, ListenerName, OldConf, NewConf) -> - case do_stop_listener(Type, ListenerName, OldConf) of + case stop_listener(Type, ListenerName, OldConf) of ok -> start_listener(Type, ListenerName, NewConf); - {error, not_found} -> start_listener(Type, ListenerName, NewConf); {error, Reason} -> {error, Reason} end. @@ -296,42 +295,63 @@ stop_listener(ListenerId) -> apply_on_listener(ListenerId, fun stop_listener/3). stop_listener(Type, ListenerName, #{bind := Bind} = Conf) -> - case do_stop_listener(Type, ListenerName, Conf) of + Id = listener_id(Type, ListenerName), + ok = del_limiter_bucket(Id, Conf), + case do_stop_listener(Type, Id, Conf) of ok -> console_print( "Listener ~ts on ~ts stopped.~n", - [listener_id(Type, ListenerName), format_bind(Bind)] + [Id, format_bind(Bind)] ), ok; {error, not_found} -> - ?ELOG( - "Failed to stop listener ~ts on ~ts: ~0p~n", - [listener_id(Type, ListenerName), format_bind(Bind), already_stopped] - ), ok; {error, Reason} -> ?ELOG( "Failed to stop listener ~ts on ~ts: ~0p~n", - [listener_id(Type, ListenerName), format_bind(Bind), Reason] + [Id, format_bind(Bind), Reason] ), {error, Reason} end. -spec do_stop_listener(atom(), atom(), map()) -> ok | {error, term()}. -do_stop_listener(Type, ListenerName, #{bind := ListenOn} = Conf) when Type == tcp; Type == ssl -> - Id = listener_id(Type, ListenerName), - del_limiter_bucket(Id, Conf), +do_stop_listener(Type, Id, #{bind := ListenOn}) when Type == tcp; Type == ssl -> esockd:close(Id, ListenOn); -do_stop_listener(Type, ListenerName, Conf) when Type == ws; Type == wss -> - Id = listener_id(Type, ListenerName), - del_limiter_bucket(Id, Conf), - cowboy:stop_listener(Id); -do_stop_listener(quic, ListenerName, Conf) -> - Id = listener_id(quic, ListenerName), - del_limiter_bucket(Id, Conf), +do_stop_listener(Type, Id, #{bind := ListenOn}) when Type == ws; Type == wss -> + case cowboy:stop_listener(Id) of + ok -> + wait_listener_stopped(ListenOn); + Error -> + Error + end; +do_stop_listener(quic, Id, _Conf) -> quicer:stop_listener(Id). +wait_listener_stopped(ListenOn) -> + % NOTE + % `cowboy:stop_listener/1` will not close the listening socket explicitly, + % it will be closed by the runtime system **only after** the process exits. + Endpoint = maps:from_list(ip_port(ListenOn)), + case + gen_tcp:connect( + maps:get(ip, Endpoint, loopback), + maps:get(port, Endpoint), + [{active, false}] + ) + of + {error, _EConnrefused} -> + %% NOTE + %% We should get `econnrefused` here because acceptors are already dead + %% but don't want to crash if not, because this doesn't make any difference. + ok; + {ok, Socket} -> + %% NOTE + %% Tiny chance to get a connected socket here, when some other process + %% concurrently binds to the same port. + gen_tcp:close(Socket) + end. + -ifndef(TEST). console_print(Fmt, Args) -> ?ULOG(Fmt, Args). -else. diff --git a/apps/emqx/src/emqx_mqueue.erl b/apps/emqx/src/emqx_mqueue.erl index fbf29d754..fb43941ee 100644 --- a/apps/emqx/src/emqx_mqueue.erl +++ b/apps/emqx/src/emqx_mqueue.erl @@ -129,7 +129,7 @@ init(Opts = #{max_len := MaxLen0, store_qos0 := Qos0}) -> #mqueue{ max_len = MaxLen, store_qos0 = Qos0, - p_table = get_opt(priorities, Opts, ?NO_PRIORITY_TABLE), + p_table = p_table(get_opt(priorities, Opts, ?NO_PRIORITY_TABLE)), default_p = get_priority_opt(Opts), shift_opts = get_shift_opt(Opts) }. @@ -295,3 +295,18 @@ get_shift_opt(Opts) -> multiplier = Mult, base = Base }. + +%% topic from mqtt.mqueue_priorities(map()) is atom. +p_table(PTab = #{}) -> + maps:fold( + fun + (Topic, Priority, Acc) when is_atom(Topic) -> + maps:put(atom_to_binary(Topic), Priority, Acc); + (Topic, Priority, Acc) when is_binary(Topic) -> + maps:put(Topic, Priority, Acc) + end, + #{}, + PTab + ); +p_table(PTab) -> + PTab. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 80894319b..ab3d7bf71 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -78,6 +78,7 @@ validate_heap_size/1, user_lookup_fun_tr/2, validate_alarm_actions/1, + validate_keepalive_multiplier/1, non_empty_string/1, validations/0, naive_env_interpolation/1 @@ -110,7 +111,8 @@ servers_validator/2, servers_sc/2, convert_servers/1, - convert_servers/2 + convert_servers/2, + mqtt_converter/2 ]). %% tombstone types @@ -151,6 +153,8 @@ -define(BIT(Bits), (1 bsl (Bits))). -define(MAX_UINT(Bits), (?BIT(Bits) - 1)). +-define(DEFAULT_MULTIPLIER, 1.5). +-define(DEFAULT_BACKOFF, 0.75). namespace() -> broker. @@ -173,6 +177,7 @@ roots(high) -> ref("mqtt"), #{ desc => ?DESC(mqtt), + converter => fun ?MODULE:mqtt_converter/2, importance => ?IMPORTANCE_MEDIUM } )}, @@ -523,8 +528,19 @@ fields("mqtt") -> sc( number(), #{ - default => 0.75, - desc => ?DESC(mqtt_keepalive_backoff) + default => ?DEFAULT_BACKOFF, + %% Must add required => false, zone schema has no default. + required => false, + importance => ?IMPORTANCE_HIDDEN + } + )}, + {"keepalive_multiplier", + sc( + number(), + #{ + default => ?DEFAULT_MULTIPLIER, + validator => fun ?MODULE:validate_keepalive_multiplier/1, + desc => ?DESC(mqtt_keepalive_multiplier) } )}, {"max_subscriptions", @@ -593,7 +609,7 @@ fields("mqtt") -> )}, {"mqueue_priorities", sc( - hoconsc:union([map(), disabled]), + hoconsc:union([disabled, map()]), #{ default => disabled, desc => ?DESC(mqtt_mqueue_priorities) @@ -641,7 +657,7 @@ fields("mqtt") -> )} ]; fields("zone") -> - emqx_zone_schema:zone(); + emqx_zone_schema:zones_without_default(); fields("flapping_detect") -> [ {"enable", @@ -2291,6 +2307,17 @@ common_ssl_opts_schema(Defaults, Type) -> desc => ?DESC(common_ssl_opts_schema_secure_renegotiate) } )}, + {"log_level", + sc( + hoconsc:enum([ + emergency, alert, critical, error, warning, notice, info, debug, none, all + ]), + #{ + default => notice, + desc => ?DESC(common_ssl_opts_schema_log_level), + importance => ?IMPORTANCE_LOW + } + )}, {"hibernate_after", sc( @@ -2735,6 +2762,13 @@ validate_heap_size(Siz) when is_integer(Siz) -> validate_heap_size(_SizStr) -> {error, invalid_heap_size}. +validate_keepalive_multiplier(Multiplier) when + is_number(Multiplier) andalso Multiplier >= 1.0 andalso Multiplier =< 65535.0 +-> + ok; +validate_keepalive_multiplier(_Multiplier) -> + {error, #{reason => keepalive_multiplier_out_of_range, min => 1, max => 65535}}. + validate_alarm_actions(Actions) -> UnSupported = lists:filter( fun(Action) -> Action =/= log andalso Action =/= publish end, Actions @@ -3381,3 +3415,20 @@ ensure_default_listener(Map, ListenerType) -> cert_file(_File, client) -> undefined; cert_file(File, server) -> iolist_to_binary(filename:join(["${EMQX_ETC_DIR}", "certs", File])). + +mqtt_converter(#{<<"keepalive_multiplier">> := Multi} = Mqtt, _Opts) -> + case round(Multi * 100) =:= round(?DEFAULT_MULTIPLIER * 100) of + false -> + %% Multiplier is provided, and it's not default value + Mqtt; + true -> + %% Multiplier is default value, fallback to use Backoff value + %% Backoff default value was half of Multiplier default value + %% so there is no need to compare Backoff with its default. + Backoff = maps:get(<<"keepalive_backoff">>, Mqtt, ?DEFAULT_BACKOFF), + Mqtt#{<<"keepalive_multiplier">> => Backoff * 2} + end; +mqtt_converter(#{<<"keepalive_backoff">> := Backoff} = Mqtt, _Opts) -> + Mqtt#{<<"keepalive_multiplier">> => Backoff * 2}; +mqtt_converter(Mqtt, _Opts) -> + Mqtt. diff --git a/apps/emqx/src/emqx_secret.erl b/apps/emqx/src/emqx_secret.erl index 5340f36ba..72c4f3c08 100644 --- a/apps/emqx/src/emqx_secret.erl +++ b/apps/emqx/src/emqx_secret.erl @@ -21,6 +21,10 @@ %% API: -export([wrap/1, unwrap/1]). +-export_type([t/1]). + +-opaque t(T) :: T | fun(() -> t(T)). + %%================================================================================ %% API funcions %%================================================================================ diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index 8b15340e9..25bee629e 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -291,16 +291,16 @@ stats(Session) -> info(?STATS_KEYS, Session). ignore_local(ClientInfo, Delivers, Subscriber, Session) -> Subs = info(subscriptions, Session), - lists:dropwhile( + lists:filter( fun({deliver, Topic, #message{from = Publisher} = Msg}) -> case maps:find(Topic, Subs) of {ok, #{nl := 1}} when Subscriber =:= Publisher -> ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, no_local]), ok = emqx_metrics:inc('delivery.dropped'), ok = emqx_metrics:inc('delivery.dropped.no_local'), - true; + false; _ -> - false + true end end, Delivers diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 997364898..bc13a06c6 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -158,9 +158,18 @@ dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) -> -spec strategy(emqx_topic:group()) -> strategy(). strategy(Group) -> - case emqx:get_config([broker, shared_subscription_group, Group, strategy], undefined) of - undefined -> emqx:get_config([broker, shared_subscription_strategy]); - Strategy -> Strategy + try + emqx:get_config([ + broker, + shared_subscription_group, + binary_to_existing_atom(Group), + strategy + ]) + catch + error:{config_not_found, _} -> + get_default_shared_subscription_strategy(); + error:badarg -> + get_default_shared_subscription_strategy() end. -spec ack_enabled() -> boolean(). @@ -544,3 +553,6 @@ delete_route_if_needed({Group, Topic} = GroupTopic) -> if_no_more_subscribers(GroupTopic, fun() -> ok = emqx_router:do_delete_route(Topic, {Group, node()}) end). + +get_default_shared_subscription_strategy() -> + emqx:get_config([broker, shared_subscription_strategy]). diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index 91194772f..9ce6f9d38 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -144,7 +144,7 @@ list() -> list(Enable) -> ets:match_object(?TRACE, #?TRACE{enable = Enable, _ = '_'}). --spec create([{Key :: binary(), Value :: binary()}] | #{atom() => binary()}) -> +-spec create([{Key :: binary(), Value :: any()}] | #{atom() => any()}) -> {ok, #?TRACE{}} | {error, {duplicate_condition, iodata()} diff --git a/apps/emqx/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl index 75bba8d59..a7ec0cec4 100644 --- a/apps/emqx/src/emqx_types.erl +++ b/apps/emqx/src/emqx_types.erl @@ -131,7 +131,7 @@ socktype := socktype(), sockname := peername(), peername := peername(), - peercert := nossl | undefined | esockd_peercert:peercert(), + peercert => nossl | undefined | esockd_peercert:peercert(), conn_mod := module(), proto_name => binary(), proto_ver => proto_ver(), diff --git a/apps/emqx/src/emqx_zone_schema.erl b/apps/emqx/src/emqx_zone_schema.erl index 5d6720986..ccef4e54b 100644 --- a/apps/emqx/src/emqx_zone_schema.erl +++ b/apps/emqx/src/emqx_zone_schema.erl @@ -18,7 +18,8 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). --export([namespace/0, roots/0, fields/1, desc/1, zone/0, zone_without_hidden/0]). +-export([namespace/0, roots/0, fields/1, desc/1]). +-export([zones_without_default/0, global_zone_with_default/0]). namespace() -> zone. @@ -35,7 +36,7 @@ roots() -> "overload_protection" ]. -zone() -> +zones_without_default() -> Fields = roots(), Hidden = hidden(), lists:map( @@ -50,8 +51,8 @@ zone() -> Fields ). -zone_without_hidden() -> - lists:map(fun(F) -> {F, ?HOCON(?R_REF(F), #{})} end, roots() -- hidden()). +global_zone_with_default() -> + lists:map(fun(F) -> {F, ?HOCON(?R_REF(emqx_schema, F), #{})} end, roots() -- hidden()). hidden() -> [ @@ -69,9 +70,10 @@ fields(Name) -> desc(Name) -> emqx_schema:desc(Name). -%% no default values for zone settings +%% no default values for zone settings, don't required either. no_default(Sc) -> fun (default) -> undefined; + (required) -> false; (Other) -> hocon_schema:field_schema(Sc, Other) end. diff --git a/apps/emqx/test/emqx_SUITE.erl b/apps/emqx/test/emqx_SUITE.erl index 64ed2ea19..cfabff401 100644 --- a/apps/emqx/test/emqx_SUITE.erl +++ b/apps/emqx/test/emqx_SUITE.erl @@ -156,6 +156,19 @@ t_cluster_nodes(_) -> ?assertEqual(Expected, emqx:cluster_nodes(cores)), ?assertEqual([], emqx:cluster_nodes(stopped)). +t_get_config(_) -> + ?assertEqual(false, emqx:get_config([overload_protection, enable])), + ?assertEqual(false, emqx:get_config(["overload_protection", <<"enable">>])). + +t_get_config_default_1(_) -> + ?assertEqual(false, emqx:get_config([overload_protection, enable], undefined)), + ?assertEqual(false, emqx:get_config(["overload_protection", <<"enable">>], undefined)). + +t_get_config_default_2(_) -> + AtomPathRes = emqx:get_config([overload_protection, <<"_!no_@exist_">>], undefined), + NonAtomPathRes = emqx:get_config(["doesnotexist", <<"db_backend">>], undefined), + ?assertEqual(undefined, NonAtomPathRes), + ?assertEqual(undefined, AtomPathRes). %%-------------------------------------------------------------------- %% Hook fun %%-------------------------------------------------------------------- diff --git a/apps/emqx/test/emqx_access_control_SUITE.erl b/apps/emqx/test/emqx_access_control_SUITE.erl index 4bf1e05fd..f5fdf223b 100644 --- a/apps/emqx/test/emqx_access_control_SUITE.erl +++ b/apps/emqx/test/emqx_access_control_SUITE.erl @@ -116,7 +116,6 @@ clientinfo(InitProps) -> username => <<"username">>, password => <<"passwd">>, is_superuser => false, - peercert => undefined, mountpoint => undefined }, InitProps diff --git a/apps/emqx/test/emqx_bpapi_static_checks.erl b/apps/emqx/test/emqx_bpapi_static_checks.erl index 34ff149c1..c944293d2 100644 --- a/apps/emqx/test/emqx_bpapi_static_checks.erl +++ b/apps/emqx/test/emqx_bpapi_static_checks.erl @@ -51,6 +51,8 @@ "gen_rpc, recon, redbug, observer_cli, snabbkaffe, ekka, mria, amqp_client, rabbit_common" ). -define(IGNORED_MODULES, "emqx_rpc"). +-define(FORCE_DELETED_MODULES, [emqx_statsd, emqx_statsd_proto_v1]). +-define(FORCE_DELETED_APIS, [{emqx_statsd, 1}]). %% List of known RPC backend modules: -define(RPC_MODULES, "gen_rpc, erpc, rpc, emqx_rpc"). %% List of known functions also known to do RPC: @@ -127,11 +129,16 @@ check_api_immutability(#{release := Rel1, api := APIs1}, #{release := Rel2, api Val -> ok; undefined -> - setnok(), - logger:error( - "API ~p v~p was removed in release ~p without being deprecated.", - [API, Version, Rel2] - ); + case lists:member({API, Version}, ?FORCE_DELETED_APIS) of + true -> + ok; + false -> + setnok(), + logger:error( + "API ~p v~p was removed in release ~p without being deprecated.", + [API, Version, Rel2] + ) + end; _Val -> setnok(), logger:error( @@ -146,16 +153,24 @@ check_api_immutability(#{release := Rel1, api := APIs1}, #{release := Rel2, api check_api_immutability(_, _) -> ok. +filter_calls(Calls) -> + F = fun({{Mf, _, _}, {Mt, _, _}}) -> + (not lists:member(Mf, ?FORCE_DELETED_MODULES)) andalso + (not lists:member(Mt, ?FORCE_DELETED_MODULES)) + end, + lists:filter(F, Calls). + %% Note: sets nok flag -spec typecheck_apis(fulldump(), fulldump()) -> ok. typecheck_apis( #{release := CallerRelease, api := CallerAPIs, signatures := CallerSigs}, #{release := CalleeRelease, signatures := CalleeSigs} ) -> - AllCalls = lists:flatten([ + AllCalls0 = lists:flatten([ [Calls, Casts] || #{calls := Calls, casts := Casts} <- maps:values(CallerAPIs) ]), + AllCalls = filter_calls(AllCalls0), lists:foreach( fun({From, To}) -> Caller = get_param_types(CallerSigs, From), @@ -213,7 +228,7 @@ get_param_types(Signatures, {M, F, A}) -> maps:from_list(lists:zip(A, AttrTypes)); _ -> logger:critical("Call ~p:~p/~p is not found in PLT~n", [M, F, Arity]), - error(badkey) + error({badkey, {M, F, A}}) end. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 0b88ff045..6f14293ed 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -1211,7 +1211,6 @@ clientinfo(InitProps) -> clientid => <<"clientid">>, username => <<"username">>, is_superuser => false, - peercert => undefined, mountpoint => undefined }, InitProps diff --git a/apps/emqx/test/emqx_client_SUITE.erl b/apps/emqx/test/emqx_client_SUITE.erl index ca5f53070..14617a152 100644 --- a/apps/emqx/test/emqx_client_SUITE.erl +++ b/apps/emqx/test/emqx_client_SUITE.erl @@ -22,6 +22,8 @@ -import(lists, [nth/2]). -include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_hooks.hrl"). +-include_lib("emqx/include/asserts.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -75,7 +77,8 @@ groups() -> t_username_as_clientid, t_certcn_as_clientid_default_config_tls, t_certcn_as_clientid_tlsv1_3, - t_certcn_as_clientid_tlsv1_2 + t_certcn_as_clientid_tlsv1_2, + t_peercert_preserved_before_connected ]} ]. @@ -379,6 +382,42 @@ t_certcn_as_clientid_tlsv1_3(_) -> t_certcn_as_clientid_tlsv1_2(_) -> tls_certcn_as_clientid('tlsv1.2'). +t_peercert_preserved_before_connected(_) -> + ok = emqx_config:put_zone_conf(default, [mqtt], #{}), + ok = emqx_hooks:add( + 'client.connect', + {?MODULE, on_hook, ['client.connect', self()]}, + ?HP_HIGHEST + ), + ok = emqx_hooks:add( + 'client.connected', + {?MODULE, on_hook, ['client.connected', self()]}, + ?HP_HIGHEST + ), + ClientId = atom_to_binary(?FUNCTION_NAME), + SslConf = emqx_common_test_helpers:client_ssl_twoway(default), + {ok, Client} = emqtt:start_link([ + {port, 8883}, + {clientid, ClientId}, + {ssl, true}, + {ssl_opts, SslConf} + ]), + {ok, _} = emqtt:connect(Client), + _ = ?assertReceive({'client.connect', #{peercert := PC}} when is_binary(PC)), + _ = ?assertReceive({'client.connected', #{peercert := PC}} when is_binary(PC)), + [ConnPid] = emqx_cm:lookup_channels(ClientId), + ?assertMatch( + #{conninfo := ConnInfo} when not is_map_key(peercert, ConnInfo), + emqx_connection:info(ConnPid) + ). + +on_hook(ConnInfo, _, 'client.connect' = HP, Pid) -> + _ = Pid ! {HP, ConnInfo}, + ok; +on_hook(_ClientInfo, ConnInfo, 'client.connected' = HP, Pid) -> + _ = Pid ! {HP, ConnInfo}, + ok. + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- @@ -421,10 +460,4 @@ tls_certcn_as_clientid(TLSVsn, RequiredTLSVsn) -> {ok, _} = emqtt:connect(Client), #{clientinfo := #{clientid := CN}} = emqx_cm:get_chan_info(CN), confirm_tls_version(Client, RequiredTLSVsn), - %% verify that the peercert won't be stored in the conninfo - [ChannPid] = emqx_cm:lookup_channels(CN), - SysState = sys:get_state(ChannPid), - ChannelRecord = lists:keyfind(channel, 1, tuple_to_list(SysState)), - ConnInfo = lists:nth(2, tuple_to_list(ChannelRecord)), - ?assertMatch(#{peercert := undefined}, ConnInfo), emqtt:disconnect(Client). diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 15869fb36..bfa0e36cb 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -232,11 +232,12 @@ render_and_load_app_config(App, Opts) -> try do_render_app_config(App, Schema, Conf, Opts) catch + throw:skip -> + ok; throw:E:St -> %% turn throw into error error({Conf, E, St}) end. - do_render_app_config(App, Schema, ConfigFile, Opts) -> %% copy acl_conf must run before read_schema_configs copy_acl_conf(), @@ -257,6 +258,7 @@ start_app(App, SpecAppConfig, Opts) -> {ok, _} -> ok = ensure_dashboard_listeners_started(App), ok = wait_for_app_processes(App), + ok = perform_sanity_checks(App), ok; {error, Reason} -> error({failed_to_start_app, App, Reason}) @@ -270,6 +272,27 @@ wait_for_app_processes(emqx_conf) -> wait_for_app_processes(_) -> ok. +%% These are checks to detect inter-suite or inter-testcase flakiness +%% early. For example, one suite might forget one application running +%% and stop others, and then the `application:start/2' callback is +%% never called again for this application. +perform_sanity_checks(emqx_rule_engine) -> + ensure_config_handler(emqx_rule_engine, [rule_engine, rules]), + ok; +perform_sanity_checks(emqx_bridge) -> + ensure_config_handler(emqx_bridge, [bridges]), + ok; +perform_sanity_checks(_App) -> + ok. + +ensure_config_handler(Module, ConfigPath) -> + #{handlers := Handlers} = sys:get_state(emqx_config_handler), + case emqx_utils_maps:deep_get(ConfigPath, Handlers, not_found) of + #{{mod} := Module} -> ok; + _NotFound -> error({config_handler_missing, ConfigPath, Module}) + end, + ok. + app_conf_file(emqx_conf) -> "emqx.conf.all"; app_conf_file(App) -> atom_to_list(App) ++ ".conf". @@ -296,6 +319,7 @@ render_config_file(ConfigFile, Vars0) -> Temp = case file:read_file(ConfigFile) of {ok, T} -> T; + {error, enoent} -> throw(skip); {error, Reason} -> error({failed_to_read_config_template, ConfigFile, Reason}) end, Vars = [{atom_to_list(N), iolist_to_binary(V)} || {N, V} <- maps:to_list(Vars0)], @@ -842,8 +866,8 @@ setup_node(Node, Opts) when is_map(Opts) -> LoadSchema andalso begin %% to avoid sharing data between executions and/or - %% nodes. these variables might notbe in the - %% config file (e.g.: emqx_ee_conf_schema). + %% nodes. these variables might not be in the + %% config file (e.g.: emqx_enterprise_schema). NodeDataDir = filename:join([ PrivDataDir, node(), diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index 0692ec8f5..de3672bf3 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -676,7 +676,6 @@ channel(InitFields) -> clientid => <<"clientid">>, username => <<"username">>, is_superuser => false, - peercert => undefined, mountpoint => undefined }, Conf = emqx_cm:get_session_confs(ClientInfo, #{ diff --git a/apps/emqx/test/emqx_keepalive_SUITE.erl b/apps/emqx/test/emqx_keepalive_SUITE.erl index dce55409e..480beeaa4 100644 --- a/apps/emqx/test/emqx_keepalive_SUITE.erl +++ b/apps/emqx/test/emqx_keepalive_SUITE.erl @@ -27,20 +27,14 @@ t_check(_) -> Keepalive = emqx_keepalive:init(60), ?assertEqual(60, emqx_keepalive:info(interval, Keepalive)), ?assertEqual(0, emqx_keepalive:info(statval, Keepalive)), - ?assertEqual(0, emqx_keepalive:info(repeat, Keepalive)), Info = emqx_keepalive:info(Keepalive), ?assertEqual( #{ interval => 60, - statval => 0, - repeat => 0 + statval => 0 }, Info ), {ok, Keepalive1} = emqx_keepalive:check(1, Keepalive), ?assertEqual(1, emqx_keepalive:info(statval, Keepalive1)), - ?assertEqual(0, emqx_keepalive:info(repeat, Keepalive1)), - {ok, Keepalive2} = emqx_keepalive:check(1, Keepalive1), - ?assertEqual(1, emqx_keepalive:info(statval, Keepalive2)), - ?assertEqual(1, emqx_keepalive:info(repeat, Keepalive2)), - ?assertEqual({error, timeout}, emqx_keepalive:check(1, Keepalive2)). + ?assertEqual({error, timeout}, emqx_keepalive:check(1, Keepalive1)). diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl index d3de74f72..fe608f600 100644 --- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl @@ -829,6 +829,42 @@ t_subscribe_no_local(Config) -> ?assertEqual(1, length(receive_messages(2))), ok = emqtt:disconnect(Client1). +t_subscribe_no_local_mixed(Config) -> + ConnFun = ?config(conn_fun, Config), + Topic = nth(1, ?TOPICS), + {ok, Client1} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client1), + + {ok, Client2} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client2), + + %% Given tow clients and client1 subscribe to topic with 'no local' set to true + {ok, _, [2]} = emqtt:subscribe(Client1, #{}, [{Topic, [{nl, true}, {qos, 2}]}]), + + %% When mixed publish traffic are sent from both clients (Client1 sent 6 and Client2 sent 2) + CB = {fun emqtt:sync_publish_result/3, [self(), async_res]}, + ok = emqtt:publish_async(Client1, Topic, <<"t_subscribe_no_local_mixed1">>, 0, CB), + ok = emqtt:publish_async(Client2, Topic, <<"t_subscribe_no_local_mixed2">>, 0, CB), + ok = emqtt:publish_async(Client1, Topic, <<"t_subscribe_no_local_mixed3">>, 0, CB), + ok = emqtt:publish_async(Client1, Topic, <<"t_subscribe_no_local_mixed4">>, 0, CB), + ok = emqtt:publish_async(Client1, Topic, <<"t_subscribe_no_local_mixed5">>, 0, CB), + ok = emqtt:publish_async(Client2, Topic, <<"t_subscribe_no_local_mixed6">>, 0, CB), + ok = emqtt:publish_async(Client1, Topic, <<"t_subscribe_no_local_mixed7">>, 0, CB), + ok = emqtt:publish_async(Client1, Topic, <<"t_subscribe_no_local_mixed8">>, 0, CB), + [ + receive + {async_res, Res} -> ?assertEqual(ok, Res) + end + || _ <- lists:seq(1, 8) + ], + + %% Then only two messages from clients 2 are received + PubRecvd = receive_messages(9), + ct:pal("~p", [PubRecvd]), + ?assertEqual(2, length(PubRecvd)), + ok = emqtt:disconnect(Client1), + ok = emqtt:disconnect(Client2). + t_subscribe_actions(Config) -> ConnFun = ?config(conn_fun, Config), Topic = nth(1, ?TOPICS), diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl index b0ba4f0e2..8bf965cc3 100644 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -165,6 +165,7 @@ init_per_testcase(_TestCase, Config) -> {ok, {{"HTTP/1.0", 200, 'OK'}, [], <<"ocsp response">>}} end ), + snabbkaffe:start_trace(), _Heir = spawn_dummy_heir(), {ok, CachePid} = emqx_ocsp_cache:start_link(), DataDir = ?config(data_dir, Config), @@ -187,7 +188,6 @@ init_per_testcase(_TestCase, Config) -> ConfBin = emqx_utils_maps:binary_key_map(Conf), hocon_tconf:check_plain(emqx_schema, ConfBin, #{required => false, atom_keys => false}), emqx_config:put_listener_conf(Type, Name, [], ListenerOpts), - snabbkaffe:start_trace(), [ {cache_pid, CachePid} | Config @@ -231,12 +231,19 @@ end_per_testcase(_TestCase, Config) -> %% In some tests, we don't start the full supervision tree, so we need %% this dummy process. spawn_dummy_heir() -> - spawn_link(fun() -> - true = register(emqx_kernel_sup, self()), - receive - stop -> ok - end - end). + {_, {ok, _}} = + ?wait_async_action( + spawn_link(fun() -> + true = register(emqx_kernel_sup, self()), + ?tp(heir_name_registered, #{}), + receive + stop -> ok + end + end), + #{?snk_kind := heir_name_registered}, + 1_000 + ), + ok. does_module_exist(Mod) -> case erlang:module_loaded(Mod) of diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl index 81991f26e..3dcfa331e 100644 --- a/apps/emqx/test/emqx_schema_tests.erl +++ b/apps/emqx/test/emqx_schema_tests.erl @@ -655,6 +655,43 @@ password_converter_test() -> ?assertThrow("must_quote", emqx_schema:password_converter(foobar, #{})), ok. +-define(MQTT(B, M), #{<<"keepalive_backoff">> => B, <<"keepalive_multiplier">> => M}). + +keepalive_convert_test() -> + ?assertEqual(undefined, emqx_schema:mqtt_converter(undefined, #{})), + DefaultBackoff = 0.75, + DefaultMultiplier = 1.5, + Default = ?MQTT(DefaultBackoff, DefaultMultiplier), + ?assertEqual(Default, emqx_schema:mqtt_converter(Default, #{})), + ?assertEqual(?MQTT(1.5, 3), emqx_schema:mqtt_converter(?MQTT(1.5, 3), #{})), + ?assertEqual( + ?MQTT(DefaultBackoff, 3), emqx_schema:mqtt_converter(?MQTT(DefaultBackoff, 3), #{}) + ), + ?assertEqual(?MQTT(1, 2), emqx_schema:mqtt_converter(?MQTT(1, DefaultMultiplier), #{})), + ?assertEqual(?MQTT(1.5, 3), emqx_schema:mqtt_converter(?MQTT(1.5, 3), #{})), + + ?assertEqual(#{}, emqx_schema:mqtt_converter(#{}, #{})), + ?assertEqual( + #{<<"keepalive_backoff">> => 1.5, <<"keepalive_multiplier">> => 3.0}, + emqx_schema:mqtt_converter(#{<<"keepalive_backoff">> => 1.5}, #{}) + ), + ?assertEqual( + #{<<"keepalive_multiplier">> => 5.0}, + emqx_schema:mqtt_converter(#{<<"keepalive_multiplier">> => 5.0}, #{}) + ), + ?assertEqual( + #{ + <<"keepalive_backoff">> => DefaultBackoff, + <<"keepalive_multiplier">> => DefaultMultiplier + }, + emqx_schema:mqtt_converter(#{<<"keepalive_backoff">> => DefaultBackoff}, #{}) + ), + ?assertEqual( + #{<<"keepalive_multiplier">> => DefaultMultiplier}, + emqx_schema:mqtt_converter(#{<<"keepalive_multiplier">> => DefaultMultiplier}, #{}) + ), + ok. + url_type_test_() -> [ ?_assertEqual( diff --git a/apps/emqx/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl index 60abe3d3c..bc91bc0ef 100644 --- a/apps/emqx/test/emqx_ws_connection_SUITE.erl +++ b/apps/emqx/test/emqx_ws_connection_SUITE.erl @@ -33,17 +33,6 @@ ] ). --define(STATS_KEYS, [ - recv_oct, - recv_cnt, - send_oct, - send_cnt, - recv_pkt, - recv_msg, - send_pkt, - send_msg -]). - -define(ws_conn, emqx_ws_connection). all() -> emqx_common_test_helpers:all(?MODULE). @@ -618,7 +607,6 @@ channel(InitFields) -> clientid => <<"clientid">>, username => <<"username">>, is_superuser => false, - peercert => undefined, mountpoint => undefined }, Conf = emqx_cm:get_session_confs(ClientInfo, #{ diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index 3e0cf786e..a291f2f61 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authn, [ {description, "EMQX Authentication"}, - {vsn, "0.1.19"}, + {vsn, "0.1.20"}, {modules, []}, {registered, [emqx_authn_sup, emqx_authn_registry]}, {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]}, diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index f46718842..f00ca8ed1 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -805,7 +805,11 @@ with_listener(ListenerID, Fun) -> find_listener(ListenerID) -> case binary:split(ListenerID, <<":">>) of [BType, BName] -> - case emqx_config:find([listeners, BType, BName]) of + case + emqx_config:find([ + listeners, binary_to_existing_atom(BType), binary_to_existing_atom(BName) + ]) + of {ok, _} -> {ok, {BType, BName}}; {not_found, _, _} -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index eddad92a3..421af074e 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -100,7 +100,6 @@ common_fields() -> maps:to_list( maps:without( [ - base_url, pool_type ], maps:from_list(emqx_connector_http:fields(config)) diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index 4856008e6..7620f5548 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.20"}, + {vsn, "0.1.21"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl index d6e5a3eb2..049c84713 100644 --- a/apps/emqx_authz/src/emqx_authz_api_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl @@ -116,7 +116,6 @@ authz_http_common_fields() -> maps:to_list( maps:without( [ - base_url, pool_type ], maps:from_list(emqx_connector_http:fields(config)) diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 3b318c39f..8e847b93e 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -240,7 +240,6 @@ http_common_fields() -> maps:to_list( maps:without( [ - base_url, pool_type ], maps:from_list(connector_fields(http)) diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index d2bf0f0c2..125f53c81 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.19"}, + {vsn, "0.1.20"}, {registered, [emqx_bridge_sup]}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 29fac2416..bffa7b7f9 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -687,11 +687,15 @@ get_metrics_from_local_node(BridgeType, BridgeName) -> ). is_enabled_bridge(BridgeType, BridgeName) -> - try emqx:get_config([bridges, BridgeType, BridgeName]) of + try emqx:get_config([bridges, BridgeType, binary_to_existing_atom(BridgeName)]) of ConfMap -> maps:get(enable, ConfMap, false) catch error:{config_not_found, _} -> + throw(not_found); + error:badarg -> + %% catch non-existing atom, + %% none-existing atom means it is not available in config PT storage. throw(not_found) end. diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl index 4e1492a9c..5c2c7b461 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl @@ -68,7 +68,7 @@ basic_config() -> )} ] ++ webhook_creation_opts() ++ proplists:delete( - max_retries, proplists:delete(base_url, emqx_connector_http:fields(config)) + max_retries, emqx_connector_http:fields(config) ). request_config() -> diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index a902783d7..1ac6750a4 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -160,6 +160,7 @@ init_node(Type) -> ok = emqx_common_test_helpers:start_apps(?SUITE_APPS, fun load_suite_config/1), case Type of primary -> + ok = emqx_dashboard_desc_cache:init(), ok = emqx_config:put( [dashboard, listeners], #{http => #{enable => true, bind => 18083, proxy_header => false}} diff --git a/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl b/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl index af9daef0a..7c8ee4f4d 100644 --- a/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl +++ b/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl @@ -121,7 +121,7 @@ assert_upgraded1(Map) -> ?assert(maps:is_key(<<"ssl">>, Map)). check(Conf) when is_map(Conf) -> - hocon_tconf:check_plain(emqx_bridge_schema, Conf). + hocon_tconf:check_plain(emqx_bridge_schema, Conf, #{required => false}). %% erlfmt-ignore %% this is config generated from v5.0.11 diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl index bd5cda3f0..f0de07da2 100644 --- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl @@ -100,17 +100,21 @@ ?assertMetrics(Pat, true, BridgeID) ). -define(assertMetrics(Pat, Guard, BridgeID), - ?assertMatch( - #{ - <<"metrics">> := Pat, - <<"node_metrics">> := [ - #{ - <<"node">> := _, - <<"metrics">> := Pat - } - ] - } when Guard, - request_bridge_metrics(BridgeID) + ?retry( + _Sleep = 300, + _Attempts0 = 20, + ?assertMatch( + #{ + <<"metrics">> := Pat, + <<"node_metrics">> := [ + #{ + <<"node">> := _, + <<"metrics">> := Pat + } + ] + } when Guard, + request_bridge_metrics(BridgeID) + ) ) ). diff --git a/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl index f08c87b6e..45cc82251 100644 --- a/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl @@ -23,6 +23,7 @@ -compile(export_all). -import(emqx_mgmt_api_test_util, [request/3, uri/1]). +-import(emqx_common_test_helpers, [on_exit/1]). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -52,6 +53,13 @@ end_per_suite(_Config) -> suite() -> [{timetrap, {seconds, 60}}]. +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + emqx_common_test_helpers:call_janitor(), + ok. + %%------------------------------------------------------------------------------ %% HTTP server for testing %% (Orginally copied from emqx_bridge_api_SUITE) @@ -158,7 +166,8 @@ bridge_async_config(#{port := Port} = Config) -> QueryMode = maps:get(query_mode, Config, "async"), ConnectTimeout = maps:get(connect_timeout, Config, 1), RequestTimeout = maps:get(request_timeout, Config, 10000), - ResourceRequestTimeout = maps:get(resouce_request_timeout, Config, "infinity"), + ResumeInterval = maps:get(resume_interval, Config, "1s"), + ResourceRequestTimeout = maps:get(resource_request_timeout, Config, "infinity"), ConfigString = io_lib:format( "bridges.~s.~s {\n" " url = \"http://localhost:~p\"\n" @@ -177,7 +186,8 @@ bridge_async_config(#{port := Port} = Config) -> " health_check_interval = \"15s\"\n" " max_buffer_bytes = \"1GB\"\n" " query_mode = \"~s\"\n" - " request_timeout = \"~s\"\n" + " request_timeout = \"~p\"\n" + " resume_interval = \"~s\"\n" " start_after_created = \"true\"\n" " start_timeout = \"5s\"\n" " worker_pool_size = \"1\"\n" @@ -194,7 +204,8 @@ bridge_async_config(#{port := Port} = Config) -> PoolSize, RequestTimeout, QueryMode, - ResourceRequestTimeout + ResourceRequestTimeout, + ResumeInterval ] ), ct:pal(ConfigString), @@ -236,7 +247,7 @@ t_send_async_connection_timeout(_Config) -> query_mode => "async", connect_timeout => ResponseDelayMS * 2, request_timeout => 10000, - resouce_request_timeout => "infinity" + resource_request_timeout => "infinity" }), NumberOfMessagesToSend = 10, [ @@ -250,6 +261,97 @@ t_send_async_connection_timeout(_Config) -> stop_http_server(Server), ok. +t_async_free_retries(_Config) -> + #{port := Port} = start_http_server(#{response_delay_ms => 0}), + BridgeID = make_bridge(#{ + port => Port, + pool_size => 1, + query_mode => "sync", + connect_timeout => 1_000, + request_timeout => 10_000, + resource_request_timeout => "10000s" + }), + %% Fail 5 times then succeed. + Context = #{error_attempts => 5}, + ExpectedAttempts = 6, + Fn = fun(Get, Error) -> + ?assertMatch( + {ok, 200, _, _}, + emqx_bridge:send_message(BridgeID, #{<<"hello">> => <<"world">>}), + #{error => Error} + ), + ?assertEqual(ExpectedAttempts, Get(), #{error => Error}) + end, + do_t_async_retries(Context, {error, normal}, Fn), + do_t_async_retries(Context, {error, {shutdown, normal}}, Fn), + ok. + +t_async_common_retries(_Config) -> + #{port := Port} = start_http_server(#{response_delay_ms => 0}), + BridgeID = make_bridge(#{ + port => Port, + pool_size => 1, + query_mode => "sync", + resume_interval => "100ms", + connect_timeout => 1_000, + request_timeout => 10_000, + resource_request_timeout => "10000s" + }), + %% Keeps failing until connector gives up. + Context = #{error_attempts => infinity}, + ExpectedAttempts = 3, + FnSucceed = fun(Get, Error) -> + ?assertMatch( + {ok, 200, _, _}, + emqx_bridge:send_message(BridgeID, #{<<"hello">> => <<"world">>}), + #{error => Error, attempts => Get()} + ), + ?assertEqual(ExpectedAttempts, Get(), #{error => Error}) + end, + FnFail = fun(Get, Error) -> + ?assertMatch( + Error, + emqx_bridge:send_message(BridgeID, #{<<"hello">> => <<"world">>}), + #{error => Error, attempts => Get()} + ), + ?assertEqual(ExpectedAttempts, Get(), #{error => Error}) + end, + %% These two succeed because they're further retried by the buffer + %% worker synchronously, and we're not mock that call. + do_t_async_retries(Context, {error, {closed, "The connection was lost."}}, FnSucceed), + do_t_async_retries(Context, {error, {shutdown, closed}}, FnSucceed), + %% This fails because this error is treated as unrecoverable. + do_t_async_retries(Context, {error, something_else}, FnFail), + ok. + +do_t_async_retries(TestContext, Error, Fn) -> + #{error_attempts := ErrorAttempts} = TestContext, + persistent_term:put({?MODULE, ?FUNCTION_NAME, attempts}, 0), + on_exit(fun() -> persistent_term:erase({?MODULE, ?FUNCTION_NAME, attempts}) end), + Get = fun() -> persistent_term:get({?MODULE, ?FUNCTION_NAME, attempts}) end, + GetAndBump = fun() -> + Attempts = persistent_term:get({?MODULE, ?FUNCTION_NAME, attempts}), + persistent_term:put({?MODULE, ?FUNCTION_NAME, attempts}, Attempts + 1), + Attempts + 1 + end, + emqx_common_test_helpers:with_mock( + emqx_connector_http, + reply_delegator, + fun(Context, ReplyFunAndArgs, Result) -> + Attempts = GetAndBump(), + case Attempts > ErrorAttempts of + true -> + ct:pal("succeeding ~p : ~p", [Error, Attempts]), + meck:passthrough([Context, ReplyFunAndArgs, Result]); + false -> + ct:pal("failing ~p : ~p", [Error, Attempts]), + meck:passthrough([Context, ReplyFunAndArgs, Error]) + end + end, + fun() -> Fn(Get, Error) end + ), + ok. + receive_request_notifications(MessageIDs, _ResponseDelay) when map_size(MessageIDs) =:= 0 -> ok; receive_request_notifications(MessageIDs, ResponseDelay) -> diff --git a/apps/emqx_bridge_cassandra/README.md b/apps/emqx_bridge_cassandra/README.md index c5a2609a5..d26bd2fbb 100644 --- a/apps/emqx_bridge_cassandra/README.md +++ b/apps/emqx_bridge_cassandra/README.md @@ -11,7 +11,6 @@ The application is used to connect EMQX and Cassandra. User can create a rule and easily ingest IoT data into Cassandra by leveraging [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). - # HTTP APIs diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl index 79220321e..8f093ef5c 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl @@ -506,7 +506,17 @@ t_write_failure(Config) -> ProxyPort = ?config(proxy_port, Config), ProxyHost = ?config(proxy_host, Config), QueryMode = ?config(query_mode, Config), - {ok, _} = create_bridge(Config), + {ok, _} = create_bridge( + Config, + #{ + <<"resource_opts">> => + #{ + <<"auto_restart_interval">> => <<"100ms">>, + <<"resume_interval">> => <<"100ms">>, + <<"health_check_interval">> => <<"100ms">> + } + } + ), Val = integer_to_binary(erlang:unique_integer()), SentData = #{ topic => atom_to_binary(?FUNCTION_NAME), @@ -523,7 +533,9 @@ t_write_failure(Config) -> async -> send_message(Config, SentData) end, - #{?snk_kind := buffer_worker_flush_nack}, + #{?snk_kind := Evt} when + Evt =:= buffer_worker_flush_nack orelse + Evt =:= buffer_worker_retry_inflight_failed, 10_000 ) end), diff --git a/apps/emqx_bridge_clickhouse/README.md b/apps/emqx_bridge_clickhouse/README.md index ff870e87d..e74afab3d 100644 --- a/apps/emqx_bridge_clickhouse/README.md +++ b/apps/emqx_bridge_clickhouse/README.md @@ -23,7 +23,7 @@ User can create a rule and easily ingest IoT data into ClickHouse by leveraging - Several APIs are provided for bridge management, which includes create bridge, update bridge, get bridge, stop or restart bridge and list bridges etc. - Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) +- Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) for more detailed information. diff --git a/apps/emqx_bridge_clickhouse/docker-ct b/apps/emqx_bridge_clickhouse/docker-ct new file mode 100644 index 000000000..938ba95c6 --- /dev/null +++ b/apps/emqx_bridge_clickhouse/docker-ct @@ -0,0 +1 @@ +clickhouse diff --git a/apps/emqx_statsd/etc/emqx_statsd.conf b/apps/emqx_bridge_clickhouse/etc/emqx_bridge_clickhouse.conf similarity index 100% rename from apps/emqx_statsd/etc/emqx_statsd.conf rename to apps/emqx_bridge_clickhouse/etc/emqx_bridge_clickhouse.conf diff --git a/apps/emqx_bridge_clickhouse/rebar.config b/apps/emqx_bridge_clickhouse/rebar.config new file mode 100644 index 000000000..a8da74b43 --- /dev/null +++ b/apps/emqx_bridge_clickhouse/rebar.config @@ -0,0 +1,11 @@ +%% -*- mode: erlang; -*- +{erl_opts, [debug_info]}. +{deps, [ {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}} + , {emqx_connector, {path, "../../apps/emqx_connector"}} + , {emqx_resource, {path, "../../apps/emqx_resource"}} + , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + ]}. + +{shell, [ + {apps, [emqx_bridge_clickhouse]} +]}. diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src index a0b409d5b..72669ba8f 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src @@ -1,8 +1,8 @@ {application, emqx_bridge_clickhouse, [ {description, "EMQX Enterprise ClickHouse Bridge"}, - {vsn, "0.1.0"}, + {vsn, "0.2.0"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, clickhouse, emqx_resource]}, {env, []}, {modules, []}, {links, []} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.erl similarity index 96% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl rename to apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.erl index 56671c586..9abcadbba 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.erl @@ -1,9 +1,8 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_clickhouse). +-module(emqx_bridge_clickhouse). --include_lib("emqx_bridge/include/emqx_bridge.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). @@ -101,7 +100,7 @@ fields("config") -> } )} ] ++ - emqx_ee_connector_clickhouse:fields(config); + emqx_bridge_clickhouse_connector:fields(config); fields("creation_opts") -> emqx_resource_schema:fields("creation_opts"); fields("post") -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl similarity index 99% rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl rename to apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl index a7afcd6d5..a2de1d3c9 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_clickhouse). +-module(emqx_bridge_clickhouse_connector). -include_lib("emqx_connector/include/emqx_connector.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_clickhouse_SUITE.erl b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl similarity index 94% rename from lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_clickhouse_SUITE.erl rename to apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl index 6d4762882..787fb81ff 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_clickhouse_SUITE.erl +++ b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl @@ -2,17 +2,17 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_clickhouse_SUITE). +-module(emqx_bridge_clickhouse_SUITE). -compile(nowarn_export_all). -compile(export_all). +-define(APP, emqx_bridge_clickhouse). -define(CLICKHOUSE_HOST, "clickhouse"). --define(CLICKHOUSE_RESOURCE_MOD, emqx_ee_connector_clickhouse). -include_lib("emqx_connector/include/emqx_connector.hrl"). %% See comment in -%% lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl for how to +%% lib-ee/emqx_ee_connector/test/ee_bridge_clickhouse_connector_SUITE.erl for how to %% run this without bringing up the whole CI infrastucture %%------------------------------------------------------------------------------ @@ -26,10 +26,7 @@ init_per_suite(Config) -> true -> emqx_common_test_helpers:render_and_load_app_config(emqx_conf), ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), - ok = emqx_connector_test_helpers:start_apps([emqx_resource]), - {ok, _} = application:ensure_all_started(emqx_connector), - {ok, _} = application:ensure_all_started(emqx_ee_connector), - {ok, _} = application:ensure_all_started(emqx_ee_bridge), + ok = emqx_connector_test_helpers:start_apps([emqx_resource, ?APP]), snabbkaffe:fix_ct_logging(), %% Create the db table Conn = start_clickhouse_connection(), @@ -76,11 +73,8 @@ start_clickhouse_connection() -> end_per_suite(Config) -> ClickhouseConnection = proplists:get_value(clickhouse_connection, Config), clickhouse:stop(ClickhouseConnection), - ok = emqx_common_test_helpers:stop_apps([emqx_conf]), - ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), - _ = application:stop(emqx_connector), - _ = application:stop(emqx_ee_connector), - _ = application:stop(emqx_bridge). + ok = emqx_connector_test_helpers:stop_apps([?APP, emqx_resource]), + ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]). init_per_testcase(_, Config) -> reset_table(Config), diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl similarity index 89% rename from lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl rename to apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl index e704a2c0c..0bae413e0 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl +++ b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl @@ -2,18 +2,18 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_clickhouse_SUITE). +-module(emqx_bridge_clickhouse_connector_SUITE). -compile(nowarn_export_all). -compile(export_all). -include("emqx_connector.hrl"). -include_lib("eunit/include/eunit.hrl"). --include_lib("emqx/include/emqx.hrl"). -include_lib("stdlib/include/assert.hrl"). +-define(APP, emqx_bridge_clickhouse). -define(CLICKHOUSE_HOST, "clickhouse"). --define(CLICKHOUSE_RESOURCE_MOD, emqx_ee_connector_clickhouse). +-define(CLICKHOUSE_RESOURCE_MOD, emqx_bridge_clickhouse_connector). %% This test SUITE requires a running clickhouse instance. If you don't want to %% bring up the whole CI infrastuctucture with the `scripts/ct/run.sh` script @@ -21,7 +21,15 @@ %% from root of the EMQX directory.). You also need to set ?CLICKHOUSE_HOST and %% ?CLICKHOUSE_PORT to appropriate values. %% -%% docker run -d -p 18123:8123 -p19000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 -v "`pwd`/.ci/docker-compose-file/clickhouse/users.xml:/etc/clickhouse-server/users.xml" -v "`pwd`/.ci/docker-compose-file/clickhouse/config.xml:/etc/clickhouse-server/config.xml" clickhouse/clickhouse-server +%% docker run \ +%% -d \ +%% -p 18123:8123 \ +%% -p 19000:9000 \ +%% --name some-clickhouse-server \ +%% --ulimit nofile=262144:262144 \ +%% -v "`pwd`/.ci/docker-compose-file/clickhouse/users.xml:/etc/clickhouse-server/users.xml" \ +%% -v "`pwd`/.ci/docker-compose-file/clickhouse/config.xml:/etc/clickhouse-server/config.xml" \ +%% clickhouse/clickhouse-server all() -> emqx_common_test_helpers:all(?MODULE). @@ -43,9 +51,7 @@ init_per_suite(Config) -> of true -> ok = emqx_common_test_helpers:start_apps([emqx_conf]), - ok = emqx_connector_test_helpers:start_apps([emqx_resource]), - {ok, _} = application:ensure_all_started(emqx_connector), - {ok, _} = application:ensure_all_started(emqx_ee_connector), + ok = emqx_connector_test_helpers:start_apps([emqx_resource, ?APP]), %% Create the db table {ok, Conn} = clickhouse:start_link([ @@ -68,8 +74,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_conf]), - ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), - _ = application:stop(emqx_connector). + ok = emqx_connector_test_helpers:stop_apps([?APP, emqx_resource]). init_per_testcase(_, Config) -> Config. @@ -119,7 +124,6 @@ perform_lifecycle_check(ResourceID, InitialConfig) -> ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceID)), % % Perform query as further check that the resource is working as expected (fun() -> - erlang:display({pool_name, ResourceID}), QueryNoParamsResWrapper = emqx_resource:query(ResourceID, test_query_no_params()), ?assertMatch({ok, _}, QueryNoParamsResWrapper), {_, QueryNoParamsRes} = QueryNoParamsResWrapper, diff --git a/apps/emqx_bridge_dynamo/README.md b/apps/emqx_bridge_dynamo/README.md index 48dcb781d..245a68941 100644 --- a/apps/emqx_bridge_dynamo/README.md +++ b/apps/emqx_bridge_dynamo/README.md @@ -1,6 +1,6 @@ # EMQX DynamoDB Bridge -[Dynamodb](https://aws.amazon.com/dynamodb/) is a high-performance NoSQL database +[DynamoDB](https://aws.amazon.com/dynamodb/) is a high-performance NoSQL database service provided by Amazon that's designed for scalability and low-latency access to structured data. diff --git a/apps/emqx_bridge_gcp_pubsub/README.md b/apps/emqx_bridge_gcp_pubsub/README.md index e33c5ab15..6d136a5d3 100644 --- a/apps/emqx_bridge_gcp_pubsub/README.md +++ b/apps/emqx_bridge_gcp_pubsub/README.md @@ -10,7 +10,7 @@ User can create a rule and easily ingest IoT data into GCP Pub/Sub by leveraging # Documentation -- Refer to [Ingest data into GCP Pub/Sub](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-gcp-pubsub.html) +- Refer to [Ingest Data into GCP Pub/Sub](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-gcp-pubsub.html) for how to use EMQX dashboard to ingest IoT data into GCP Pub/Sub. - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl index 3945b2eaf..814051733 100644 --- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl @@ -288,6 +288,7 @@ gcp_pubsub_config(Config) -> " pipelining = ~b\n" " resource_opts = {\n" " request_timeout = 500ms\n" + " metrics_flush_interval = 700ms\n" " worker_pool_size = 1\n" " query_mode = ~s\n" " batch_size = ~b\n" @@ -529,12 +530,14 @@ wait_until_gauge_is(GaugeName, ExpectedValue, Timeout) -> end. receive_all_events(EventName, Timeout) -> - receive_all_events(EventName, Timeout, []). + receive_all_events(EventName, Timeout, _MaxEvents = 10, _Count = 0, _Acc = []). -receive_all_events(EventName, Timeout, Acc) -> +receive_all_events(_EventName, _Timeout, MaxEvents, Count, Acc) when Count >= MaxEvents -> + lists:reverse(Acc); +receive_all_events(EventName, Timeout, MaxEvents, Count, Acc) -> receive {telemetry, #{name := [_, _, EventName]} = Event} -> - receive_all_events(EventName, Timeout, [Event | Acc]) + receive_all_events(EventName, Timeout, MaxEvents, Count + 1, [Event | Acc]) after Timeout -> lists:reverse(Acc) end. @@ -557,8 +560,9 @@ wait_n_events(_TelemetryTable, _ResourceId, NEvents, _Timeout, _EventName) when ok; wait_n_events(TelemetryTable, ResourceId, NEvents, Timeout, EventName) -> receive - {telemetry, #{name := [_, _, EventName]}} -> - wait_n_events(TelemetryTable, ResourceId, NEvents - 1, Timeout, EventName) + {telemetry, #{name := [_, _, EventName], measurements := #{counter_inc := Inc}} = Event} -> + ct:pal("telemetry event: ~p", [Event]), + wait_n_events(TelemetryTable, ResourceId, NEvents - Inc, Timeout, EventName) after Timeout -> RecordedEvents = ets:tab2list(TelemetryTable), CurrentMetrics = current_metrics(ResourceId), @@ -575,7 +579,6 @@ t_publish_success(Config) -> ResourceId = ?config(resource_id, Config), ServiceAccountJSON = ?config(service_account_json, Config), TelemetryTable = ?config(telemetry_table, Config), - QueryMode = ?config(query_mode, Config), Topic = <<"t/topic">>, ?check_trace( create_bridge(Config), @@ -604,17 +607,6 @@ t_publish_success(Config) -> ), %% to avoid test flakiness wait_telemetry_event(TelemetryTable, success, ResourceId), - ExpectedInflightEvents = - case QueryMode of - sync -> 1; - async -> 3 - end, - wait_telemetry_event( - TelemetryTable, - inflight, - ResourceId, - #{n_events => ExpectedInflightEvents, timeout => 5_000} - ), wait_until_gauge_is(queuing, 0, 500), wait_until_gauge_is(inflight, 0, 500), assert_metrics( @@ -659,7 +651,6 @@ t_publish_success_local_topic(Config) -> ResourceId = ?config(resource_id, Config), ServiceAccountJSON = ?config(service_account_json, Config), TelemetryTable = ?config(telemetry_table, Config), - QueryMode = ?config(query_mode, Config), LocalTopic = <<"local/topic">>, {ok, _} = create_bridge(Config, #{<<"local_topic">> => LocalTopic}), assert_empty_metrics(ResourceId), @@ -678,17 +669,6 @@ t_publish_success_local_topic(Config) -> ), %% to avoid test flakiness wait_telemetry_event(TelemetryTable, success, ResourceId), - ExpectedInflightEvents = - case QueryMode of - sync -> 1; - async -> 3 - end, - wait_telemetry_event( - TelemetryTable, - inflight, - ResourceId, - #{n_events => ExpectedInflightEvents, timeout => 5_000} - ), wait_until_gauge_is(queuing, 0, 500), wait_until_gauge_is(inflight, 0, 500), assert_metrics( @@ -720,7 +700,6 @@ t_publish_templated(Config) -> ResourceId = ?config(resource_id, Config), ServiceAccountJSON = ?config(service_account_json, Config), TelemetryTable = ?config(telemetry_table, Config), - QueryMode = ?config(query_mode, Config), Topic = <<"t/topic">>, PayloadTemplate = << "{\"payload\": \"${payload}\"," @@ -766,17 +745,6 @@ t_publish_templated(Config) -> ), %% to avoid test flakiness wait_telemetry_event(TelemetryTable, success, ResourceId), - ExpectedInflightEvents = - case QueryMode of - sync -> 1; - async -> 3 - end, - wait_telemetry_event( - TelemetryTable, - inflight, - ResourceId, - #{n_events => ExpectedInflightEvents, timeout => 5_000} - ), wait_until_gauge_is(queuing, 0, 500), wait_until_gauge_is(inflight, 0, 500), assert_metrics( @@ -1113,9 +1081,6 @@ do_econnrefused_or_timeout_test(Config, Error) -> %% message as dropped; and since it never considers the %% response expired, this succeeds. econnrefused -> - wait_telemetry_event(TelemetryTable, queuing, ResourceId, #{ - timeout => 10_000, n_events => 1 - }), %% even waiting, hard to avoid flakiness... simpler to just sleep %% a bit until stabilization. ct:sleep(200), @@ -1135,8 +1100,8 @@ do_econnrefused_or_timeout_test(Config, Error) -> CurrentMetrics ); timeout -> - wait_until_gauge_is(inflight, 0, _Timeout = 400), - wait_until_gauge_is(queuing, 0, _Timeout = 400), + wait_until_gauge_is(inflight, 0, _Timeout = 1_000), + wait_until_gauge_is(queuing, 0, _Timeout = 1_000), assert_metrics( #{ dropped => 0, diff --git a/apps/emqx_bridge_influxdb/README.md b/apps/emqx_bridge_influxdb/README.md index fe0f14600..69df002c4 100644 --- a/apps/emqx_bridge_influxdb/README.md +++ b/apps/emqx_bridge_influxdb/README.md @@ -15,7 +15,7 @@ easily ingest IoT data into InfluxDB by leveraging # Documentation -- Refer to [Ingest data into InfluxDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-influxdb.html) +- Refer to [Ingest Data into InfluxDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-influxdb.html) for how to use EMQX dashboard to ingest IoT data into InfluxDB. - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) diff --git a/apps/emqx_bridge_iotdb/README.md b/apps/emqx_bridge_iotdb/README.md index 48f5d74c2..29b634dbc 100644 --- a/apps/emqx_bridge_iotdb/README.md +++ b/apps/emqx_bridge_iotdb/README.md @@ -12,13 +12,15 @@ It implements the connection management and interaction without need for a For more information on Apache IoTDB, please see its [official site](https://iotdb.apache.org/). + -# Contributing - [Mandatory] +# Contributing Please see our [contributing.md](../../CONTRIBUTING.md). # License diff --git a/apps/emqx_bridge_kafka/README.md b/apps/emqx_bridge_kafka/README.md index 07cae256b..56c340a70 100644 --- a/apps/emqx_bridge_kafka/README.md +++ b/apps/emqx_bridge_kafka/README.md @@ -16,7 +16,7 @@ For more information about Apache Kafka, please see its [official site](https:// # Configurations -Please see [Ingest data into Kafka](https://www.emqx.io/docs/en/v5.0/data-integration/data-bridge-kafka.html) for more detailed info. +Please see [Ingest Data into Kafka](https://www.emqx.io/docs/en/v5.0/data-integration/data-bridge-kafka.html) for more detailed info. # Contributing diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index 42729c002..6d18bbdf2 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -1074,7 +1074,7 @@ cluster(Config) -> {priv_data_dir, PrivDataDir}, {load_schema, true}, {start_autocluster, true}, - {schema_mod, emqx_ee_conf_schema}, + {schema_mod, emqx_enterprise_schema}, {env_handler, fun (emqx) -> application:set_env(emqx, boot_modules, [broker, router]), diff --git a/apps/emqx_bridge_mongodb/README.md b/apps/emqx_bridge_mongodb/README.md index 088c8467f..63a541dc5 100644 --- a/apps/emqx_bridge_mongodb/README.md +++ b/apps/emqx_bridge_mongodb/README.md @@ -13,7 +13,7 @@ User can create a rule and easily ingest IoT data into MongoDB by leveraging # Documentation -- Refer to [Ingest data into MongoDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-mongodb.html) +- Refer to [Ingest Data into MongoDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-mongodb.html) for how to use EMQX dashboard to ingest IoT data into MongoDB. - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) diff --git a/apps/emqx_bridge_mysql/README.md b/apps/emqx_bridge_mysql/README.md index d7c9b5647..8b4295e96 100644 --- a/apps/emqx_bridge_mysql/README.md +++ b/apps/emqx_bridge_mysql/README.md @@ -10,7 +10,7 @@ User can create a rule and easily ingest IoT data into MySQL by leveraging # Documentation -- Refer to [Ingest data into MySQL](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-mysql.html) +- Refer to [Ingest Data into MySQL](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-mysql.html) for how to use EMQX dashboard to ingest IoT data into MySQL. - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) diff --git a/apps/emqx_bridge_pgsql/README.md b/apps/emqx_bridge_pgsql/README.md index fc0bd3c6f..20e8ef86d 100644 --- a/apps/emqx_bridge_pgsql/README.md +++ b/apps/emqx_bridge_pgsql/README.md @@ -1,6 +1,6 @@ # EMQX PostgreSQL Bridge -[PostgreSQL](https://github.com/PostgreSQL/PostgreSQL) is an open-source relational +[PostgreSQL](https://www.postgresql.org/) is an open-source relational database management system (RDBMS) that uses and extends the SQL language. It is known for its reliability, data integrity, and advanced features such as support for JSON, XML, and other data formats. @@ -12,7 +12,7 @@ User can create a rule and easily ingest IoT data into PostgreSQL by leveraging # Documentation -- Refer to [Ingest data into PostgreSQL](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-pgsql.html) +- Refer to [Ingest Data into PostgreSQL](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-pgsql.html) for how to use EMQX dashboard to ingest IoT data into PostgreSQL. - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) diff --git a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl index 9f2011779..e4f17d76a 100644 --- a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl +++ b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl @@ -258,13 +258,18 @@ query_resource(Config, Request) -> emqx_resource:query(ResourceID, Request, #{timeout => 1_000}). query_resource_async(Config, Request) -> + query_resource_async(Config, Request, _Opts = #{}). + +query_resource_async(Config, Request, Opts) -> Name = ?config(pgsql_name, Config), BridgeType = ?config(pgsql_bridge_type, Config), Ref = alias([reply]), AsyncReplyFun = fun(Result) -> Ref ! {result, Ref, Result} end, ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + Timeout = maps:get(timeout, Opts, 500), Return = emqx_resource:query(ResourceID, Request, #{ - timeout => 500, async_reply_fun => {AsyncReplyFun, []} + timeout => Timeout, + async_reply_fun => {AsyncReplyFun, []} }), {Return, Ref}. @@ -498,9 +503,9 @@ t_write_timeout(Config) -> Config, #{ <<"resource_opts">> => #{ - <<"request_timeout">> => 500, - <<"resume_interval">> => 100, - <<"health_check_interval">> => 100 + <<"auto_restart_interval">> => <<"100ms">>, + <<"resume_interval">> => <<"100ms">>, + <<"health_check_interval">> => <<"100ms">> } } ), @@ -515,7 +520,7 @@ t_write_timeout(Config) -> Res1 = case QueryMode of async -> - query_resource_async(Config, {send_message, SentData}); + query_resource_async(Config, {send_message, SentData}, #{timeout => 60_000}); sync -> query_resource(Config, {send_message, SentData}) end, @@ -526,7 +531,17 @@ t_write_timeout(Config) -> {_, Ref} when is_reference(Ref) -> case receive_result(Ref, 15_000) of {ok, Res} -> - ?assertMatch({error, {unrecoverable_error, _}}, Res); + %% we may receive a successful result depending on + %% timing, if the request is retried after the + %% failure is healed. + case Res of + {error, {unrecoverable_error, _}} -> + ok; + {ok, _} -> + ok; + _ -> + ct:fail("unexpected result: ~p", [Res]) + end; timeout -> ct:pal("mailbox:\n ~p", [process_info(self(), messages)]), ct:fail("no response received") diff --git a/apps/emqx_bridge_pulsar/README.md b/apps/emqx_bridge_pulsar/README.md index fbd8bf81d..cbdaeff2f 100644 --- a/apps/emqx_bridge_pulsar/README.md +++ b/apps/emqx_bridge_pulsar/README.md @@ -15,11 +15,13 @@ used by authentication and authorization applications. For more information on Apache Pulsar, please see its [official site](https://pulsar.apache.org/). + # Contributing diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src index 1665548ae..f50c34391 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_pulsar, [ {description, "EMQX Pulsar Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl index 5ed706511..5906cc57a 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl @@ -81,6 +81,7 @@ on_start(InstanceId, Config) -> } = Config, Servers = format_servers(Servers0), ClientId = make_client_id(InstanceId, BridgeName), + ok = emqx_resource:allocate_resource(InstanceId, pulsar_client_id, ClientId), SSLOpts = emqx_tls_lib:to_client_opts(SSL), ConnectTimeout = maps:get(connect_timeout, Config, timer:seconds(5)), ClientOpts = #{ @@ -116,15 +117,29 @@ on_start(InstanceId, Config) -> start_producer(Config, InstanceId, ClientId, ClientOpts). -spec on_stop(resource_id(), state()) -> ok. -on_stop(_InstanceId, State) -> - #{ - pulsar_client_id := ClientId, - producers := Producers - } = State, - stop_producers(ClientId, Producers), - stop_client(ClientId), - ?tp(pulsar_bridge_stopped, #{instance_id => _InstanceId}), - ok. +on_stop(InstanceId, _State) -> + case emqx_resource:get_allocated_resources(InstanceId) of + #{pulsar_client_id := ClientId, pulsar_producers := Producers} -> + stop_producers(ClientId, Producers), + stop_client(ClientId), + ?tp(pulsar_bridge_stopped, #{ + instance_id => InstanceId, + pulsar_client_id => ClientId, + pulsar_producers => Producers + }), + ok; + #{pulsar_client_id := ClientId} -> + stop_client(ClientId), + ?tp(pulsar_bridge_stopped, #{ + instance_id => InstanceId, + pulsar_client_id => ClientId, + pulsar_producers => undefined + }), + ok; + _ -> + ?tp(pulsar_bridge_stopped, #{instance_id => InstanceId}), + ok + end. -spec on_get_status(resource_id(), state()) -> connected | disconnected. on_get_status(_InstanceId, State = #{}) -> @@ -325,6 +340,8 @@ start_producer(Config, InstanceId, ClientId, ClientOpts) -> ?tp(pulsar_producer_about_to_start_producers, #{producer_name => ProducerName}), try pulsar:ensure_supervised_producers(ClientId, PulsarTopic, ProducerOpts) of {ok, Producers} -> + ok = emqx_resource:allocate_resource(InstanceId, pulsar_producers, Producers), + ?tp(pulsar_producer_producers_allocated, #{}), State = #{ pulsar_client_id => ClientId, producers => Producers, diff --git a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl index be38f6625..3605baaab 100644 --- a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl @@ -43,7 +43,9 @@ only_once_tests() -> t_send_when_down, t_send_when_timeout, t_failure_to_start_producer, - t_producer_process_crash + t_producer_process_crash, + t_resource_manager_crash_after_producers_started, + t_resource_manager_crash_before_producers_started ]. init_per_suite(Config) -> @@ -429,7 +431,19 @@ wait_until_producer_connected() -> wait_until_connected(pulsar_producers_sup, pulsar_producer). wait_until_connected(SupMod, Mod) -> - Pids = [ + Pids = get_pids(SupMod, Mod), + ?retry( + _Sleep = 300, + _Attempts0 = 20, + lists:foreach(fun(P) -> {connected, _} = sys:get_state(P) end, Pids) + ), + ok. + +get_pulsar_producers() -> + get_pids(pulsar_producers_sup, pulsar_producer). + +get_pids(SupMod, Mod) -> + [ P || {_Name, SupPid, _Type, _Mods} <- supervisor:which_children(SupMod), P <- element(2, process_info(SupPid, links)), @@ -437,13 +451,7 @@ wait_until_connected(SupMod, Mod) -> {Mod, init, _} -> true; _ -> false end - ], - ?retry( - _Sleep = 300, - _Attempts0 = 20, - lists:foreach(fun(P) -> {connected, _} = sys:get_state(P) end, Pids) - ), - ok. + ]. create_rule_and_action_http(Config) -> PulsarName = ?config(pulsar_name, Config), @@ -496,7 +504,7 @@ cluster(Config) -> {priv_data_dir, PrivDataDir}, {load_schema, true}, {start_autocluster, true}, - {schema_mod, emqx_ee_conf_schema}, + {schema_mod, emqx_enterprise_schema}, {env_handler, fun (emqx) -> application:set_env(emqx, boot_modules, [broker, router]), @@ -528,6 +536,18 @@ start_cluster(Cluster) -> end), Nodes. +kill_resource_managers() -> + ct:pal("gonna kill resource managers"), + lists:foreach( + fun({_, Pid, _, _}) -> + ct:pal("terminating resource manager ~p", [Pid]), + %% sys:terminate(Pid, stop), + exit(Pid, kill), + ok + end, + supervisor:which_children(emqx_resource_manager_sup) + ). + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ @@ -921,7 +941,11 @@ t_producer_process_crash(Config) -> ok after 1_000 -> ct:fail("pid didn't die") end, - ?assertEqual({ok, connecting}, emqx_resource_manager:health_check(ResourceId)), + ?retry( + _Sleep0 = 50, + _Attempts0 = 50, + ?assertEqual({ok, connecting}, emqx_resource_manager:health_check(ResourceId)) + ), %% Should recover given enough time. ?retry( _Sleep = 1_000, @@ -952,6 +976,69 @@ t_producer_process_crash(Config) -> ), ok. +t_resource_manager_crash_after_producers_started(Config) -> + ?check_trace( + begin + ?force_ordering( + #{?snk_kind := pulsar_producer_producers_allocated}, + #{?snk_kind := will_kill_resource_manager} + ), + ?force_ordering( + #{?snk_kind := resource_manager_killed}, + #{?snk_kind := pulsar_producer_bridge_started} + ), + spawn_link(fun() -> + ?tp(will_kill_resource_manager, #{}), + kill_resource_managers(), + ?tp(resource_manager_killed, #{}), + ok + end), + %% even if the resource manager is dead, we can still + %% clear the allocated resources. + {{error, {config_update_crashed, {killed, _}}}, {ok, _}} = + ?wait_async_action( + create_bridge(Config), + #{?snk_kind := pulsar_bridge_stopped, pulsar_producers := Producers} when + Producers =/= undefined, + 10_000 + ), + ok + end, + [] + ), + ok. + +t_resource_manager_crash_before_producers_started(Config) -> + ?check_trace( + begin + ?force_ordering( + #{?snk_kind := pulsar_producer_capture_name}, + #{?snk_kind := will_kill_resource_manager} + ), + ?force_ordering( + #{?snk_kind := resource_manager_killed}, + #{?snk_kind := pulsar_producer_about_to_start_producers} + ), + spawn_link(fun() -> + ?tp(will_kill_resource_manager, #{}), + kill_resource_managers(), + ?tp(resource_manager_killed, #{}), + ok + end), + %% even if the resource manager is dead, we can still + %% clear the allocated resources. + {{error, {config_update_crashed, {killed, _}}}, {ok, _}} = + ?wait_async_action( + create_bridge(Config), + #{?snk_kind := pulsar_bridge_stopped, pulsar_producers := undefined}, + 10_000 + ), + ok + end, + [] + ), + ok. + t_cluster(Config) -> MQTTTopic = ?config(mqtt_topic, Config), ResourceId = resource_id(Config), diff --git a/apps/emqx_bridge_rabbitmq/README.md b/apps/emqx_bridge_rabbitmq/README.md index 420a9e048..a85f5507d 100644 --- a/apps/emqx_bridge_rabbitmq/README.md +++ b/apps/emqx_bridge_rabbitmq/README.md @@ -21,8 +21,10 @@ and easily ingest IoT data into RabbitMQ by leveraging # Documentation + - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) for an introduction to the EMQX rules engine. diff --git a/apps/emqx_bridge_redis/README.md b/apps/emqx_bridge_redis/README.md index 73ec41f07..5aca9fddb 100644 --- a/apps/emqx_bridge_redis/README.md +++ b/apps/emqx_bridge_redis/README.md @@ -11,7 +11,7 @@ User can create a rule and easily ingest IoT data into Redis by leveraging # Documentation -- Refer to [Ingest data into Redis](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-redis.html) +- Refer to [Ingest Data into Redis](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-redis.html) for how to use EMQX dashboard to ingest IoT data into Redis. - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) diff --git a/apps/emqx_bridge_rocketmq/README.md b/apps/emqx_bridge_rocketmq/README.md index 252e6beac..e6b1ceff3 100644 --- a/apps/emqx_bridge_rocketmq/README.md +++ b/apps/emqx_bridge_rocketmq/README.md @@ -11,7 +11,7 @@ User can create a rule and easily ingest IoT data into RocketMQ by leveraging # Documentation -- Refer to [Ingest data into RocketMQ](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-rocketmq.html) +- Refer to [Ingest Data into RocketMQ](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-rocketmq.html) for how to use EMQX dashboard to ingest IoT data into RocketMQ. - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) diff --git a/apps/emqx_bridge_sqlserver/README.md b/apps/emqx_bridge_sqlserver/README.md index ccb1267d8..629560036 100644 --- a/apps/emqx_bridge_sqlserver/README.md +++ b/apps/emqx_bridge_sqlserver/README.md @@ -16,7 +16,7 @@ For more information about Microsoft SQL Server, please see the [official site]( # Configurations -Please see [Ingest data into SQL Server](https://www.emqx.io/docs/en/v5.0/data-integration/data-bridge-sqlserver.html) for more detailed information. +Please see [Ingest Data into SQL Server](https://www.emqx.io/docs/en/v5.0/data-integration/data-bridge-sqlserver.html) for more detailed information. # HTTP APIs diff --git a/apps/emqx_bridge_tdengine/README.md b/apps/emqx_bridge_tdengine/README.md index 25faf4c14..e32e16f09 100644 --- a/apps/emqx_bridge_tdengine/README.md +++ b/apps/emqx_bridge_tdengine/README.md @@ -13,7 +13,7 @@ User can create a rule and easily ingest IoT data into TDEngine by leveraging # Documentation -- Refer to [Ingest data into TDEngine](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-tdengine.html) +- Refer to [Ingest Data into TDEngine](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-tdengine.html) for how to use EMQX dashboard to ingest IoT data into TDEngine. - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) diff --git a/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl b/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl index fe72cd65b..c558dc908 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl @@ -38,8 +38,8 @@ ]). start_link() -> - MaxHistory = emqx_conf:get(["node", "cluster_call", "max_history"], 100), - CleanupMs = emqx_conf:get(["node", "cluster_call", "cleanup_interval"], 5 * 60 * 1000), + MaxHistory = emqx_conf:get([node, cluster_call, max_history], 100), + CleanupMs = emqx_conf:get([node, cluster_call, cleanup_interval], 5 * 60 * 1000), start_link(MaxHistory, CleanupMs). start_link(MaxHistory, CleanupMs) -> diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index c31a16b9b..3a5ebc77a 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.20"}, + {vsn, "0.1.21"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index b37c3f71e..792dea16e 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -49,10 +49,10 @@ -define(MERGED_CONFIGS, [ emqx_bridge_schema, emqx_retainer_schema, - emqx_statsd_schema, emqx_authn_schema, emqx_authz_schema, emqx_auto_subscribe_schema, + {emqx_telemetry_schema, ce}, emqx_modules_schema, emqx_plugins_schema, emqx_dashboard_schema, @@ -109,11 +109,25 @@ roots() -> ] ++ emqx_schema:roots(medium) ++ emqx_schema:roots(low) ++ - lists:flatmap(fun roots/1, ?MERGED_CONFIGS). + lists:flatmap(fun roots/1, common_apps()). validations() -> hocon_schema:validations(emqx_schema) ++ - lists:flatmap(fun hocon_schema:validations/1, ?MERGED_CONFIGS). + lists:flatmap(fun hocon_schema:validations/1, common_apps()). + +common_apps() -> + Edition = emqx_release:edition(), + lists:filtermap( + fun + ({N, E}) -> + case E =:= Edition of + true -> {true, N}; + false -> false + end; + (N) when is_atom(N) -> {true, N} + end, + ?MERGED_CONFIGS + ). fields("cluster") -> [ @@ -561,7 +575,7 @@ fields("node") -> emqx_schema:comma_separated_atoms(), #{ mapping => "emqx_machine.applications", - default => [], + default => <<"">>, 'readOnly' => true, importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(node_applications) @@ -688,11 +702,12 @@ fields("rpc") -> desc => ?DESC(rpc_mode) } )}, - {"driver", + {"protocol", sc( hoconsc:enum([tcp, ssl]), #{ mapping => "gen_rpc.driver", + aliases => [driver], default => tcp, desc => ?DESC(rpc_driver) } @@ -870,19 +885,22 @@ fields("rpc") -> ]; fields("log") -> [ - {"console_handler", + {"console", + sc(?R_REF("console_handler"), #{ + aliases => [console_handler], + importance => ?IMPORTANCE_HIGH + })}, + {"file", sc( - ?R_REF("console_handler"), - #{importance => ?IMPORTANCE_HIGH} - )}, - {"file_handlers", - sc( - map(name, ?R_REF("log_file_handler")), + ?UNION([ + ?R_REF("log_file_handler"), + ?MAP(handler_name, ?R_REF("log_file_handler")) + ]), #{ desc => ?DESC("log_file_handlers"), - %% because file_handlers is a map - %% so there has to be a default value in order to populate the raw configs - default => #{<<"default">> => #{<<"level">> => <<"warning">>}}, + converter => fun ensure_file_handlers/2, + default => #{<<"level">> => <<"warning">>}, + aliases => [file_handlers], importance => ?IMPORTANCE_HIGH } )} @@ -891,51 +909,41 @@ fields("console_handler") -> log_handler_common_confs(console); fields("log_file_handler") -> [ - {"file", + {"to", sc( file(), #{ desc => ?DESC("log_file_handler_file"), + default => <<"${EMQX_LOG_DIR}/emqx.log">>, + aliases => [file], + importance => ?IMPORTANCE_HIGH, converter => fun(Path, Opts) -> emqx_schema:naive_env_interpolation(ensure_unicode_path(Path, Opts)) - end, - default => <<"${EMQX_LOG_DIR}/emqx.log">> + end } )}, - {"rotation", + {"rotation_count", sc( - ?R_REF("log_rotation"), - #{} + range(1, 128), + #{ + aliases => [rotation], + default => 10, + converter => fun convert_rotation/2, + desc => ?DESC("log_rotation_count"), + importance => ?IMPORTANCE_MEDIUM + } )}, - {"max_size", + {"rotation_size", sc( hoconsc:union([infinity, emqx_schema:bytesize()]), #{ default => <<"50MB">>, desc => ?DESC("log_file_handler_max_size"), + aliases => [max_size], importance => ?IMPORTANCE_MEDIUM } )} ] ++ log_handler_common_confs(file); -fields("log_rotation") -> - [ - {"enable", - sc( - boolean(), - #{ - default => true, - desc => ?DESC("log_rotation_enable") - } - )}, - {"count", - sc( - range(1, 2048), - #{ - default => 10, - desc => ?DESC("log_rotation_count") - } - )} - ]; fields("log_overload_kill") -> [ {"enable", @@ -1043,8 +1051,8 @@ translation("ekka") -> [{"cluster_discovery", fun tr_cluster_discovery/1}]; translation("kernel") -> [ - {"logger_level", fun tr_logger_level/1}, - {"logger", fun tr_logger_handlers/1}, + {"logger_level", fun emqx_config_logger:tr_level/1}, + {"logger", fun emqx_config_logger:tr_handlers/1}, {"error_logger", fun(_) -> silent end} ]; translation("emqx") -> @@ -1118,24 +1126,9 @@ tr_cluster_discovery(Conf) -> Strategy = conf_get("cluster.discovery_strategy", Conf), {Strategy, filter(cluster_options(Strategy, Conf))}. --spec tr_logger_level(hocon:config()) -> logger:level(). -tr_logger_level(Conf) -> - emqx_config_logger:tr_level(Conf). - -tr_logger_handlers(Conf) -> - emqx_config_logger:tr_handlers(Conf). - log_handler_common_confs(Handler) -> - lists:map( - fun - ({_Name, #{importance := _}} = F) -> F; - ({Name, Sc}) -> {Name, Sc#{importance => ?IMPORTANCE_LOW}} - end, - do_log_handler_common_confs(Handler) - ). -do_log_handler_common_confs(Handler) -> %% we rarely support dynamic defaults like this - %% for this one, we have build-time defualut the same as runtime default + %% for this one, we have build-time default the same as runtime default %% so it's less tricky EnableValues = case Handler of @@ -1145,21 +1138,31 @@ do_log_handler_common_confs(Handler) -> EnvValue = os:getenv("EMQX_DEFAULT_LOG_HANDLER"), Enable = lists:member(EnvValue, EnableValues), [ + {"level", + sc( + log_level(), + #{ + default => warning, + desc => ?DESC("common_handler_level"), + importance => ?IMPORTANCE_HIGH + } + )}, {"enable", sc( boolean(), #{ default => Enable, desc => ?DESC("common_handler_enable"), - importance => ?IMPORTANCE_LOW + importance => ?IMPORTANCE_MEDIUM } )}, - {"level", + {"formatter", sc( - log_level(), + hoconsc:enum([text, json]), #{ - default => warning, - desc => ?DESC("common_handler_level") + default => text, + desc => ?DESC("common_handler_formatter"), + importance => ?IMPORTANCE_MEDIUM } )}, {"time_offset", @@ -1178,16 +1181,7 @@ do_log_handler_common_confs(Handler) -> #{ default => unlimited, desc => ?DESC("common_handler_chars_limit"), - importance => ?IMPORTANCE_LOW - } - )}, - {"formatter", - sc( - hoconsc:enum([text, json]), - #{ - default => text, - desc => ?DESC("common_handler_formatter"), - importance => ?IMPORTANCE_MEDIUM + importance => ?IMPORTANCE_HIDDEN } )}, {"single_line", @@ -1196,7 +1190,7 @@ do_log_handler_common_confs(Handler) -> #{ default => true, desc => ?DESC("common_handler_single_line"), - importance => ?IMPORTANCE_LOW + importance => ?IMPORTANCE_HIDDEN } )}, {"sync_mode_qlen", @@ -1204,7 +1198,8 @@ do_log_handler_common_confs(Handler) -> non_neg_integer(), #{ default => 100, - desc => ?DESC("common_handler_sync_mode_qlen") + desc => ?DESC("common_handler_sync_mode_qlen"), + importance => ?IMPORTANCE_HIDDEN } )}, {"drop_mode_qlen", @@ -1212,7 +1207,8 @@ do_log_handler_common_confs(Handler) -> pos_integer(), #{ default => 3000, - desc => ?DESC("common_handler_drop_mode_qlen") + desc => ?DESC("common_handler_drop_mode_qlen"), + importance => ?IMPORTANCE_HIDDEN } )}, {"flush_qlen", @@ -1220,17 +1216,19 @@ do_log_handler_common_confs(Handler) -> pos_integer(), #{ default => 8000, - desc => ?DESC("common_handler_flush_qlen") + desc => ?DESC("common_handler_flush_qlen"), + importance => ?IMPORTANCE_HIDDEN } )}, - {"overload_kill", sc(?R_REF("log_overload_kill"), #{})}, - {"burst_limit", sc(?R_REF("log_burst_limit"), #{})}, + {"overload_kill", sc(?R_REF("log_overload_kill"), #{importance => ?IMPORTANCE_HIDDEN})}, + {"burst_limit", sc(?R_REF("log_burst_limit"), #{importance => ?IMPORTANCE_HIDDEN})}, {"supervisor_reports", sc( hoconsc:enum([error, progress]), #{ default => error, - desc => ?DESC("common_handler_supervisor_reports") + desc => ?DESC("common_handler_supervisor_reports"), + importance => ?IMPORTANCE_HIDDEN } )}, {"max_depth", @@ -1238,7 +1236,8 @@ do_log_handler_common_confs(Handler) -> hoconsc:union([unlimited, non_neg_integer()]), #{ default => 100, - desc => ?DESC("common_handler_max_depth") + desc => ?DESC("common_handler_max_depth"), + importance => ?IMPORTANCE_HIDDEN } )} ]. @@ -1356,6 +1355,22 @@ validator_string_re(Val, RE, Error) -> node_array() -> hoconsc:union([emqx_schema:comma_separated_atoms(), hoconsc:array(atom())]). +ensure_file_handlers(Conf, _Opts) -> + FileFields = lists:flatmap( + fun({F, Schema}) -> + Alias = [atom_to_binary(A) || A <- maps:get(aliases, Schema, [])], + [list_to_binary(F) | Alias] + end, + fields("log_file_handler") + ), + HandlersWithoutName = maps:with(FileFields, Conf), + HandlersWithName = maps:without(FileFields, Conf), + emqx_utils_maps:deep_merge(#{<<"default">> => HandlersWithoutName}, HandlersWithName). + +convert_rotation(undefined, _Opts) -> undefined; +convert_rotation(#{} = Rotation, _Opts) -> maps:get(<<"count">>, Rotation, 10); +convert_rotation(Count, _Opts) when is_integer(Count) -> Count. + ensure_unicode_path(undefined, _) -> undefined; ensure_unicode_path(Path, #{make_serializable := true}) -> diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 4eaf3db6b..32c66fb90 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -48,6 +48,200 @@ array_nodes_test() -> ), ok. +%% erlfmt-ignore +-define(OUTDATED_LOG_CONF, + """ +log.console_handler { + burst_limit { + enable = true + max_count = 10000 + window_time = 1000 + } + chars_limit = unlimited + drop_mode_qlen = 3000 + enable = true + flush_qlen = 8000 + formatter = text + level = warning + max_depth = 100 + overload_kill { + enable = true + mem_size = 31457280 + qlen = 20000 + restart_after = 5000 + } + single_line = true + supervisor_reports = error + sync_mode_qlen = 100 + time_offset = \"+02:00\" +} +log.file_handlers { + default { + burst_limit { + enable = true + max_count = 10000 + window_time = 1000 + } + chars_limit = unlimited + drop_mode_qlen = 3000 + enable = true + file = \"log/my-emqx.log\" + flush_qlen = 8000 + formatter = text + level = debug + max_depth = 100 + max_size = \"1024MB\" + overload_kill { + enable = true + mem_size = 31457280 + qlen = 20000 + restart_after = 5000 + } + rotation {count = 20, enable = true} + single_line = true + supervisor_reports = error + sync_mode_qlen = 100 + time_offset = \"+01:00\" + } +} + """ +). +-define(FORMATTER(TimeOffset), + {emqx_logger_textfmt, #{ + chars_limit => unlimited, + depth => 100, + single_line => true, + template => [time, " [", level, "] ", msg, "\n"], + time_offset => TimeOffset + }} +). + +-define(FILTERS, [{drop_progress_reports, {fun logger_filters:progress/2, stop}}]). +-define(LOG_CONFIG, #{ + burst_limit_enable => true, + burst_limit_max_count => 10000, + burst_limit_window_time => 1000, + drop_mode_qlen => 3000, + flush_qlen => 8000, + overload_kill_enable => true, + overload_kill_mem_size => 31457280, + overload_kill_qlen => 20000, + overload_kill_restart_after => 5000, + sync_mode_qlen => 100 +}). + +outdated_log_test() -> + validate_log(?OUTDATED_LOG_CONF). + +validate_log(Conf) -> + ensure_acl_conf(), + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + Conf0 = <>, + {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), + ConfList = hocon_tconf:generate(emqx_conf_schema, ConfMap0), + Kernel = proplists:get_value(kernel, ConfList), + + ?assertEqual(silent, proplists:get_value(error_logger, Kernel)), + ?assertEqual(debug, proplists:get_value(logger_level, Kernel)), + Loggers = proplists:get_value(logger, Kernel), + FileHandler = lists:keyfind(logger_disk_log_h, 3, Loggers), + ?assertEqual( + {handler, default, logger_disk_log_h, #{ + config => ?LOG_CONFIG#{ + type => wrap, + file => "log/my-emqx.log", + max_no_bytes => 1073741824, + max_no_files => 20 + }, + filesync_repeat_interval => no_repeat, + filters => ?FILTERS, + formatter => ?FORMATTER("+01:00"), + level => debug + }}, + FileHandler + ), + ConsoleHandler = lists:keyfind(logger_std_h, 3, Loggers), + ?assertEqual( + {handler, console, logger_std_h, #{ + config => ?LOG_CONFIG#{type => standard_io}, + filters => ?FILTERS, + formatter => ?FORMATTER("+02:00"), + level => warning + }}, + ConsoleHandler + ). + +%% erlfmt-ignore +-define(KERNEL_LOG_CONF, + """ + log.console { + enable = true + formatter = text + level = warning + time_offset = \"+02:00\" + } + log.file { + enable = false + file = \"log/xx-emqx.log\" + formatter = text + level = debug + rotation_count = 20 + rotation_size = \"1024MB\" + time_offset = \"+01:00\" + } + log.file_handlers.default { + enable = true + file = \"log/my-emqx.log\" + } + """ +). + +log_test() -> + validate_log(?KERNEL_LOG_CONF). + +%% erlfmt-ignore +log_rotation_count_limit_test() -> + ensure_acl_conf(), + Format = + """ + log.file { + enable = true + to = \"log/emqx.log\" + formatter = text + level = debug + rotation = {count = ~w} + rotation_size = \"1024MB\" + } + """, + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + lists:foreach(fun({Conf, Count}) -> + Conf0 = <>, + {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), + ConfList = hocon_tconf:generate(emqx_conf_schema, ConfMap0), + Kernel = proplists:get_value(kernel, ConfList), + Loggers = proplists:get_value(logger, Kernel), + ?assertMatch( + {handler, default, logger_disk_log_h, #{ + config := #{max_no_files := Count} + }}, + lists:keyfind(logger_disk_log_h, 3, Loggers) + ) + end, + [{to_bin(Format, [1]), 1}, {to_bin(Format, [128]), 128}]), + lists:foreach(fun({Conf, Count}) -> + Conf0 = <>, + {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), + ?assertThrow({emqx_conf_schema, + [#{kind := validation_error, + mismatches := #{"handler_name" := + #{kind := validation_error, + path := "log.file.default.rotation_count", + reason := #{expected_type := "1..128"}, + value := Count} + }}]}, + hocon_tconf:generate(emqx_conf_schema, ConfMap0)) + end, [{to_bin(Format, [0]), 0}, {to_bin(Format, [129]), 129}]). + %% erlfmt-ignore -define(BASE_AUTHN_ARRAY, """ @@ -86,36 +280,44 @@ authn_validations_test() -> OKHttps = to_bin(?BASE_AUTHN_ARRAY, [post, true, <<"https://127.0.0.1:8080">>]), Conf0 = <>, {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), - ?assert(is_list(hocon_tconf:generate(emqx_conf_schema, ConfMap0))), + {_, Res0} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap0, #{format => richmap}), + Headers0 = authentication_headers(Res0), + ?assertEqual(<<"application/json">>, maps:get(<<"content-type">>, Headers0)), + %% accept from converter + ?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers0)), OKHttp = to_bin(?BASE_AUTHN_ARRAY, [post, false, <<"http://127.0.0.1:8080">>]), Conf1 = <>, {ok, ConfMap1} = hocon:binary(Conf1, #{format => richmap}), - ?assert(is_list(hocon_tconf:generate(emqx_conf_schema, ConfMap1))), + {_, Res1} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap1, #{format => richmap}), + Headers1 = authentication_headers(Res1), + ?assertEqual(<<"application/json">>, maps:get(<<"content-type">>, Headers1), Headers1), + ?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers1), Headers1), DisableSSLWithHttps = to_bin(?BASE_AUTHN_ARRAY, [post, false, <<"https://127.0.0.1:8080">>]), Conf2 = <>, {ok, ConfMap2} = hocon:binary(Conf2, #{format => richmap}), ?assertThrow( ?ERROR(check_http_ssl_opts), - hocon_tconf:generate(emqx_conf_schema, ConfMap2) + hocon_tconf:map_translate(emqx_conf_schema, ConfMap2, #{format => richmap}) ), BadHeader = to_bin(?BASE_AUTHN_ARRAY, [get, true, <<"https://127.0.0.1:8080">>]), Conf3 = <>, {ok, ConfMap3} = hocon:binary(Conf3, #{format => richmap}), - ?assertThrow( - ?ERROR(check_http_headers), - hocon_tconf:generate(emqx_conf_schema, ConfMap3) - ), + {_, Res3} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap3, #{format => richmap}), + Headers3 = authentication_headers(Res3), + %% remove the content-type header when get method + ?assertEqual(false, maps:is_key(<<"content-type">>, Headers3), Headers3), + ?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers3), Headers3), BadHeaderWithTuple = binary:replace(BadHeader, [<<"[">>, <<"]">>], <<"">>, [global]), Conf4 = <>, {ok, ConfMap4} = hocon:binary(Conf4, #{format => richmap}), - ?assertThrow( - ?ERROR(check_http_headers), - hocon_tconf:generate(emqx_conf_schema, ConfMap4) - ), + {_, Res4} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap4, #{}), + Headers4 = authentication_headers(Res4), + ?assertEqual(false, maps:is_key(<<"content-type">>, Headers4), Headers4), + ?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers4), Headers4), ok. %% erlfmt-ignore @@ -200,6 +402,10 @@ listeners_test() -> ), ok. +authentication_headers(Conf) -> + [#{<<"headers">> := Headers}] = hocon_maps:get("authentication", Conf), + Headers. + doc_gen_test() -> ensure_acl_conf(), %% the json file too large to encode. @@ -238,7 +444,7 @@ log_path_test_() -> #{<<"log">> => #{<<"file_handlers">> => #{<<"name1">> => #{<<"file">> => Path}}}} end, Assert = fun(Name, Path, Conf) -> - ?assertMatch(#{log := #{file_handlers := #{Name := #{file := Path}}}}, Conf) + ?assertMatch(#{log := #{file := #{Name := #{to := Path}}}}, Conf) end, [ @@ -251,7 +457,15 @@ log_path_test_() -> {emqx_conf_schema, [ #{ kind := validation_error, - reason := {"bad_file_path_string", _} + mismatches := + #{ + "handler_name" := + #{ + kind := validation_error, + path := "log.file.name1.to", + reason := {"bad_file_path_string", _} + } + } } ]}, check(Fh(<<239, 32, 132, 47, 117, 116, 102, 56>>)) @@ -262,7 +476,15 @@ log_path_test_() -> {emqx_conf_schema, [ #{ kind := validation_error, - reason := {"not_string", _} + mismatches := + #{ + "handler_name" := + #{ + kind := validation_error, + path := "log.file.name1.to", + reason := {"not_string", _} + } + } } ]}, check(Fh(#{<<"foo">> => <<"bar">>})) diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index 76c3e8bb9..e9f79723a 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "EMQX Data Integration Connectors"}, - {vsn, "0.1.23"}, + {vsn, "0.1.24"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index cce62b410..7c95febe8 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -32,22 +32,17 @@ on_query/3, on_query_async/4, on_get_status/2, - reply_delegator/2 + reply_delegator/3 ]). --type url() :: emqx_http_lib:uri_map(). --reflect_type([url/0]). --typerefl_from_string({url/0, emqx_http_lib, uri_parse}). - -export([ roots/0, fields/1, desc/1, - validations/0, namespace/0 ]). --export([check_ssl_opts/2, validate_method/1, join_paths/2]). +-export([validate_method/1, join_paths/2]). -type connect_timeout() :: emqx_schema:duration() | infinity. -type pool_type() :: random | hash. @@ -70,20 +65,6 @@ roots() -> fields(config) -> [ - {base_url, - sc( - url(), - #{ - required => true, - validator => fun - (#{query := _Query}) -> - {error, "There must be no query in the base_url"}; - (_) -> - ok - end, - desc => ?DESC("base_url") - } - )}, {connect_timeout, sc( emqx_schema:duration_ms(), @@ -172,9 +153,6 @@ desc("request") -> desc(_) -> undefined. -validations() -> - [{check_ssl_opts, fun check_ssl_opts/1}]. - validate_method(M) when M =:= <<"post">>; M =:= <<"put">>; M =:= <<"get">>; M =:= <<"delete">> -> ok; validate_method(M) -> @@ -268,10 +246,11 @@ on_query(InstId, {send_message, Msg}, State) -> request_timeout := Timeout } = process_request(Request, Msg), %% bridge buffer worker has retry, do not let ehttpc retry - Retry = 0, + Retry = 2, + ClientId = maps:get(clientid, Msg, undefined), on_query( InstId, - {undefined, Method, {Path, Headers, Body}, Timeout, Retry}, + {ClientId, Method, {Path, Headers, Body}, Timeout, Retry}, State ) end; @@ -371,9 +350,10 @@ on_query_async(InstId, {send_message, Msg}, ReplyFunAndArgs, State) -> headers := Headers, request_timeout := Timeout } = process_request(Request, Msg), + ClientId = maps:get(clientid, Msg, undefined), on_query_async( InstId, - {undefined, Method, {Path, Headers, Body}, Timeout}, + {ClientId, Method, {Path, Headers, Body}, Timeout}, ReplyFunAndArgs, State ) @@ -395,12 +375,22 @@ on_query_async( } ), NRequest = formalize_request(Method, BasePath, Request), + MaxAttempts = maps:get(max_attempts, State, 3), + Context = #{ + attempt => 1, + max_attempts => MaxAttempts, + state => State, + key_or_num => KeyOrNum, + method => Method, + request => NRequest, + timeout => Timeout + }, ok = ehttpc:request_async( Worker, Method, NRequest, Timeout, - {fun ?MODULE:reply_delegator/2, [ReplyFunAndArgs]} + {fun ?MODULE:reply_delegator/3, [Context, ReplyFunAndArgs]} ), {ok, Worker}. @@ -582,18 +572,6 @@ make_method(M) when M == <<"PUT">>; M == <<"put">> -> put; make_method(M) when M == <<"GET">>; M == <<"get">> -> get; make_method(M) when M == <<"DELETE">>; M == <<"delete">> -> delete. -check_ssl_opts(Conf) -> - check_ssl_opts("base_url", Conf). - -check_ssl_opts(URLFrom, Conf) -> - #{scheme := Scheme} = hocon_maps:get(URLFrom, Conf), - SSL = hocon_maps:get("ssl", Conf), - case {Scheme, maps:get(enable, SSL, false)} of - {http, false} -> true; - {https, true} -> true; - {_, _} -> false - end. - formalize_request(Method, BasePath, {Path, Headers, _Body}) when Method =:= get; Method =:= delete -> @@ -636,7 +614,10 @@ to_bin(Str) when is_list(Str) -> to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). -reply_delegator(ReplyFunAndArgs, Result) -> +reply_delegator(Context, ReplyFunAndArgs, Result) -> + spawn(fun() -> maybe_retry(Result, Context, ReplyFunAndArgs) end). + +transform_result(Result) -> case Result of %% The normal reason happens when the HTTP connection times out before %% the request has been fully processed @@ -647,16 +628,47 @@ reply_delegator(ReplyFunAndArgs, Result) -> Reason =:= {shutdown, normal}; Reason =:= {shutdown, closed} -> - Result1 = {error, {recoverable_error, Reason}}, - emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result1); + {error, {recoverable_error, Reason}}; {error, {closed, _Message} = Reason} -> %% _Message = "The connection was lost." - Result1 = {error, {recoverable_error, Reason}}, - emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result1); + {error, {recoverable_error, Reason}}; _ -> - emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result) + Result end. +maybe_retry(Result0, _Context = #{attempt := N, max_attempts := Max}, ReplyFunAndArgs) when + N >= Max +-> + Result = transform_result(Result0), + emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result); +maybe_retry({error, Reason}, Context, ReplyFunAndArgs) -> + #{ + state := State, + attempt := Attempt, + key_or_num := KeyOrNum, + method := Method, + request := Request, + timeout := Timeout + } = Context, + %% TODO: reset the expiration time for free retries? + IsFreeRetry = Reason =:= normal orelse Reason =:= {shutdown, normal}, + NContext = + case IsFreeRetry of + true -> Context; + false -> Context#{attempt := Attempt + 1} + end, + Worker = resolve_pool_worker(State, KeyOrNum), + ok = ehttpc:request_async( + Worker, + Method, + Request, + Timeout, + {fun ?MODULE:reply_delegator/3, [NContext, ReplyFunAndArgs]} + ), + ok; +maybe_retry(Result, _Context, ReplyFunAndArgs) -> + emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result). + %% The HOCON schema system may generate sensitive keys with this format is_sensitive_key([{str, StringKey}]) -> is_sensitive_key(StringKey); diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index a3d2b4e75..f23752653 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -29,6 +29,10 @@ -compile(nowarn_export_all). -compile(export_all). +-type url() :: emqx_http_lib:uri_map(). +-reflect_type([url/0]). +-typerefl_from_string({url/0, emqx_http_lib, uri_parse}). + all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> @@ -314,7 +318,7 @@ t_sub_fields(_Config) -> ok. t_complicated_type(_Config) -> - Path = "/ref/complicated_type", + Path = "/ref/complex_type", Object = #{ <<"content">> => #{ <<"application/json">> => @@ -633,14 +637,14 @@ schema("/error") -> } } }; -schema("/ref/complicated_type") -> +schema("/ref/complex_type") -> #{ operationId => test, post => #{ responses => #{ 200 => [ {no_neg_integer, hoconsc:mk(non_neg_integer(), #{})}, - {url, hoconsc:mk(emqx_connector_http:url(), #{})}, + {url, hoconsc:mk(url(), #{})}, {server, hoconsc:mk(emqx_schema:ip_port(), #{})}, {connect_timeout, hoconsc:mk(emqx_connector_http:connect_timeout(), #{})}, {pool_type, hoconsc:mk(emqx_connector_http:pool_type(), #{})}, diff --git a/apps/emqx_statsd/.gitignore b/apps/emqx_enterprise/.gitignore similarity index 100% rename from apps/emqx_statsd/.gitignore rename to apps/emqx_enterprise/.gitignore diff --git a/apps/emqx_enterprise/BSL.txt b/apps/emqx_enterprise/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_enterprise/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_enterprise/README.md b/apps/emqx_enterprise/README.md new file mode 100644 index 000000000..06f249581 --- /dev/null +++ b/apps/emqx_enterprise/README.md @@ -0,0 +1,6 @@ +# EMQX Enterprise Application + +This application so fart only holds EMQX config schema for enterprise edition. +In the future this application will collect more responsibilities in managing +enterprise edition specific features. + diff --git a/apps/emqx_enterprise/etc/emqx-enterprise.conf b/apps/emqx_enterprise/etc/emqx-enterprise.conf new file mode 100644 index 000000000..e69de29bb diff --git a/lib-ee/emqx_ee_conf/rebar.config b/apps/emqx_enterprise/rebar.config similarity index 100% rename from lib-ee/emqx_ee_conf/rebar.config rename to apps/emqx_enterprise/rebar.config diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src b/apps/emqx_enterprise/src/emqx_enterprise.app.src similarity index 53% rename from lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src rename to apps/emqx_enterprise/src/emqx_enterprise.app.src index 599b0798c..84c3ffc02 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src +++ b/apps/emqx_enterprise/src/emqx_enterprise.app.src @@ -1,6 +1,6 @@ -{application, emqx_ee_conf, [ - {description, "EMQX Enterprise Edition configuration schema"}, - {vsn, "0.1.3"}, +{application, emqx_enterprise, [ + {description, "EMQX Enterprise Edition"}, + {vsn, "0.1.0"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl similarity index 62% rename from lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl rename to apps/emqx_enterprise/src/emqx_enterprise_schema.erl index f4a0b3a28..d128aa844 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl +++ b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_conf_schema). +-module(emqx_enterprise_schema). -behaviour(hocon_schema). @@ -18,8 +18,10 @@ namespace() -> emqx_conf_schema:namespace(). roots() -> - emqx_conf_schema:roots() ++ ee_roots(). + redefine_roots(emqx_conf_schema:roots()) ++ ee_roots(). +fields("node") -> + redefine_node(emqx_conf_schema:fields("node")); fields(Name) -> ee_delegate(fields, ?EE_SCHEMA_MODULES, Name). @@ -56,3 +58,26 @@ ee_delegate(Method, [EEMod | EEMods], Name) -> end; ee_delegate(Method, [], Name) -> apply(emqx_conf_schema, Method, [Name]). + +redefine_roots(Roots) -> + Overrides = [{"node", #{type => hoconsc:ref(?MODULE, "node")}}], + override(Roots, Overrides). + +redefine_node(Fields) -> + Overrides = [{"applications", #{default => <<"emqx_license">>}}], + override(Fields, Overrides). + +override(Fields, []) -> + Fields; +override(Fields, [{Name, Override} | More]) -> + Schema = find_schema(Name, Fields), + NewSchema = hocon_schema:override(Schema, Override), + NewFields = replace_schema(Name, NewSchema, Fields), + override(NewFields, More). + +find_schema(Name, Fields) -> + {Name, Schema} = lists:keyfind(Name, 1, Fields), + Schema. + +replace_schema(Name, Schema, Fields) -> + lists:keyreplace(Name, 1, Fields, {Name, Schema}). diff --git a/apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl b/apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl new file mode 100644 index 000000000..e7f4c0075 --- /dev/null +++ b/apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl @@ -0,0 +1,52 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_enterprise_schema_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +%%------------------------------------------------------------------------------ +%% Tests +%%------------------------------------------------------------------------------ + +t_namespace(_Config) -> + ?assertEqual( + emqx_conf_schema:namespace(), + emqx_enterprise_schema:namespace() + ). + +t_roots(_Config) -> + EnterpriseRoots = emqx_enterprise_schema:roots(), + ?assertMatch({license, _}, lists:keyfind(license, 1, EnterpriseRoots)). + +t_fields(_Config) -> + CeFields = emqx_conf_schema:fields("node"), + EeFields = emqx_enterprise_schema:fields("node"), + ?assertEqual(length(CeFields), length(EeFields)), + lists:foreach( + fun({{CeName, CeSchema}, {EeName, EeSchema}}) -> + ?assertEqual(CeName, EeName), + case CeName of + "applications" -> + ok; + _ -> + ?assertEqual({CeName, CeSchema}, {EeName, EeSchema}) + end + end, + lists:zip(CeFields, EeFields) + ). + +t_translations(_Config) -> + [Root | _] = emqx_enterprise_schema:translations(), + ?assertEqual( + emqx_conf_schema:translation(Root), + emqx_enterprise_schema:translation(Root) + ). diff --git a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl b/apps/emqx_enterprise/test/emqx_enterprise_schema_tests.erl similarity index 85% rename from lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl rename to apps/emqx_enterprise/test/emqx_enterprise_schema_tests.erl index 5e1d4e551..a78bbcb2e 100644 --- a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl +++ b/apps/emqx_enterprise/test/emqx_enterprise_schema_tests.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_conf_schema_tests). +-module(emqx_enterprise_schema_tests). -include_lib("eunit/include/eunit.hrl"). @@ -22,7 +22,7 @@ doc_gen_test() -> "priv", "i18n.conf" ]), - _ = emqx_conf:dump_schema(Dir, emqx_ee_conf_schema, I18nFile), + _ = emqx_conf:dump_schema(Dir, emqx_enterprise_schema, I18nFile), ok end }. diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl index 0f1b948c7..883407a94 100644 --- a/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl +++ b/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl @@ -159,8 +159,6 @@ t_explicit_session_takeover(Config) -> ]), {ok, _, _} = emqtt:subscribe(C0, <<"t1">>), - ok = rpc:call(Node1, emqx_eviction_agent, enable, [test_eviction, undefined]), - ?assertEqual( 1, rpc:call(Node1, emqx_eviction_agent, connection_count, []) @@ -168,6 +166,8 @@ t_explicit_session_takeover(Config) -> [ChanPid] = rpc:call(Node1, emqx_cm, lookup_channels, [<<"client_with_session">>]), + ok = rpc:call(Node1, emqx_eviction_agent, enable, [test_eviction, undefined]), + ?assertWaitEvent( begin ok = rpc:call(Node1, emqx_eviction_agent, evict_connections, [1]), diff --git a/apps/emqx_ft/src/emqx_ft.app.src b/apps/emqx_ft/src/emqx_ft.app.src index 058fe984a..c503ab8e0 100644 --- a/apps/emqx_ft/src/emqx_ft.app.src +++ b/apps/emqx_ft/src/emqx_ft.app.src @@ -1,6 +1,6 @@ {application, emqx_ft, [ {description, "EMQX file transfer over MQTT"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {mod, {emqx_ft_app, []}}, {applications, [ diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index b9f07d5c0..4db2255f6 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -79,7 +79,7 @@ start_export(_Options, Transfer, Filemeta) -> -spec write(export_st(), iodata()) -> {ok, export_st()} | {error, term()}. write(#{pid := Pid} = ExportSt, IoData) -> - case emqx_s3_uploader:write(Pid, IoData) of + case emqx_s3_uploader:write(Pid, IoData, emqx_ft_conf:store_segment_timeout()) of ok -> {ok, ExportSt}; {error, _Reason} = Error -> @@ -89,12 +89,13 @@ write(#{pid := Pid} = ExportSt, IoData) -> -spec complete(export_st(), emqx_ft:checksum()) -> ok | {error, term()}. complete(#{pid := Pid} = _ExportSt, _Checksum) -> - emqx_s3_uploader:complete(Pid). + emqx_s3_uploader:complete(Pid, emqx_ft_conf:assemble_timeout()). -spec discard(export_st()) -> ok. discard(#{pid := Pid} = _ExportSt) -> - emqx_s3_uploader:abort(Pid). + % NOTE: will abort upload asynchronously if needed + emqx_s3_uploader:shutdown(Pid). -spec list(options(), query()) -> {ok, page(exportinfo())} | {error, term()}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 010d004a1..7a0a6b3b4 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -267,8 +267,12 @@ lookup_assembler([Source | Sources]) -> check_if_already_exported(Storage, Transfer) -> case files(Storage, #{transfer => Transfer}) of - {ok, #{items := [_ | _]}} -> ok; - _ -> {error, not_found} + {ok, #{items := [_ | _]}} -> + % NOTE: we don't know coverage here, let's just clean up locally. + _ = emqx_ft_storage_fs_gc:collect(Storage, Transfer, [node()]), + ok; + _ -> + {error, not_found} end. lookup_local_assembler(Transfer) -> diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index a7ffd5675..842ae6bad 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -266,6 +266,38 @@ t_gc_incomplete_transfers(_Config) -> 0 ). +t_gc_repeated_transfer(_Config) -> + {local, Storage} = emqx_ft_storage:backend(), + Transfer = { + TID = {<<"clientclient">>, mk_file_id()}, + #{name => "repeat.please", segments_ttl => 10}, + emqx_ft_content_gen:new({?LINE, Size = 42}, 16) + }, + Size = start_transfer(Storage, Transfer), + {ok, {ok, #{stats := Stats1}}} = ?wait_async_action( + ?assertEqual(ok, complete_transfer(Storage, TID, Size)), + #{?snk_kind := garbage_collection}, + 1000 + ), + Size = start_transfer(Storage, Transfer), + {ok, {ok, #{stats := Stats2}}} = ?wait_async_action( + ?assertEqual(ok, complete_transfer(Storage, TID, Size)), + #{?snk_kind := garbage_collection}, + 1000 + ), + ?assertMatch( + #gcstats{files = 4, directories = 2}, + Stats1 + ), + ?assertMatch( + #gcstats{files = 4, directories = 2}, + Stats2 + ), + ?assertEqual( + {ok, []}, + emqx_ft_storage_fs:list(Storage, TID, fragment) + ). + t_gc_handling_errors(_Config) -> ok = set_gc_config(minimum_segments_ttl, 0), ok = set_gc_config(maximum_segments_ttl, 0), @@ -349,14 +381,18 @@ complete_transfer(Storage, Transfer, Size) -> complete_transfer(Storage, Transfer, Size, 100). complete_transfer(Storage, Transfer, Size, Timeout) -> - {async, Pid} = emqx_ft_storage_fs:assemble(Storage, Transfer, Size), - MRef = erlang:monitor(process, Pid), - Pid ! kickoff, - receive - {'DOWN', MRef, process, Pid, {shutdown, Result}} -> - Result - after Timeout -> - ct:fail("Assembler did not finish in time") + case emqx_ft_storage_fs:assemble(Storage, Transfer, Size) of + ok -> + ok; + {async, Pid} -> + MRef = erlang:monitor(process, Pid), + Pid ! kickoff, + receive + {'DOWN', MRef, process, Pid, {shutdown, Result}} -> + Result + after Timeout -> + ct:fail("Assembler did not finish in time") + end end. mk_file_id() -> diff --git a/apps/emqx_gateway/README.md b/apps/emqx_gateway/README.md index ebab3a7a9..c71737639 100644 --- a/apps/emqx_gateway/README.md +++ b/apps/emqx_gateway/README.md @@ -19,8 +19,8 @@ More introduction: [Extended Protocol Gateway](https://www.emqx.io/docs/en/v5.0/ ## Usage This application is just a Framework, we provide some standard implementations, -such as [Stomp](../emqx_stomp/README.md), [MQTT-SN](../emqx_mqttsn/README.md), -[CoAP](../emqx_coap/README.md) and [LwM2M](../emqx_lwm2m/README.md) gateway. +such as [Stomp](../emqx_gateway_stomp/README.md), [MQTT-SN](../emqx_gateway_mqttsn/README.md), +[CoAP](../emqx_gateway_coap/README.md) and [LwM2M](../emqx_gateway_lwm2m/README.md) gateway. These applications are all packaged by default in the EMQX distribution. If you need to start a certain gateway, you only need to enable it via diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index 26dab4c9c..487e71a06 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.17"}, + {vsn, "0.1.18"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, emqx, emqx_authn, emqx_ctl]}, diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index 7f8f30c8b..e052647fe 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -147,10 +147,9 @@ clients(get, #{ clients_insta(get, #{ bindings := #{ name := Name0, - clientid := ClientId0 + clientid := ClientId } }) -> - ClientId = emqx_mgmt_util:urldecode(ClientId0), with_gateway(Name0, fun(GwName, _) -> case emqx_gateway_http:lookup_client( @@ -174,10 +173,9 @@ clients_insta(get, #{ clients_insta(delete, #{ bindings := #{ name := Name0, - clientid := ClientId0 + clientid := ClientId } }) -> - ClientId = emqx_mgmt_util:urldecode(ClientId0), with_gateway(Name0, fun(GwName, _) -> _ = emqx_gateway_http:kickout_client(GwName, ClientId), {204} @@ -187,10 +185,9 @@ clients_insta(delete, #{ subscriptions(get, #{ bindings := #{ name := Name0, - clientid := ClientId0 + clientid := ClientId } }) -> - ClientId = emqx_mgmt_util:urldecode(ClientId0), with_gateway(Name0, fun(GwName, _) -> case emqx_gateway_http:list_client_subscriptions(GwName, ClientId) of {error, not_found} -> @@ -205,11 +202,10 @@ subscriptions(get, #{ subscriptions(post, #{ bindings := #{ name := Name0, - clientid := ClientId0 + clientid := ClientId }, body := Body }) -> - ClientId = emqx_mgmt_util:urldecode(ClientId0), with_gateway(Name0, fun(GwName, _) -> case {maps:get(<<"topic">>, Body, undefined), subopts(Body)} of {undefined, _} -> @@ -233,12 +229,10 @@ subscriptions(post, #{ subscriptions(delete, #{ bindings := #{ name := Name0, - clientid := ClientId0, - topic := Topic0 + clientid := ClientId, + topic := Topic } }) -> - ClientId = emqx_mgmt_util:urldecode(ClientId0), - Topic = emqx_mgmt_util:urldecode(Topic0), with_gateway(Name0, fun(GwName, _) -> _ = emqx_gateway_http:client_unsubscribe(GwName, ClientId, Topic), {204} diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index d90bf3689..2a6d59e35 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -110,14 +110,12 @@ listeners(post, #{bindings := #{name := Name0}, body := LConf}) -> end end). -listeners_insta(delete, #{bindings := #{name := Name0, id := ListenerId0}}) -> - ListenerId = emqx_mgmt_util:urldecode(ListenerId0), +listeners_insta(delete, #{bindings := #{name := Name0, id := ListenerId}}) -> with_gateway(Name0, fun(_GwName, _) -> ok = emqx_gateway_http:remove_listener(ListenerId), {204} end); -listeners_insta(get, #{bindings := #{name := Name0, id := ListenerId0}}) -> - ListenerId = emqx_mgmt_util:urldecode(ListenerId0), +listeners_insta(get, #{bindings := #{name := Name0, id := ListenerId}}) -> with_gateway(Name0, fun(_GwName, _) -> case emqx_gateway_conf:listener(ListenerId) of {ok, Listener} -> @@ -130,9 +128,8 @@ listeners_insta(get, #{bindings := #{name := Name0, id := ListenerId0}}) -> end); listeners_insta(put, #{ body := LConf, - bindings := #{name := Name0, id := ListenerId0} + bindings := #{name := Name0, id := ListenerId} }) -> - ListenerId = emqx_mgmt_util:urldecode(ListenerId0), with_gateway(Name0, fun(_GwName, _) -> {ok, RespConf} = emqx_gateway_http:update_listener(ListenerId, LConf), {200, RespConf} @@ -141,10 +138,9 @@ listeners_insta(put, #{ listeners_insta_authn(get, #{ bindings := #{ name := Name0, - id := ListenerId0 + id := ListenerId } }) -> - ListenerId = emqx_mgmt_util:urldecode(ListenerId0), with_gateway(Name0, fun(GwName, _) -> try emqx_gateway_http:authn(GwName, ListenerId) of Authn -> {200, Authn} @@ -157,10 +153,9 @@ listeners_insta_authn(post, #{ body := Conf, bindings := #{ name := Name0, - id := ListenerId0 + id := ListenerId } }) -> - ListenerId = emqx_mgmt_util:urldecode(ListenerId0), with_gateway(Name0, fun(GwName, _) -> {ok, Authn} = emqx_gateway_http:add_authn(GwName, ListenerId, Conf), {201, Authn} @@ -169,10 +164,9 @@ listeners_insta_authn(put, #{ body := Conf, bindings := #{ name := Name0, - id := ListenerId0 + id := ListenerId } }) -> - ListenerId = emqx_mgmt_util:urldecode(ListenerId0), with_gateway(Name0, fun(GwName, _) -> {ok, Authn} = emqx_gateway_http:update_authn( GwName, ListenerId, Conf @@ -182,10 +176,9 @@ listeners_insta_authn(put, #{ listeners_insta_authn(delete, #{ bindings := #{ name := Name0, - id := ListenerId0 + id := ListenerId } }) -> - ListenerId = emqx_mgmt_util:urldecode(ListenerId0), with_gateway(Name0, fun(GwName, _) -> ok = emqx_gateway_http:remove_authn(GwName, ListenerId), {204} diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl index 56a3e2068..da86d6a58 100644 --- a/apps/emqx_gateway/src/emqx_gateway_conf.erl +++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl @@ -448,10 +448,12 @@ pre_config_update(_, {add_authn, GwName, Conf}, RawConf) -> ) of undefined -> + CertsDir = authn_certs_dir(GwName, Conf), + Conf1 = emqx_authentication_config:convert_certs(CertsDir, Conf), {ok, emqx_utils_maps:deep_merge( RawConf, - #{GwName => #{?AUTHN_BIN => Conf}} + #{GwName => #{?AUTHN_BIN => Conf1}} )}; _ -> badres_authn(already_exist, GwName) @@ -469,7 +471,9 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) -> Listener -> case maps:get(?AUTHN_BIN, Listener, undefined) of undefined -> - NListener = maps:put(?AUTHN_BIN, Conf, Listener), + CertsDir = authn_certs_dir(GwName, LType, LName, Conf), + Conf1 = emqx_authentication_config:convert_certs(CertsDir, Conf), + NListener = maps:put(?AUTHN_BIN, Conf1, Listener), NGateway = #{ GwName => #{ @@ -490,8 +494,10 @@ pre_config_update(_, {update_authn, GwName, Conf}, RawConf) -> of undefined -> badres_authn(not_found, GwName); - _Authn -> - {ok, emqx_utils_maps:deep_put([GwName, ?AUTHN_BIN], RawConf, Conf)} + OldAuthnConf -> + CertsDir = authn_certs_dir(GwName, Conf), + Conf1 = emqx_authentication_config:convert_certs(CertsDir, Conf, OldAuthnConf), + {ok, emqx_utils_maps:deep_put([GwName, ?AUTHN_BIN], RawConf, Conf1)} end; pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> case @@ -507,10 +513,16 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> case maps:get(?AUTHN_BIN, Listener, undefined) of undefined -> badres_listener_authn(not_found, GwName, LType, LName); - _Auth -> + OldAuthnConf -> + CertsDir = authn_certs_dir(GwName, LType, LName, OldAuthnConf), + Conf1 = emqx_authentication_config:convert_certs( + CertsDir, + Conf, + OldAuthnConf + ), NListener = maps:put( ?AUTHN_BIN, - Conf, + Conf1, Listener ), {ok, @@ -522,12 +534,36 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> end end; pre_config_update(_, {remove_authn, GwName}, RawConf) -> + case + emqx_utils_maps:deep_get( + [GwName, ?AUTHN_BIN], RawConf, undefined + ) + of + undefined -> + ok; + OldAuthnConf -> + CertsDir = authn_certs_dir(GwName, OldAuthnConf), + emqx_authentication_config:clear_certs(CertsDir, OldAuthnConf) + end, {ok, emqx_utils_maps:deep_remove( [GwName, ?AUTHN_BIN], RawConf )}; pre_config_update(_, {remove_authn, GwName, {LType, LName}}, RawConf) -> Path = [GwName, <<"listeners">>, LType, LName, ?AUTHN_BIN], + case + emqx_utils_maps:deep_get( + Path, + RawConf, + undefined + ) + of + undefined -> + ok; + OldAuthnConf -> + CertsDir = authn_certs_dir(GwName, LType, LName, OldAuthnConf), + emqx_authentication_config:clear_certs(CertsDir, OldAuthnConf) + end, {ok, emqx_utils_maps:deep_remove(Path, RawConf)}; pre_config_update(_, UnknownReq, _RawConf) -> logger:error("Unknown configuration update request: ~0p", [UnknownReq]), @@ -678,6 +714,18 @@ apply_to_gateway_basic_confs(_Fun, _GwName, Conf) -> certs_dir(GwName) when is_binary(GwName) -> GwName. +authn_certs_dir(GwName, ListenerType, ListenerName, AuthnConf) -> + ChainName = emqx_gateway_utils:listener_chain(GwName, ListenerType, ListenerName), + emqx_authentication_config:certs_dir(ChainName, AuthnConf). + +authn_certs_dir(GwName, AuthnConf) when is_binary(GwName) -> + authn_certs_dir(binary_to_existing_atom(GwName), AuthnConf); +authn_certs_dir(GwName, AuthnConf) -> + emqx_authentication_config:certs_dir( + emqx_gateway_utils:global_chain(GwName), + AuthnConf + ). + convert_certs(SubDir, Conf) -> convert_certs(<<"dtls_options">>, SubDir, convert_certs(<<"ssl_options">>, SubDir, Conf)). diff --git a/apps/emqx_gateway_exproto/README.md b/apps/emqx_gateway_exproto/README.md index a2268f0dc..ec8b93984 100644 --- a/apps/emqx_gateway_exproto/README.md +++ b/apps/emqx_gateway_exproto/README.md @@ -7,10 +7,6 @@ The `emqx_exproto` extremely enhance the extensibility for EMQX. It allow using - [x] Based on gRPC, it brings a very wide range of applicability - [x] Allows you to use the return value to extend emqx behavior. -## Architecture - -![EMQX ExProto Arch](./docs/images/exproto-arch.jpg) - ## Usage ### gRPC service diff --git a/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src index dd48b2723..76f0f45b5 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src +++ b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_mqttsn, [ {description, "MQTT-SN Gateway"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl index 1ccc8b95a..914f837e1 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl @@ -1791,14 +1791,14 @@ message_to_packet( handle_call({subscribe, Topic, SubOpts}, _From, Channel) -> case do_subscribe({?SN_INVALID_TOPIC_ID, Topic, SubOpts}, Channel) of {ok, {_, NTopicName, NSubOpts}, NChannel} -> - reply({ok, {NTopicName, NSubOpts}}, NChannel); + reply_and_update({ok, {NTopicName, NSubOpts}}, NChannel); {error, ?SN_RC2_EXCEED_LIMITATION} -> reply({error, exceed_limitation}, Channel) end; handle_call({unsubscribe, Topic}, _From, Channel) -> TopicFilters = [emqx_topic:parse(Topic)], {ok, _, NChannel} = do_unsubscribe(TopicFilters, Channel), - reply(ok, NChannel); + reply_and_update(ok, NChannel); handle_call(subscriptions, _From, Channel = #channel{session = Session}) -> reply({ok, maps:to_list(emqx_session:info(subscriptions, Session))}, Channel); handle_call(kick, _From, Channel) -> @@ -2045,15 +2045,15 @@ handle_deliver( ignore_local(Delivers, Subscriber, Session, Ctx) -> Subs = emqx_session:info(subscriptions, Session), - lists:dropwhile( + lists:filter( fun({deliver, Topic, #message{from = Publisher}}) -> case maps:find(Topic, Subs) of {ok, #{nl := 1}} when Subscriber =:= Publisher -> ok = metrics_inc(Ctx, 'delivery.dropped'), ok = metrics_inc(Ctx, 'delivery.dropped.no_local'), - true; + false; _ -> - false + true end end, Delivers @@ -2192,6 +2192,9 @@ terminate(_Reason, _Channel) -> reply(Reply, Channel) -> {reply, Reply, Channel}. +reply_and_update(Reply, Channel) -> + {reply, Reply, [{event, updated}], Channel}. + shutdown(Reason, Channel) -> {shutdown, Reason, Channel}. diff --git a/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl index 04b1b5fb2..cce4ce904 100644 --- a/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl @@ -2259,6 +2259,46 @@ t_clients_subscription_api(_) -> ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), gen_udp:close(Socket). +t_clients_api_complex_id(_) -> + ClientId = <<"!@#$%^&*()_+{}:\"<>?/">>, + ClientIdUriEncoded = cow_qs:urlencode(ClientId), + Path = "/gateways/mqttsn/clients/" ++ binary_to_list(ClientIdUriEncoded), + {ok, Socket} = gen_udp:open(0, [binary]), + send_connect_msg(Socket, ClientId), + ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + %% get + {200, Client} = request(get, Path), + ?assertMatch(#{clientid := ClientId}, Client), + %% subscription list + {200, []} = request(get, Path ++ "/subscriptions"), + %% kickout + {204, _} = request(delete, Path), + gen_udp:close(Socket). + +t_update_info_after_subscribed_via_api(_) -> + ClientId = <<"client_id_test1">>, + Path = "/gateways/mqttsn/clients/client_id_test1/subscriptions", + {ok, Socket} = gen_udp:open(0, [binary]), + send_connect_msg(Socket, ClientId), + ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + %% create + SubReq = #{ + topic => <<"tx">>, + qos => 1, + nl => 0, + rap => 0, + rh => 0 + }, + {201, _SubsResp} = request(post, Path, SubReq), + timer:sleep(500), + %% assert + {200, Client} = request(get, "/gateways/mqttsn/clients/client_id_test1"), + ?assertMatch(#{subscriptions_cnt := 1}, Client), + + send_disconnect_msg(Socket, undefined), + ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), + gen_udp:close(Socket). + %%-------------------------------------------------------------------- %% Helper funcs %%-------------------------------------------------------------------- diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 7cf0e4b53..5cfa80369 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.4"}, + {vsn, "0.2.5"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index 266855877..2ac0f2e9b 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -134,7 +134,6 @@ basic_reboot_apps() -> emqx_dashboard, emqx_connector, emqx_gateway, - emqx_statsd, emqx_resource, emqx_rule_engine, emqx_bridge, @@ -150,7 +149,7 @@ basic_reboot_apps() -> ], case emqx_release:edition() of ce -> - CE; + CE ++ [emqx_telemetry]; ee -> CE ++ [ diff --git a/apps/emqx_machine/test/emqx_machine_SUITE.erl b/apps/emqx_machine/test/emqx_machine_SUITE.erl index 02d03d983..d8613a537 100644 --- a/apps/emqx_machine/test/emqx_machine_SUITE.erl +++ b/apps/emqx_machine/test/emqx_machine_SUITE.erl @@ -48,7 +48,6 @@ init_per_suite(Config) -> emqx_modules, emqx_dashboard, emqx_gateway, - emqx_statsd, emqx_resource, emqx_rule_engine, emqx_bridge, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index f51a83923..31c719a33 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.22"}, + {vsn, "5.0.23"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 0b91817f0..9553730ec 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -151,7 +151,7 @@ log_path() -> Configs = logger:get_handler_config(), case get_log_path(Configs) of undefined -> - <<"log.file_handler.default.enable is false, not logging to file.">>; + <<"log.file.enable is false, not logging to file.">>; Path -> iolist_to_binary(filename:join(RootDir, Path)) end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 27dc8c492..7720ab66f 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -547,7 +547,8 @@ fields(authz_cache) -> ]; fields(keepalive) -> [ - {interval, hoconsc:mk(integer(), #{desc => <<"Keepalive time, with the unit of second">>})} + {interval, + hoconsc:mk(range(0, 65535), #{desc => <<"Keepalive time, with the unit of second">>})} ]; fields(subscribe) -> [ diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 14fb07497..71e009589 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -132,7 +132,7 @@ schema("/configs/global_zone") -> }, put => #{ tags => ?TAGS, - description => ?DESC(update_globar_zone_configs), + description => ?DESC(update_global_zone_configs), 'requestBody' => Schema, responses => #{ 200 => Schema, @@ -146,7 +146,7 @@ schema("/configs/limiter") -> 'operationId' => limiter, get => #{ tags => ?TAGS, - description => ?DESC(get_node_level_limiter_congigs), + description => ?DESC(get_node_level_limiter_configs), responses => #{ 200 => hoconsc:mk(hoconsc:ref(emqx_limiter_schema, limiter)), 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"config not found">>) @@ -154,7 +154,7 @@ schema("/configs/limiter") -> }, put => #{ tags => ?TAGS, - description => ?DESC(update_node_level_limiter_congigs), + description => ?DESC(update_node_level_limiter_configs), 'requestBody' => hoconsc:mk(hoconsc:ref(emqx_limiter_schema, limiter)), responses => #{ 200 => hoconsc:mk(hoconsc:ref(emqx_limiter_schema, limiter)), @@ -221,8 +221,6 @@ config(put, #{body := NewConf}, Req) -> case emqx_conf:update(Path, NewConf, ?OPTS) of {ok, #{raw_config := RawConf}} -> {200, RawConf}; - {error, {permission_denied, Reason}} -> - {403, #{code => 'UPDATE_FAILED', message => Reason}}; {error, Reason} -> {400, #{code => 'UPDATE_FAILED', message => ?ERR_MSG(Reason)}} end. @@ -267,8 +265,6 @@ config_reset(post, _Params, Req) -> case emqx_conf:reset(Path, ?OPTS) of {ok, _} -> {200}; - {error, {permission_denied, Reason}} -> - {403, #{code => 'REST_FAILED', message => Reason}}; {error, no_default_value} -> {400, #{code => 'NO_DEFAULT_VALUE', message => <<"No Default Value.">>}}; {error, Reason} -> @@ -360,4 +356,4 @@ global_zone_roots() -> lists:map(fun({K, _}) -> list_to_binary(K) end, global_zone_schema()). global_zone_schema() -> - emqx_zone_schema:zone_without_hidden(). + emqx_zone_schema:global_zone_with_default(). diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 448940904..27de11d0f 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -457,7 +457,7 @@ trace(["list"]) -> lists:foreach( fun(Trace) -> #{type := Type, filter := Filter, level := Level, dst := Dst} = Trace, - emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Type, Filter, Level, Dst]) + emqx_ctl:print("Trace(~s=~s, level=~s, destination=~0p)~n", [Type, Filter, Level, Dst]) end, emqx_trace_handler:running() ); @@ -514,6 +514,8 @@ trace_off(Type, Filter) -> %%-------------------------------------------------------------------- %% @doc Trace Cluster Command +-define(DEFAULT_TRACE_DURATION, "1800"). + traces(["list"]) -> {200, List} = emqx_mgmt_api_trace:trace(get, []), case List of @@ -529,7 +531,7 @@ traces(["list"]) -> log_size := LogSize } = Trace, emqx_ctl:print( - "Trace(~s: ~s=~s, ~s, LogSize:~p)~n", + "Trace(~s: ~s=~s, ~s, LogSize:~0p)~n", [Name, Type, maps:get(Type, Trace), Status, LogSize] ) end, @@ -542,7 +544,7 @@ traces(["stop", Name]) -> traces(["delete", Name]) -> trace_cluster_del(Name); traces(["start", Name, Operation, Filter]) -> - traces(["start", Name, Operation, Filter, "900"]); + traces(["start", Name, Operation, Filter, ?DEFAULT_TRACE_DURATION]); traces(["start", Name, Operation, Filter0, DurationS]) -> case trace_type(Operation, Filter0) of {ok, Type, Filter} -> trace_cluster_on(Name, Type, Filter, DurationS); @@ -551,22 +553,27 @@ traces(["start", Name, Operation, Filter0, DurationS]) -> traces(_) -> emqx_ctl:usage([ {"traces list", "List all cluster traces started"}, - {"traces start client ", "Traces for a client in cluster"}, - {"traces start topic ", "Traces for a topic in cluster"}, - {"traces start ip_address ", "Traces for a IP in cluster"}, - {"traces stop ", "Stop trace in cluster"}, - {"traces delete ", "Delete trace in cluster"} + {"traces start client []", "Traces for a client in cluster"}, + {"traces start topic []", "Traces for a topic in cluster"}, + {"traces start ip_address []", + "Traces for a client IP in cluster\n" + "Trace will start immediately on all nodes, including the core and replicant,\n" + "and will end after seconds. The default value for is " + ?DEFAULT_TRACE_DURATION + " seconds."}, + {"traces stop ", "Stop trace in cluster"}, + {"traces delete ", "Delete trace in cluster"} ]). trace_cluster_on(Name, Type, Filter, DurationS0) -> + Now = emqx_trace:now_second(), DurationS = list_to_integer(DurationS0), - Now = erlang:system_time(second), Trace = #{ - name => list_to_binary(Name), - type => atom_to_binary(Type), - Type => list_to_binary(Filter), - start_at => list_to_binary(calendar:system_time_to_rfc3339(Now)), - end_at => list_to_binary(calendar:system_time_to_rfc3339(Now + DurationS)) + name => bin(Name), + type => Type, + Type => bin(Filter), + start_at => Now, + end_at => Now + DurationS }, case emqx_trace:create(Trace) of {ok, _} -> @@ -579,19 +586,19 @@ trace_cluster_on(Name, Type, Filter, DurationS0) -> end. trace_cluster_del(Name) -> - case emqx_trace:delete(list_to_binary(Name)) of + case emqx_trace:delete(bin(Name)) of ok -> emqx_ctl:print("Del cluster_trace ~s successfully~n", [Name]); {error, Error} -> emqx_ctl:print("[error] Del cluster_trace ~s: ~p~n", [Name, Error]) end. trace_cluster_off(Name) -> - case emqx_trace:update(list_to_binary(Name), false) of + case emqx_trace:update(bin(Name), false) of ok -> emqx_ctl:print("Stop cluster_trace ~s successfully~n", [Name]); {error, Error} -> emqx_ctl:print("[error] Stop cluster_trace ~s: ~p~n", [Name, Error]) end. -trace_type("client", ClientId) -> {ok, clientid, list_to_binary(ClientId)}; -trace_type("topic", Topic) -> {ok, topic, list_to_binary(Topic)}; +trace_type("client", ClientId) -> {ok, clientid, bin(ClientId)}; +trace_type("topic", Topic) -> {ok, topic, bin(Topic)}; trace_type("ip_address", IP) -> {ok, ip_address, IP}; trace_type(_, _) -> error. diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index 6d7733b22..89838c346 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -244,13 +244,31 @@ t_keepalive(_Config) -> Body = #{interval => 11}, {error, {"HTTP/1.1", 404, "Not Found"}} = emqx_mgmt_api_test_util:request_api(put, Path, <<"">>, AuthHeader, Body), - {ok, C1} = emqtt:start_link(#{username => Username, clientid => ClientId}), + %% 65535 is the max value of keepalive + MaxKeepalive = 65535, + InitKeepalive = round(MaxKeepalive / 1.5 + 1), + {ok, C1} = emqtt:start_link(#{ + username => Username, clientid => ClientId, keepalive => InitKeepalive + }), {ok, _} = emqtt:connect(C1), + [Pid] = emqx_cm:lookup_channels(list_to_binary(ClientId)), + %% will reset to max keepalive if keepalive > max keepalive + #{conninfo := #{keepalive := InitKeepalive}} = emqx_connection:info(Pid), + ?assertMatch({keepalive, 65535000, _}, element(5, element(9, sys:get_state(Pid)))), + {ok, NewClient} = emqx_mgmt_api_test_util:request_api(put, Path, <<"">>, AuthHeader, Body), #{<<"keepalive">> := 11} = emqx_utils_json:decode(NewClient, [return_maps]), - [Pid] = emqx_cm:lookup_channels(list_to_binary(ClientId)), #{conninfo := #{keepalive := Keepalive}} = emqx_connection:info(Pid), ?assertEqual(11, Keepalive), + %% Disable keepalive + Body1 = #{interval => 0}, + {ok, NewClient1} = emqx_mgmt_api_test_util:request_api(put, Path, <<"">>, AuthHeader, Body1), + #{<<"keepalive">> := 0} = emqx_utils_json:decode(NewClient1, [return_maps]), + ?assertMatch(#{conninfo := #{keepalive := 0}}, emqx_connection:info(Pid)), + %% Maximal keepalive + Body2 = #{interval => 65536}, + {error, {"HTTP/1.1", 400, _}} = + emqx_mgmt_api_test_util:request_api(put, Path, <<"">>, AuthHeader, Body2), emqtt:disconnect(C1), ok. diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 5a0116a4d..2805d260d 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -99,24 +99,24 @@ t_log(_Config) -> {ok, Log} = get_config("log"), File = "log/emqx-test.log", %% update handler - Log1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), - Log2 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), + Log1 = emqx_utils_maps:deep_put([<<"file">>, <<"default">>, <<"enable">>], Log, true), + Log2 = emqx_utils_maps:deep_put([<<"file">>, <<"default">>, <<"to">>], Log1, File), {ok, #{}} = update_config(<<"log">>, Log2), {ok, Log3} = logger:get_handler_config(default), ?assertMatch(#{config := #{file := File}}, Log3), - ErrLog1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, 1), + ErrLog1 = emqx_utils_maps:deep_put([<<"file">>, <<"default">>, <<"enable">>], Log, 1), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_config(<<"log">>, ErrLog1)), ErrLog2 = emqx_utils_maps:deep_put( - [<<"file_handlers">>, <<"default">>, <<"enabfe">>], Log, true + [<<"file">>, <<"default">>, <<"enabfe">>], Log, true ), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_config(<<"log">>, ErrLog2)), %% add new handler File1 = "log/emqx-test1.log", - Handler = emqx_utils_maps:deep_get([<<"file_handlers">>, <<"default">>], Log2), - NewLog1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"new">>], Log2, Handler), + Handler = emqx_utils_maps:deep_get([<<"file">>, <<"default">>], Log2), + NewLog1 = emqx_utils_maps:deep_put([<<"file">>, <<"new">>], Log2, Handler), NewLog2 = emqx_utils_maps:deep_put( - [<<"file_handlers">>, <<"new">>, <<"file">>], NewLog1, File1 + [<<"file">>, <<"new">>, <<"to">>], NewLog1, File1 ), {ok, #{}} = update_config(<<"log">>, NewLog2), {ok, Log4} = logger:get_handler_config(new), @@ -124,7 +124,7 @@ t_log(_Config) -> %% disable new handler Disable = emqx_utils_maps:deep_put( - [<<"file_handlers">>, <<"new">>, <<"enable">>], NewLog2, false + [<<"file">>, <<"new">>, <<"enable">>], NewLog2, false ), {ok, #{}} = update_config(<<"log">>, Disable), ?assertEqual({error, {not_found, new}}, logger:get_handler_config(new)), @@ -133,15 +133,24 @@ t_log(_Config) -> t_global_zone(_Config) -> {ok, Zones} = get_global_zone(), ZonesKeys = lists:map( - fun({K, _}) -> list_to_binary(K) end, emqx_zone_schema:zone_without_hidden() + fun({K, _}) -> list_to_binary(K) end, emqx_zone_schema:global_zone_with_default() ), ?assertEqual(lists:usort(ZonesKeys), lists:usort(maps:keys(Zones))), ?assertEqual( emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed]), emqx_utils_maps:deep_get([<<"mqtt">>, <<"max_qos_allowed">>], Zones) ), - NewZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1), - {ok, #{}} = update_global_zone(NewZones), + NewZones1 = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1), + NewZones2 = emqx_utils_maps:deep_remove([<<"mqtt">>, <<"peer_cert_as_clientid">>], NewZones1), + {ok, #{<<"mqtt">> := Res}} = update_global_zone(NewZones2), + %% Make sure peer_cert_as_clientid is not removed(fill default) + ?assertMatch( + #{ + <<"max_qos_allowed">> := 1, + <<"peer_cert_as_clientid">> := <<"disabled">> + }, + Res + ), ?assertEqual(1, emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed])), %% Make sure the override config is updated, and remove the default value. ?assertMatch(#{<<"max_qos_allowed">> := 1}, read_conf(<<"mqtt">>)), @@ -184,9 +193,11 @@ update_global_zone(Change) -> t_zones(_Config) -> {ok, Zones} = get_config("zones"), {ok, #{<<"mqtt">> := OldMqtt} = Zone1} = get_global_zone(), - {ok, #{}} = update_config("zones", Zones#{<<"new_zone">> => Zone1}), + Mqtt1 = maps:remove(<<"max_subscriptions">>, OldMqtt), + {ok, #{}} = update_config("zones", Zones#{<<"new_zone">> => Zone1#{<<"mqtt">> => Mqtt1}}), NewMqtt = emqx_config:get_raw([zones, new_zone, mqtt]), - ?assertEqual(OldMqtt, NewMqtt), + %% we remove max_subscription from global zone, so the new zone should not have it. + ?assertEqual(Mqtt1, NewMqtt), %% delete the new zones {ok, #{}} = update_config("zones", Zones), ?assertEqual(undefined, emqx_config:get_raw([new_zone, mqtt], undefined)), diff --git a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl index 1f14d075e..b356bf905 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl @@ -34,8 +34,8 @@ init_per_testcase(t_log_path, Config) -> emqx_config_logger:add_handler(), Log = emqx_conf:get_raw([log], #{}), File = "log/emqx-test.log", - Log1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), - Log2 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), + Log1 = emqx_utils_maps:deep_put([<<"file">>, <<"default">>, <<"enable">>], Log, true), + Log2 = emqx_utils_maps:deep_put([<<"file">>, <<"default">>, <<"to">>], Log1, File), {ok, #{}} = emqx_conf:update([log], Log2, #{rawconf_with_defaults => true}), Config; init_per_testcase(_, Config) -> @@ -43,7 +43,7 @@ init_per_testcase(_, Config) -> end_per_testcase(t_log_path, Config) -> Log = emqx_conf:get_raw([log], #{}), - Log1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, false), + Log1 = emqx_utils_maps:deep_put([<<"file">>, <<"default">>, <<"enable">>], Log, false), {ok, #{}} = emqx_conf:update([log], Log1, #{rawconf_with_defaults => true}), emqx_config_logger:remove_handler(), Config; diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl index 985b95d5b..c62d27904 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -146,10 +146,15 @@ build_http_header(X) -> [X]. api_path(Parts) -> - ?SERVER ++ filename:join([?BASE_PATH | Parts]). + join_http_path([?SERVER, ?BASE_PATH | Parts]). api_path_without_base_path(Parts) -> - ?SERVER ++ filename:join([Parts]). + join_http_path([?SERVER | Parts]). + +join_http_path([]) -> + []; +join_http_path([Part | Rest]) -> + lists:foldl(fun(P, Acc) -> emqx_connector_http:join_paths(Acc, P) end, Part, Rest). %% Usage: %% upload_request(<<"site.com/api/upload">>, <<"path/to/file.png">>, diff --git a/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl index e6543d6a1..f49663682 100644 --- a/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl @@ -157,6 +157,31 @@ t_traces(_Config) -> %% traces delete # Delete trace in cluster ok. +t_traces_client(_Config) -> + TraceC = "TraceNameClientID", + emqx_ctl:run_command(["traces", "start", TraceC, "client", "ClientID"]), + emqx_ctl:run_command(["traces", "stop", TraceC]), + emqx_ctl:run_command(["traces", "delete", TraceC]). + +t_traces_client_with_duration(_Config) -> + TraceC = "TraceNameClientID", + Duration = "1000", + emqx_ctl:run_command(["traces", "start", TraceC, "client", "ClientID", Duration]), + emqx_ctl:run_command(["traces", "stop", TraceC]), + emqx_ctl:run_command(["traces", "delete", TraceC]). + +t_traces_topic(_Config) -> + TraceT = "TraceNameTopic", + emqx_ctl:run_command(["traces", "start", TraceT, "topic", "a/b"]), + emqx_ctl:run_command(["traces", "stop", TraceT]), + emqx_ctl:run_command(["traces", "delete", TraceT]). + +t_traces_ip(_Config) -> + TraceI = "TraceNameIP", + emqx_ctl:run_command(["traces", "start", TraceI, "ip_address", "127.0.0.1"]), + emqx_ctl:run_command(["traces", "stop", TraceI]), + emqx_ctl:run_command(["traces", "delete", TraceI]). + t_listeners(_Config) -> %% listeners # List listeners emqx_ctl:run_command(["listeners"]), diff --git a/apps/emqx_modules/README.md b/apps/emqx_modules/README.md index dfa349514..7606cd151 100644 --- a/apps/emqx_modules/README.md +++ b/apps/emqx_modules/README.md @@ -1,7 +1,7 @@ # EMQX Modules The application provides some minor functional modules that are not included in the MQTT -protocol standard, including "Delayed Publish", "Topic Rewrite", "Topic Metrics" and "Telemetry". +protocol standard, including "Delayed Publish", "Topic Rewrite", "Topic Metrics". ## Delayed Publish @@ -42,12 +42,4 @@ See HTTP API docs to [List all monitored topics](https://www.emqx.io/docs/en/v5. and [Get the monitored result](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/MQTT/paths/~1mqtt~1topic_metrics~1%7Btopic%7D/get). -## Telemetry -Telemetry is used for collecting non-sensitive information about the EMQX cluster. - -More introduction about [Telemetry](https://www.emqx.io/docs/en/v5.0/telemetry/telemetry.html#telemetry). - -See HTTP API docs to [Enable/Disable telemetry](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Telemetry/paths/~1telemetry~1status/put), -[Get the enabled status](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Telemetry/paths/~1telemetry~1status/get) -and [Get the data of the module collected](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Telemetry/paths/~1telemetry~1data/get). diff --git a/apps/emqx_modules/include/emqx_modules.hrl b/apps/emqx_modules/include/emqx_modules.hrl index 01b3b38f5..5f49c3231 100644 --- a/apps/emqx_modules/include/emqx_modules.hrl +++ b/apps/emqx_modules/include/emqx_modules.hrl @@ -14,11 +14,5 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% The destination URL for the telemetry data report --define(TELEMETRY_URL, "https://telemetry.emqx.io/api/telemetry"). - -%% Interval for reporting telemetry data, Default: 7d --define(REPORT_INTERVAL, 604800). - -define(API_TAG_MQTT, [<<"MQTT">>]). -define(API_SCHEMA_MODULE, emqx_modules_schema). diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src index b984cf658..078bed0d7 100644 --- a/apps/emqx_modules/src/emqx_modules.app.src +++ b/apps/emqx_modules/src/emqx_modules.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_modules, [ {description, "EMQX Modules"}, - {vsn, "5.0.14"}, + {vsn, "5.0.15"}, {modules, []}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, {mod, {emqx_modules_app, []}}, diff --git a/apps/emqx_modules/src/emqx_modules_app.erl b/apps/emqx_modules/src/emqx_modules_app.erl index f1c95222d..3be81d4cd 100644 --- a/apps/emqx_modules/src/emqx_modules_app.erl +++ b/apps/emqx_modules/src/emqx_modules_app.erl @@ -34,7 +34,6 @@ stop(_State) -> maybe_enable_modules() -> emqx_conf:get([delayed, enable], true) andalso emqx_delayed:load(), - emqx_modules_conf:is_telemetry_enabled() andalso emqx_telemetry:enable(), emqx_observer_cli:enable(), emqx_conf_cli:load(), ok = emqx_rewrite:enable(), @@ -43,7 +42,6 @@ maybe_enable_modules() -> maybe_disable_modules() -> emqx_conf:get([delayed, enable], true) andalso emqx_delayed:unload(), - emqx_modules_conf:is_telemetry_enabled() andalso emqx_telemetry:disable(), emqx_conf:get([observer_cli, enable], true) andalso emqx_observer_cli:disable(), emqx_rewrite:disable(), emqx_conf_cli:unload(), diff --git a/apps/emqx_modules/src/emqx_modules_conf.erl b/apps/emqx_modules/src/emqx_modules_conf.erl index 2162afc70..69a69cb12 100644 --- a/apps/emqx_modules/src/emqx_modules_conf.erl +++ b/apps/emqx_modules/src/emqx_modules_conf.erl @@ -28,9 +28,7 @@ -export([ topic_metrics/0, add_topic_metrics/1, - remove_topic_metrics/1, - is_telemetry_enabled/0, - set_telemetry_status/1 + remove_topic_metrics/1 ]). %% config handlers @@ -45,12 +43,10 @@ -spec load() -> ok. load() -> - emqx_conf:add_handler([topic_metrics], ?MODULE), - emqx_conf:add_handler([telemetry], ?MODULE). + emqx_conf:add_handler([topic_metrics], ?MODULE). -spec unload() -> ok. unload() -> - emqx_conf:remove_handler([telemetry]), emqx_conf:remove_handler([topic_metrics]). %%-------------------------------------------------------------------- @@ -82,18 +78,6 @@ remove_topic_metrics(Topic) -> {error, Reason} -> {error, Reason} end. --spec is_telemetry_enabled() -> boolean(). -is_telemetry_enabled() -> - IsOfficial = emqx_telemetry:official_version(emqx_release:version()), - emqx_conf:get([telemetry, enable], IsOfficial). - --spec set_telemetry_status(boolean()) -> ok | {error, term()}. -set_telemetry_status(Status) -> - case cfg_update([telemetry], set_telemetry_status, Status) of - {ok, _} -> ok; - {error, _} = Error -> Error - end. - %%-------------------------------------------------------------------- %% Config Handler %%-------------------------------------------------------------------- @@ -119,9 +103,7 @@ pre_config_update(_, {remove_topic_metrics, Topic0}, RawConf) -> {ok, RawConf -- [Topic]}; _ -> {error, not_found} - end; -pre_config_update(_, {set_telemetry_status, Status}, RawConf) -> - {ok, RawConf#{<<"enable">> => Status}}. + end. -spec post_config_update( list(atom()), @@ -153,17 +135,6 @@ post_config_update( case emqx_topic_metrics:deregister(Topic) of ok -> ok; {error, Reason} -> {error, Reason} - end; -post_config_update( - _, - {set_telemetry_status, Status}, - _NewConfig, - _OldConfig, - _AppEnvs -) -> - case Status of - true -> emqx_telemetry:enable(); - false -> emqx_telemetry:disable() end. %%-------------------------------------------------------------------- diff --git a/apps/emqx_modules/src/emqx_modules_schema.erl b/apps/emqx_modules/src/emqx_modules_schema.erl index 9057333d5..5eb8ca148 100644 --- a/apps/emqx_modules/src/emqx_modules_schema.erl +++ b/apps/emqx_modules/src/emqx_modules_schema.erl @@ -33,7 +33,6 @@ namespace() -> modules. roots() -> [ "delayed", - "telemetry", array("rewrite", #{ desc => "List of topic rewrite rules.", importance => ?IMPORTANCE_HIDDEN, @@ -46,8 +45,6 @@ roots() -> }) ]. -fields("telemetry") -> - [{enable, ?HOCON(boolean(), #{default => true, desc => "Enable telemetry."})}]; fields("delayed") -> [ {enable, ?HOCON(boolean(), #{default => true, desc => ?DESC(enable)})}, @@ -76,8 +73,6 @@ fields("rewrite") -> fields("topic_metrics") -> [{topic, ?HOCON(binary(), #{desc => "Collect metrics for the topic."})}]. -desc("telemetry") -> - "Settings for the telemetry module."; desc("delayed") -> "Settings for the delayed module."; desc("rewrite") -> diff --git a/apps/emqx_modules/src/emqx_modules_sup.erl b/apps/emqx_modules/src/emqx_modules_sup.erl index 463be28ea..96fdcab3d 100644 --- a/apps/emqx_modules/src/emqx_modules_sup.erl +++ b/apps/emqx_modules/src/emqx_modules_sup.erl @@ -41,7 +41,6 @@ start_link() -> init([]) -> {ok, {{one_for_one, 10, 3600}, [ - ?CHILD(emqx_telemetry), ?CHILD(emqx_topic_metrics), ?CHILD(emqx_trace), ?CHILD(emqx_delayed) diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 24d606c02..c1f1d1391 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -103,6 +103,10 @@ -define(HEALTHCHECK_INTERVAL, 15000). -define(HEALTHCHECK_INTERVAL_RAW, <<"15s">>). +%% milliseconds +-define(DEFAULT_METRICS_FLUSH_INTERVAL, 5_000). +-define(DEFAULT_METRICS_FLUSH_INTERVAL_RAW, <<"5s">>). + %% milliseconds -define(START_TIMEOUT, 5000). -define(START_TIMEOUT_RAW, <<"5s">>). @@ -117,3 +121,5 @@ -define(TEST_ID_PREFIX, "_probe_:"). -define(RES_METRICS, resource_metrics). + +-define(RESOURCE_ALLOCATION_TAB, emqx_resource_allocations). diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 3b92f1200..25ce1b79e 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.16"}, + {vsn, "0.1.17"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ @@ -9,6 +9,7 @@ stdlib, gproc, jsx, + ecpool, emqx, telemetry ]}, diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 80f270b13..10f1de6c4 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -79,7 +79,13 @@ query/2, query/3, %% query the instance without batching and queuing messages. - simple_sync_query/2 + simple_sync_query/2, + %% functions used by connectors to register resources that must be + %% freed when stopping or even when a resource manager crashes. + allocate_resource/3, + has_allocated_resources/1, + get_allocated_resources/1, + forget_allocated_resources/1 ]). %% Direct calls to the callback module @@ -372,6 +378,9 @@ is_buffer_supported(Module) -> {ok, resource_state()} | {error, Reason :: term()}. call_start(ResId, Mod, Config) -> try + %% If the previous manager process crashed without cleaning up + %% allocated resources, clean them up. + clean_allocated_resources(ResId, Mod), Mod:on_start(ResId, Config) catch throw:Error -> @@ -390,7 +399,16 @@ call_health_check(ResId, Mod, ResourceState) -> -spec call_stop(resource_id(), module(), resource_state()) -> term(). call_stop(ResId, Mod, ResourceState) -> - ?SAFE_CALL(Mod:on_stop(ResId, ResourceState)). + ?SAFE_CALL(begin + Res = Mod:on_stop(ResId, ResourceState), + case Res of + ok -> + emqx_resource:forget_allocated_resources(ResId); + _ -> + ok + end, + Res + end). -spec check_config(resource_type(), raw_resource_config()) -> {ok, resource_config()} | {error, term()}. @@ -486,7 +504,37 @@ apply_reply_fun({F, A}, Result) when is_function(F) -> apply_reply_fun(From, Result) -> gen_server:reply(From, Result). +-spec allocate_resource(resource_id(), any(), term()) -> ok. +allocate_resource(InstanceId, Key, Value) -> + true = ets:insert(?RESOURCE_ALLOCATION_TAB, {InstanceId, Key, Value}), + ok. + +-spec has_allocated_resources(resource_id()) -> boolean(). +has_allocated_resources(InstanceId) -> + ets:member(?RESOURCE_ALLOCATION_TAB, InstanceId). + +-spec get_allocated_resources(resource_id()) -> map(). +get_allocated_resources(InstanceId) -> + Objects = ets:lookup(?RESOURCE_ALLOCATION_TAB, InstanceId), + maps:from_list([{K, V} || {_InstanceId, K, V} <- Objects]). + +-spec forget_allocated_resources(resource_id()) -> ok. +forget_allocated_resources(InstanceId) -> + true = ets:delete(?RESOURCE_ALLOCATION_TAB, InstanceId), + ok. + %% ================================================================================= filter_instances(Filter) -> [Id || #{id := Id, mod := Mod} <- list_instances_verbose(), Filter(Id, Mod)]. + +clean_allocated_resources(ResourceId, ResourceMod) -> + case emqx_resource:has_allocated_resources(ResourceId) of + true -> + %% The resource entries in the ETS table are erased inside + %% `call_stop' if the call is successful. + ok = emqx_resource:call_stop(ResourceId, ResourceMod, _ResourceState = undefined), + ok; + false -> + ok + end. diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 6145c3d87..35761822d 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -63,8 +63,8 @@ -define(QUERY(FROM, REQUEST, SENT, EXPIRE_AT), {query, FROM, REQUEST, SENT, EXPIRE_AT}). -define(SIMPLE_QUERY(REQUEST), ?QUERY(undefined, REQUEST, false, infinity)). -define(REPLY(FROM, SENT, RESULT), {reply, FROM, SENT, RESULT}). --define(INFLIGHT_ITEM(Ref, BatchOrQuery, IsRetriable, WorkerMRef), - {Ref, BatchOrQuery, IsRetriable, WorkerMRef} +-define(INFLIGHT_ITEM(Ref, BatchOrQuery, IsRetriable, AsyncWorkerMRef), + {Ref, BatchOrQuery, IsRetriable, AsyncWorkerMRef} ). -define(ITEM_IDX, 2). -define(RETRY_IDX, 3). @@ -80,16 +80,30 @@ -type health_check_interval() :: timer:time(). -type state() :: blocked | running. -type inflight_key() :: integer(). +-type counters() :: #{ + dropped_expired => non_neg_integer(), + dropped_queue_full => non_neg_integer(), + dropped_resource_not_found => non_neg_integer(), + dropped_resource_stopped => non_neg_integer(), + success => non_neg_integer(), + failed => non_neg_integer(), + retried_success => non_neg_integer(), + retried_failed => non_neg_integer() +}. +-type inflight_table() :: ets:tid() | atom() | reference(). -type data() :: #{ id := id(), index := index(), - inflight_tid := ets:tid(), + inflight_tid := inflight_table(), async_workers := #{pid() => reference()}, batch_size := pos_integer(), batch_time := timer:time(), + counters := counters(), + metrics_flush_interval := timer:time(), queue := replayq:q(), resume_interval := timer:time(), - tref := undefined | timer:tref() + tref := undefined | {timer:tref() | reference(), reference()}, + metrics_tref := undefined | {timer:tref() | reference(), reference()} }. callback_mode() -> [state_functions, state_enter]. @@ -171,24 +185,29 @@ init({Id, Index, Opts}) -> emqx_resource_metrics:queuing_set(Id, Index, queue_count(Queue)), emqx_resource_metrics:inflight_set(Id, Index, 0), InflightWinSize = maps:get(inflight_window, Opts, ?DEFAULT_INFLIGHT), - InflightTID = inflight_new(InflightWinSize, Id, Index), + InflightTID = inflight_new(InflightWinSize), HealthCheckInterval = maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL), RequestTimeout = maps:get(request_timeout, Opts, ?DEFAULT_REQUEST_TIMEOUT), BatchTime0 = maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME), BatchTime = adjust_batch_time(Id, RequestTimeout, BatchTime0), DefaultResumeInterval = default_resume_interval(RequestTimeout, HealthCheckInterval), ResumeInterval = maps:get(resume_interval, Opts, DefaultResumeInterval), - Data = #{ + MetricsFlushInterval = maps:get(metrics_flush_interval, Opts, ?DEFAULT_METRICS_FLUSH_INTERVAL), + Data0 = #{ id => Id, index => Index, inflight_tid => InflightTID, async_workers => #{}, batch_size => BatchSize, batch_time => BatchTime, + counters => #{}, + metrics_flush_interval => MetricsFlushInterval, queue => Queue, resume_interval => ResumeInterval, - tref => undefined + tref => undefined, + metrics_tref => undefined }, + Data = ensure_metrics_flush_timer(Data0), ?tp(buffer_worker_init, #{id => Id, index => Index, queue_opts => QueueOpts}), {ok, running, Data}. @@ -208,11 +227,16 @@ running(cast, block, St) -> {next_state, blocked, St}; running(info, ?SEND_REQ(_ReplyTo, _Req) = Request0, Data) -> handle_query_requests(Request0, Data); -running(info, {flush, Ref}, St = #{tref := {_TRef, Ref}}) -> - flush(St#{tref := undefined}); -running(info, {flush, _Ref}, _St) -> +running(info, {flush, Ref}, Data = #{tref := {_TRef, Ref}}) -> + flush(Data#{tref := undefined}); +running(info, {flush, _Ref}, _Data) -> ?tp(discarded_stale_flush, #{}), keep_state_and_data; +running(info, {flush_metrics, Ref}, Data0 = #{metrics_tref := {_TRef, Ref}}) -> + Data = flush_metrics(Data0#{metrics_tref := undefined}), + {keep_state, Data}; +running(info, {flush_metrics, _Ref}, _Data) -> + keep_state_and_data; running(info, {'DOWN', _MRef, process, Pid, Reason}, Data0 = #{async_workers := AsyncWorkers0}) when is_map_key(Pid, AsyncWorkers0) -> @@ -241,6 +265,11 @@ blocked(info, ?SEND_REQ(_ReplyTo, _Req) = Request0, Data0) -> blocked(info, {flush, _Ref}, _Data) -> %% ignore stale timer keep_state_and_data; +blocked(info, {flush_metrics, Ref}, Data0 = #{metrics_tref := {_TRef, Ref}}) -> + Data = flush_metrics(Data0#{metrics_tref := undefined}), + {keep_state, Data}; +blocked(info, {flush_metrics, _Ref}, _Data) -> + keep_state_and_data; blocked(info, {'DOWN', _MRef, process, Pid, Reason}, Data0 = #{async_workers := AsyncWorkers0}) when is_map_key(Pid, AsyncWorkers0) -> @@ -310,11 +339,7 @@ pick_cast(Id, Key, Query) -> resume_from_blocked(Data) -> ?tp(buffer_worker_resume_from_blocked_enter, #{}), - #{ - id := Id, - index := Index, - inflight_tid := InflightTID - } = Data, + #{inflight_tid := InflightTID} = Data, Now = now_(), case inflight_get_first_retriable(InflightTID, Now) of none -> @@ -325,11 +350,16 @@ resume_from_blocked(Data) -> {next_state, running, Data} end; {expired, Ref, Batch} -> - WorkerPid = self(), - IsAcked = ack_inflight(InflightTID, Ref, Id, Index, WorkerPid), - IsAcked andalso emqx_resource_metrics:dropped_expired_inc(Id, length(Batch)), + BufferWorkerPid = self(), + IsAcked = ack_inflight(InflightTID, Ref, BufferWorkerPid), + Counters = + case IsAcked of + true -> #{dropped_expired => length(Batch)}; + false -> #{} + end, + NData = aggregate_counters(Data, Counters), ?tp(buffer_worker_retry_expired, #{expired => Batch}), - resume_from_blocked(Data); + resume_from_blocked(NData); {single, Ref, Query} -> %% We retry msgs in inflight window sync, as if we send them %% async, they will be appended to the end of inflight window again. @@ -339,11 +369,11 @@ resume_from_blocked(Data) -> {batch, Ref, NotExpired, Expired} -> NumExpired = length(Expired), ok = update_inflight_item(InflightTID, Ref, NotExpired, NumExpired), - emqx_resource_metrics:dropped_expired_inc(Id, NumExpired), + NData = aggregate_counters(Data, #{dropped_expired => NumExpired}), ?tp(buffer_worker_retry_expired, #{expired => Expired}), %% We retry msgs in inflight window sync, as if we send them %% async, they will be appended to the end of inflight window again. - retry_inflight_sync(Ref, NotExpired, Data) + retry_inflight_sync(Ref, NotExpired, NData) end. retry_inflight_sync(Ref, QueryOrBatch, Data0) -> @@ -356,7 +386,7 @@ retry_inflight_sync(Ref, QueryOrBatch, Data0) -> ?tp(buffer_worker_retry_inflight, #{query_or_batch => QueryOrBatch, ref => Ref}), QueryOpts = #{simple_query => false}, Result = call_query(force_sync, Id, Index, Ref, QueryOrBatch, QueryOpts), - ReplyResult = + {ShouldAck, PostFn, DeltaCounters} = case QueryOrBatch of ?QUERY(ReplyTo, _, HasBeenSent, _ExpireAt) -> Reply = ?REPLY(ReplyTo, HasBeenSent, Result), @@ -364,9 +394,10 @@ retry_inflight_sync(Ref, QueryOrBatch, Data0) -> [?QUERY(_, _, _, _) | _] = Batch -> batch_reply_caller_defer_metrics(Id, Result, Batch, QueryOpts) end, - case ReplyResult of + Data1 = aggregate_counters(Data0, DeltaCounters), + case ShouldAck of %% Send failed because resource is down - {nack, PostFn} -> + nack -> PostFn(), ?tp( buffer_worker_retry_inflight_failed, @@ -375,11 +406,11 @@ retry_inflight_sync(Ref, QueryOrBatch, Data0) -> query_or_batch => QueryOrBatch } ), - {keep_state, Data0, {state_timeout, ResumeT, unblock}}; + {keep_state, Data1, {state_timeout, ResumeT, unblock}}; %% Send ok or failed but the resource is working - {ack, PostFn} -> - WorkerPid = self(), - IsAcked = ack_inflight(InflightTID, Ref, Id, Index, WorkerPid), + ack -> + BufferWorkerPid = self(), + IsAcked = ack_inflight(InflightTID, Ref, BufferWorkerPid), %% we need to defer bumping the counters after %% `inflight_drop' to avoid the race condition when an %% inflight request might get completed concurrently with @@ -394,7 +425,7 @@ retry_inflight_sync(Ref, QueryOrBatch, Data0) -> query_or_batch => QueryOrBatch } ), - resume_from_blocked(Data0) + resume_from_blocked(Data1) end. %% Called during the `running' state only. @@ -426,9 +457,9 @@ collect_and_enqueue_query_requests(Request0, Data0) -> end, Requests ), - {Overflown, NewQ} = append_queue(Id, Index, Q, Queries), + {Overflown, NewQ, DeltaCounters} = append_queue(Id, Index, Q, Queries), ok = reply_overflown(Overflown), - Data0#{queue := NewQ}. + aggregate_counters(Data0#{queue := NewQ}, DeltaCounters). reply_overflown([]) -> ok; @@ -463,8 +494,6 @@ maybe_flush(Data0) -> -spec flush(data()) -> gen_statem:event_handler_result(state(), data()). flush(Data0) -> #{ - id := Id, - index := Index, batch_size := BatchSize, inflight_tid := InflightTID, queue := Q0 @@ -497,13 +526,13 @@ flush(Data0) -> case sieve_expired_requests(Batch, Now) of {[], _AllExpired} -> ok = replayq:ack(Q1, QAckRef), - emqx_resource_metrics:dropped_expired_inc(Id, length(Batch)), - emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)), + NumExpired = length(Batch), + Data3 = aggregate_counters(Data2, #{dropped_expired => NumExpired}), ?tp(buffer_worker_flush_all_expired, #{batch => Batch}), - flush(Data2); + flush(Data3); {NotExpired, Expired} -> NumExpired = length(Expired), - emqx_resource_metrics:dropped_expired_inc(Id, NumExpired), + Data3 = aggregate_counters(Data2, #{dropped_expired => NumExpired}), IsBatch = (BatchSize > 1), %% We *must* use the new queue, because we currently can't %% `nack' a `pop'. @@ -513,7 +542,7 @@ flush(Data0) -> #{expired => Expired, not_expired => NotExpired} ), Ref = make_request_ref(), - do_flush(Data2, #{ + do_flush(Data3, #{ is_batch => IsBatch, batch => NotExpired, ref => Ref, @@ -548,7 +577,9 @@ do_flush( QueryOpts = #{inflight_tid => InflightTID, simple_query => false}, Result = call_query(async_if_possible, Id, Index, Ref, Request, QueryOpts), Reply = ?REPLY(ReplyTo, HasBeenSent, Result), - case reply_caller(Id, Reply, QueryOpts) of + {ShouldAck, DeltaCounters} = reply_caller(Id, Reply, QueryOpts), + Data1 = aggregate_counters(Data0, DeltaCounters), + case ShouldAck of %% Failed; remove the request from the queue, as we cannot pop %% from it again, but we'll retry it using the inflight table. nack -> @@ -556,17 +587,16 @@ do_flush( %% we set it atomically just below; a limitation of having %% to use tuples for atomic ets updates IsRetriable = true, - WorkerMRef0 = undefined, - InflightItem = ?INFLIGHT_ITEM(Ref, Request, IsRetriable, WorkerMRef0), + AsyncWorkerMRef0 = undefined, + InflightItem = ?INFLIGHT_ITEM(Ref, Request, IsRetriable, AsyncWorkerMRef0), %% we must append again to the table to ensure that the %% request will be retried (i.e., it might not have been %% inserted during `call_query' if the resource was down %% and/or if it was a sync request). - inflight_append(InflightTID, InflightItem, Id, Index), + inflight_append(InflightTID, InflightItem), mark_inflight_as_retriable(InflightTID, Ref), - {Data1, WorkerMRef} = ensure_async_worker_monitored(Data0, Result), - store_async_worker_reference(InflightTID, Ref, WorkerMRef), - emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)), + {Data2, AsyncWorkerMRef} = ensure_async_worker_monitored(Data1, Result), + store_async_worker_reference(InflightTID, Ref, AsyncWorkerMRef), ?tp( buffer_worker_flush_nack, #{ @@ -576,7 +606,7 @@ do_flush( result => Result } ), - {next_state, blocked, Data1}; + {next_state, blocked, Data2}; %% Success; just ack. ack -> ok = replayq:ack(Q1, QAckRef), @@ -585,18 +615,17 @@ do_flush( %% must ensure the async worker is being monitored for %% such requests. IsUnrecoverableError = is_unrecoverable_error(Result), - WorkerPid = self(), + BufferWorkerPid = self(), case is_async_return(Result) of true when IsUnrecoverableError -> - ack_inflight(InflightTID, Ref, Id, Index, WorkerPid); + ack_inflight(InflightTID, Ref, BufferWorkerPid); true -> ok; false -> - ack_inflight(InflightTID, Ref, Id, Index, WorkerPid) + ack_inflight(InflightTID, Ref, BufferWorkerPid) end, - {Data1, WorkerMRef} = ensure_async_worker_monitored(Data0, Result), - store_async_worker_reference(InflightTID, Ref, WorkerMRef), - emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)), + {Data2, AsyncWorkerMRef} = ensure_async_worker_monitored(Data1, Result), + store_async_worker_reference(InflightTID, Ref, AsyncWorkerMRef), ?tp( buffer_worker_flush_ack, #{ @@ -617,7 +646,7 @@ do_flush( }), ok end, - {keep_state, Data1} + {keep_state, Data2} end; do_flush(#{queue := Q1} = Data0, #{ is_batch := true, @@ -633,7 +662,9 @@ do_flush(#{queue := Q1} = Data0, #{ } = Data0, QueryOpts = #{inflight_tid => InflightTID, simple_query => false}, Result = call_query(async_if_possible, Id, Index, Ref, Batch, QueryOpts), - case batch_reply_caller(Id, Result, Batch, QueryOpts) of + {ShouldAck, DeltaCounters} = batch_reply_caller(Id, Result, Batch, QueryOpts), + Data1 = aggregate_counters(Data0, DeltaCounters), + case ShouldAck of %% Failed; remove the request from the queue, as we cannot pop %% from it again, but we'll retry it using the inflight table. nack -> @@ -641,17 +672,16 @@ do_flush(#{queue := Q1} = Data0, #{ %% we set it atomically just below; a limitation of having %% to use tuples for atomic ets updates IsRetriable = true, - WorkerMRef0 = undefined, - InflightItem = ?INFLIGHT_ITEM(Ref, Batch, IsRetriable, WorkerMRef0), + AsyncWorkerMRef0 = undefined, + InflightItem = ?INFLIGHT_ITEM(Ref, Batch, IsRetriable, AsyncWorkerMRef0), %% we must append again to the table to ensure that the %% request will be retried (i.e., it might not have been %% inserted during `call_query' if the resource was down %% and/or if it was a sync request). - inflight_append(InflightTID, InflightItem, Id, Index), + inflight_append(InflightTID, InflightItem), mark_inflight_as_retriable(InflightTID, Ref), - {Data1, WorkerMRef} = ensure_async_worker_monitored(Data0, Result), - store_async_worker_reference(InflightTID, Ref, WorkerMRef), - emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)), + {Data2, AsyncWorkerMRef} = ensure_async_worker_monitored(Data1, Result), + store_async_worker_reference(InflightTID, Ref, AsyncWorkerMRef), ?tp( buffer_worker_flush_nack, #{ @@ -661,7 +691,7 @@ do_flush(#{queue := Q1} = Data0, #{ result => Result } ), - {next_state, blocked, Data1}; + {next_state, blocked, Data2}; %% Success; just ack. ack -> ok = replayq:ack(Q1, QAckRef), @@ -670,18 +700,17 @@ do_flush(#{queue := Q1} = Data0, #{ %% must ensure the async worker is being monitored for %% such requests. IsUnrecoverableError = is_unrecoverable_error(Result), - WorkerPid = self(), + BufferWorkerPid = self(), case is_async_return(Result) of true when IsUnrecoverableError -> - ack_inflight(InflightTID, Ref, Id, Index, WorkerPid); + ack_inflight(InflightTID, Ref, BufferWorkerPid); true -> ok; false -> - ack_inflight(InflightTID, Ref, Id, Index, WorkerPid) + ack_inflight(InflightTID, Ref, BufferWorkerPid) end, - {Data1, WorkerMRef} = ensure_async_worker_monitored(Data0, Result), - store_async_worker_reference(InflightTID, Ref, WorkerMRef), - emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)), + {Data2, AsyncWorkerMRef} = ensure_async_worker_monitored(Data1, Result), + store_async_worker_reference(InflightTID, Ref, AsyncWorkerMRef), CurrentCount = queue_count(Q1), ?tp( buffer_worker_flush_ack, @@ -691,13 +720,13 @@ do_flush(#{queue := Q1} = Data0, #{ queue_count => CurrentCount } ), - Data2 = + Data3 = case {CurrentCount > 0, CurrentCount >= BatchSize} of {false, _} -> ?tp_ignore_side_effects_in_prod(buffer_worker_queue_drained, #{ inflight => inflight_count(InflightTID) }), - Data1; + Data2; {true, true} -> ?tp(buffer_worker_flush_ack_reflush, #{ batch_or_query => Batch, @@ -706,17 +735,18 @@ do_flush(#{queue := Q1} = Data0, #{ batch_size => BatchSize }), flush_worker(self()), - Data1; + Data2; {true, false} -> - ensure_flush_timer(Data1) + ensure_flush_timer(Data2) end, - {keep_state, Data2} + {keep_state, Data3} end. batch_reply_caller(Id, BatchResult, Batch, QueryOpts) -> - {ShouldBlock, PostFn} = batch_reply_caller_defer_metrics(Id, BatchResult, Batch, QueryOpts), + {ShouldBlock, PostFn, DeltaCounters} = + batch_reply_caller_defer_metrics(Id, BatchResult, Batch, QueryOpts), PostFn(), - ShouldBlock. + {ShouldBlock, DeltaCounters}. batch_reply_caller_defer_metrics(Id, BatchResult, Batch, QueryOpts) -> %% the `Mod:on_batch_query/3` returns a single result for a batch, @@ -727,23 +757,25 @@ batch_reply_caller_defer_metrics(Id, BatchResult, Batch, QueryOpts) -> end, Batch ), - {ShouldAck, PostFns} = + {ShouldAck, PostFns, Counters} = lists:foldl( - fun(Reply, {_ShouldAck, PostFns}) -> + fun(Reply, {_ShouldAck, PostFns, OldCounters}) -> %% _ShouldAck should be the same as ShouldAck starting from the second reply - {ShouldAck, PostFn} = reply_caller_defer_metrics(Id, Reply, QueryOpts), - {ShouldAck, [PostFn | PostFns]} + {ShouldAck, PostFn, DeltaCounters} = reply_caller_defer_metrics( + Id, Reply, QueryOpts + ), + {ShouldAck, [PostFn | PostFns], merge_counters(OldCounters, DeltaCounters)} end, - {ack, []}, + {ack, [], #{}}, Replies ), PostFn = fun() -> lists:foreach(fun(F) -> F() end, lists:reverse(PostFns)) end, - {ShouldAck, PostFn}. + {ShouldAck, PostFn, Counters}. reply_caller(Id, Reply, QueryOpts) -> - {ShouldAck, PostFn} = reply_caller_defer_metrics(Id, Reply, QueryOpts), + {ShouldAck, PostFn, DeltaCounters} = reply_caller_defer_metrics(Id, Reply, QueryOpts), PostFn(), - ShouldAck. + {ShouldAck, DeltaCounters}. %% Should only reply to the caller when the decision is final (not %% retriable). See comment on `handle_query_result_pure'. @@ -752,7 +784,7 @@ reply_caller_defer_metrics(Id, ?REPLY(undefined, HasBeenSent, Result), _QueryOpt reply_caller_defer_metrics(Id, ?REPLY(ReplyTo, HasBeenSent, Result), QueryOpts) -> IsSimpleQuery = maps:get(simple_query, QueryOpts, false), IsUnrecoverableError = is_unrecoverable_error(Result), - {ShouldAck, PostFn} = handle_query_result_pure(Id, Result, HasBeenSent), + {ShouldAck, PostFn, DeltaCounters} = handle_query_result_pure(Id, Result, HasBeenSent), case {ShouldAck, Result, IsUnrecoverableError, IsSimpleQuery} of {ack, {async_return, _}, true, _} -> ok = do_reply_caller(ReplyTo, Result); @@ -765,11 +797,14 @@ reply_caller_defer_metrics(Id, ?REPLY(ReplyTo, HasBeenSent, Result), QueryOpts) {ack, _, _, _} -> ok = do_reply_caller(ReplyTo, Result) end, - {ShouldAck, PostFn}. + {ShouldAck, PostFn, DeltaCounters}. +%% This is only called by `simple_{,a}sync_query', so we can bump the +%% counters here. handle_query_result(Id, Result, HasBeenSent) -> - {ShouldBlock, PostFn} = handle_query_result_pure(Id, Result, HasBeenSent), + {ShouldBlock, PostFn, DeltaCounters} = handle_query_result_pure(Id, Result, HasBeenSent), PostFn(), + bump_counters(Id, DeltaCounters), ShouldBlock. %% We should always retry (nack), except when: @@ -778,91 +813,169 @@ handle_query_result(Id, Result, HasBeenSent) -> %% * the result is a success (or at least a delayed result) %% We also retry even sync requests. In that case, we shouldn't reply %% the caller until one of those final results above happen. +-spec handle_query_result_pure(id(), term(), HasBeenSent :: boolean()) -> + {ack | nack, function(), counters()}. handle_query_result_pure(_Id, ?RESOURCE_ERROR_M(exception, Msg), _HasBeenSent) -> PostFn = fun() -> ?SLOG(error, #{msg => resource_exception, info => Msg}), ok end, - {nack, PostFn}; + {nack, PostFn, #{}}; handle_query_result_pure(_Id, ?RESOURCE_ERROR_M(NotWorking, _), _HasBeenSent) when NotWorking == not_connected; NotWorking == blocked -> - {nack, fun() -> ok end}; + {nack, fun() -> ok end, #{}}; handle_query_result_pure(Id, ?RESOURCE_ERROR_M(not_found, Msg), _HasBeenSent) -> PostFn = fun() -> ?SLOG(error, #{id => Id, msg => resource_not_found, info => Msg}), - emqx_resource_metrics:dropped_resource_not_found_inc(Id), ok end, - {ack, PostFn}; + {ack, PostFn, #{dropped_resource_not_found => 1}}; handle_query_result_pure(Id, ?RESOURCE_ERROR_M(stopped, Msg), _HasBeenSent) -> PostFn = fun() -> ?SLOG(error, #{id => Id, msg => resource_stopped, info => Msg}), - emqx_resource_metrics:dropped_resource_stopped_inc(Id), ok end, - {ack, PostFn}; + {ack, PostFn, #{dropped_resource_stopped => 1}}; handle_query_result_pure(Id, ?RESOURCE_ERROR_M(Reason, _), _HasBeenSent) -> PostFn = fun() -> ?SLOG(error, #{id => Id, msg => other_resource_error, reason => Reason}), ok end, - {nack, PostFn}; + {nack, PostFn, #{}}; handle_query_result_pure(Id, {error, Reason} = Error, HasBeenSent) -> case is_unrecoverable_error(Error) of true -> PostFn = fun() -> ?SLOG(error, #{id => Id, msg => unrecoverable_error, reason => Reason}), - inc_sent_failed(Id, HasBeenSent), ok end, - {ack, PostFn}; + Counters = + case HasBeenSent of + true -> #{retried_failed => 1}; + false -> #{failed => 1} + end, + {ack, PostFn, Counters}; false -> PostFn = fun() -> ?SLOG(error, #{id => Id, msg => send_error, reason => Reason}), ok end, - {nack, PostFn} + {nack, PostFn, #{}} end; handle_query_result_pure(Id, {async_return, Result}, HasBeenSent) -> handle_query_async_result_pure(Id, Result, HasBeenSent); -handle_query_result_pure(Id, Result, HasBeenSent) -> +handle_query_result_pure(_Id, Result, HasBeenSent) -> PostFn = fun() -> assert_ok_result(Result), - inc_sent_success(Id, HasBeenSent), ok end, - {ack, PostFn}. + Counters = + case HasBeenSent of + true -> #{retried_success => 1}; + false -> #{success => 1} + end, + {ack, PostFn, Counters}. +-spec handle_query_async_result_pure(id(), term(), HasBeenSent :: boolean()) -> + {ack | nack, function(), counters()}. handle_query_async_result_pure(Id, {error, Reason} = Error, HasBeenSent) -> case is_unrecoverable_error(Error) of true -> PostFn = fun() -> ?SLOG(error, #{id => Id, msg => unrecoverable_error, reason => Reason}), - inc_sent_failed(Id, HasBeenSent), ok end, - {ack, PostFn}; + Counters = + case HasBeenSent of + true -> #{retried_failed => 1}; + false -> #{failed => 1} + end, + {ack, PostFn, Counters}; false -> PostFn = fun() -> ?SLOG(error, #{id => Id, msg => async_send_error, reason => Reason}), ok end, - {nack, PostFn} + {nack, PostFn, #{}} end; handle_query_async_result_pure(_Id, {ok, Pid}, _HasBeenSent) when is_pid(Pid) -> - {ack, fun() -> ok end}; + {ack, fun() -> ok end, #{}}; handle_query_async_result_pure(_Id, ok, _HasBeenSent) -> - {ack, fun() -> ok end}. + {ack, fun() -> ok end, #{}}. + +-spec aggregate_counters(data(), counters()) -> data(). +aggregate_counters(Data = #{counters := OldCounters}, DeltaCounters) -> + Counters = merge_counters(OldCounters, DeltaCounters), + Data#{counters := Counters}. + +-spec merge_counters(counters(), counters()) -> counters(). +merge_counters(OldCounters, DeltaCounters) -> + maps:fold( + fun(Metric, Val, Acc) -> + maps:update_with(Metric, fun(X) -> X + Val end, Val, Acc) + end, + OldCounters, + DeltaCounters + ). + +-spec flush_metrics(data()) -> data(). +flush_metrics(Data = #{id := Id, counters := Counters}) -> + bump_counters(Id, Counters), + set_gauges(Data), + ensure_metrics_flush_timer(Data#{counters := #{}}). + +-spec ensure_metrics_flush_timer(data()) -> data(). +ensure_metrics_flush_timer(Data = #{metrics_tref := undefined, metrics_flush_interval := T}) -> + Ref = make_ref(), + TRef = erlang:send_after(T, self(), {flush_metrics, Ref}), + Data#{metrics_tref := {TRef, Ref}}. + +-spec bump_counters(id(), counters()) -> ok. +bump_counters(Id, Counters) -> + Iter = maps:iterator(Counters), + do_bump_counters(Iter, Id). + +do_bump_counters(Iter, Id) -> + case maps:next(Iter) of + {Key, Val, NIter} -> + do_bump_counters1(Key, Val, Id), + do_bump_counters(NIter, Id); + none -> + ok + end. + +do_bump_counters1(dropped_expired, Val, Id) -> + emqx_resource_metrics:dropped_expired_inc(Id, Val); +do_bump_counters1(dropped_queue_full, Val, Id) -> + emqx_resource_metrics:dropped_queue_full_inc(Id, Val); +do_bump_counters1(failed, Val, Id) -> + emqx_resource_metrics:failed_inc(Id, Val); +do_bump_counters1(retried_failed, Val, Id) -> + emqx_resource_metrics:retried_failed_inc(Id, Val); +do_bump_counters1(success, Val, Id) -> + emqx_resource_metrics:success_inc(Id, Val); +do_bump_counters1(retried_success, Val, Id) -> + emqx_resource_metrics:retried_success_inc(Id, Val); +do_bump_counters1(dropped_resource_not_found, Val, Id) -> + emqx_resource_metrics:dropped_resource_not_found_inc(Id, Val); +do_bump_counters1(dropped_resource_stopped, Val, Id) -> + emqx_resource_metrics:dropped_resource_stopped_inc(Id, Val). + +-spec set_gauges(data()) -> ok. +set_gauges(_Data = #{id := Id, index := Index, queue := Q, inflight_tid := InflightTID}) -> + emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q)), + emqx_resource_metrics:inflight_set(Id, Index, inflight_num_msgs(InflightTID)), + ok. handle_async_worker_down(Data0, Pid) -> #{async_workers := AsyncWorkers0} = Data0, - {WorkerMRef, AsyncWorkers} = maps:take(Pid, AsyncWorkers0), + {AsyncWorkerMRef, AsyncWorkers} = maps:take(Pid, AsyncWorkers0), Data = Data0#{async_workers := AsyncWorkers}, - mark_inflight_items_as_retriable(Data, WorkerMRef), + mark_inflight_items_as_retriable(Data, AsyncWorkerMRef), {keep_state, Data}. -spec call_query(force_sync | async_if_possible, _, _, _, _, _) -> _. @@ -940,9 +1053,9 @@ apply_query_fun(async, Mod, Id, Index, Ref, ?QUERY(_, Request, _, _) = Query, Re min_query => minimize(Query) }, IsRetriable = false, - WorkerMRef = undefined, - InflightItem = ?INFLIGHT_ITEM(Ref, Query, IsRetriable, WorkerMRef), - ok = inflight_append(InflightTID, InflightItem, Id, Index), + AsyncWorkerMRef = undefined, + InflightItem = ?INFLIGHT_ITEM(Ref, Query, IsRetriable, AsyncWorkerMRef), + ok = inflight_append(InflightTID, InflightItem), Result = Mod:on_query_async(Id, Request, {ReplyFun, [ReplyContext]}, ResSt), {async_return, Result} end, @@ -976,9 +1089,9 @@ apply_query_fun(async, Mod, Id, Index, Ref, [?QUERY(_, _, _, _) | _] = Batch, Re fun(?QUERY(_ReplyTo, Request, _, _ExpireAt)) -> Request end, Batch ), IsRetriable = false, - WorkerMRef = undefined, - InflightItem = ?INFLIGHT_ITEM(Ref, Batch, IsRetriable, WorkerMRef), - ok = inflight_append(InflightTID, InflightItem, Id, Index), + AsyncWorkerMRef = undefined, + InflightItem = ?INFLIGHT_ITEM(Ref, Batch, IsRetriable, AsyncWorkerMRef), + ok = inflight_append(InflightTID, InflightItem), Result = Mod:on_batch_query_async(Id, Requests, {ReplyFun, [ReplyContext]}, ResSt), {async_return, Result} end, @@ -1005,8 +1118,7 @@ handle_async_reply1( request_ref := Ref, inflight_tid := InflightTID, resource_id := Id, - worker_index := Index, - buffer_worker := WorkerPid, + buffer_worker := BufferWorkerPid, min_query := ?QUERY(_, _, _, ExpireAt) = _Query } = ReplyContext, Result @@ -1018,7 +1130,9 @@ handle_async_reply1( Now = now_(), case is_expired(ExpireAt, Now) of true -> - IsAcked = ack_inflight(InflightTID, Ref, Id, Index, WorkerPid), + IsAcked = ack_inflight(InflightTID, Ref, BufferWorkerPid), + %% evalutate metrics call here since we're not inside + %% buffer worker IsAcked andalso emqx_resource_metrics:late_reply_inc(Id), ?tp(handle_async_reply_expired, #{expired => [_Query]}), ok; @@ -1031,8 +1145,7 @@ do_handle_async_reply( query_opts := QueryOpts, resource_id := Id, request_ref := Ref, - worker_index := Index, - buffer_worker := WorkerPid, + buffer_worker := BufferWorkerPid, inflight_tid := InflightTID, min_query := ?QUERY(ReplyTo, _, Sent, _ExpireAt) = _Query }, @@ -1041,7 +1154,7 @@ do_handle_async_reply( %% NOTE: 'inflight' is the count of messages that were sent async %% but received no ACK, NOT the number of messages queued in the %% inflight window. - {Action, PostFn} = reply_caller_defer_metrics( + {Action, PostFn, DeltaCounters} = reply_caller_defer_metrics( Id, ?REPLY(ReplyTo, Sent, Result), QueryOpts ), @@ -1055,10 +1168,12 @@ do_handle_async_reply( nack -> %% Keep retrying. ok = mark_inflight_as_retriable(InflightTID, Ref), - ok = ?MODULE:block(WorkerPid), + ok = ?MODULE:block(BufferWorkerPid), blocked; ack -> - ok = do_async_ack(InflightTID, Ref, Id, Index, WorkerPid, PostFn, QueryOpts) + ok = do_async_ack( + InflightTID, Ref, Id, PostFn, BufferWorkerPid, DeltaCounters, QueryOpts + ) end. handle_async_batch_reply( @@ -1107,11 +1222,10 @@ handle_async_batch_reply2([], _, _, _) -> %% this usually should never happen unless the async callback is being evaluated concurrently ok; handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> - ?INFLIGHT_ITEM(_, RealBatch, _IsRetriable, _WorkerMRef) = Inflight, + ?INFLIGHT_ITEM(_, RealBatch, _IsRetriable, _AsyncWorkerMRef) = Inflight, #{ resource_id := Id, - worker_index := Index, - buffer_worker := WorkerPid, + buffer_worker := BufferWorkerPid, inflight_tid := InflightTID, request_ref := Ref, min_batch := Batch @@ -1130,11 +1244,13 @@ handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> RealNotExpired0 ), NumExpired = length(RealExpired), + %% evalutate metrics call here since we're not inside buffer + %% worker emqx_resource_metrics:late_reply_inc(Id, NumExpired), case RealNotExpired of [] -> %% all expired, no need to update back the inflight batch - _ = ack_inflight(InflightTID, Ref, Id, Index, WorkerPid), + _ = ack_inflight(InflightTID, Ref, BufferWorkerPid), ok; _ -> %% some queries are not expired, put them back to the inflight batch @@ -1145,9 +1261,8 @@ handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> do_handle_async_batch_reply( #{ - buffer_worker := WorkerPid, + buffer_worker := BufferWorkerPid, resource_id := Id, - worker_index := Index, inflight_tid := InflightTID, request_ref := Ref, min_batch := Batch, @@ -1155,7 +1270,9 @@ do_handle_async_batch_reply( }, Result ) -> - {Action, PostFn} = batch_reply_caller_defer_metrics(Id, Result, Batch, QueryOpts), + {Action, PostFn, DeltaCounters} = batch_reply_caller_defer_metrics( + Id, Result, Batch, QueryOpts + ), ?tp(handle_async_reply, #{ action => Action, batch_or_query => Batch, @@ -1166,19 +1283,23 @@ do_handle_async_batch_reply( nack -> %% Keep retrying. ok = mark_inflight_as_retriable(InflightTID, Ref), - ok = ?MODULE:block(WorkerPid), + ok = ?MODULE:block(BufferWorkerPid), blocked; ack -> - ok = do_async_ack(InflightTID, Ref, Id, Index, WorkerPid, PostFn, QueryOpts) + ok = do_async_ack( + InflightTID, Ref, Id, PostFn, BufferWorkerPid, DeltaCounters, QueryOpts + ) end. -do_async_ack(InflightTID, Ref, Id, Index, WorkerPid, PostFn, QueryOpts) -> - IsKnownRef = ack_inflight(InflightTID, Ref, Id, Index, WorkerPid), +do_async_ack(InflightTID, Ref, Id, PostFn, BufferWorkerPid, DeltaCounters, QueryOpts) -> + IsKnownRef = ack_inflight(InflightTID, Ref, BufferWorkerPid), case maps:get(simple_query, QueryOpts, false) of true -> - PostFn(); + PostFn(), + bump_counters(Id, DeltaCounters); false when IsKnownRef -> - PostFn(); + PostFn(), + bump_counters(Id, DeltaCounters); false -> ok end, @@ -1222,31 +1343,30 @@ estimate_size(QItem) -> erlang:external_size(QItem). -spec append_queue(id(), index(), replayq:q(), [queue_query()]) -> - {[queue_query()], replayq:q()}. + {[queue_query()], replayq:q(), counters()}. append_queue(Id, Index, Q, Queries) -> %% this assertion is to ensure that we never append a raw binary %% because the marshaller will get lost. false = is_binary(hd(Queries)), Q0 = replayq:append(Q, Queries), - {Overflown, Q2} = + {Overflown, Q2, DeltaCounters} = case replayq:overflow(Q0) of OverflownBytes when OverflownBytes =< 0 -> - {[], Q0}; + {[], Q0, #{}}; OverflownBytes -> PopOpts = #{bytes_limit => OverflownBytes, count_limit => 999999999}, {Q1, QAckRef, Items2} = replayq:pop(Q0, PopOpts), ok = replayq:ack(Q1, QAckRef), Dropped = length(Items2), - emqx_resource_metrics:dropped_queue_full_inc(Id, Dropped), + Counters = #{dropped_queue_full => Dropped}, ?SLOG(info, #{ msg => buffer_worker_overflow, resource_id => Id, worker_index => Index, dropped => Dropped }), - {Items2, Q1} + {Items2, Q1, Counters} end, - emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q2)), ?tp( buffer_worker_appended_to_queue, #{ @@ -1256,7 +1376,7 @@ append_queue(Id, Index, Q, Queries) -> overflown => length(Overflown) } ), - {Overflown, Q2}. + {Overflown, Q2, DeltaCounters}. %%============================================================================== %% the inflight queue for async query @@ -1266,20 +1386,18 @@ append_queue(Id, Index, Q, Queries) -> -define(INITIAL_TIME_REF, initial_time). -define(INITIAL_MONOTONIC_TIME_REF, initial_monotonic_time). -inflight_new(InfltWinSZ, Id, Index) -> +inflight_new(InfltWinSZ) -> TableId = ets:new( emqx_resource_buffer_worker_inflight_tab, [ordered_set, public, {write_concurrency, true}] ), - inflight_append(TableId, {?MAX_SIZE_REF, InfltWinSZ}, Id, Index), + inflight_append(TableId, {?MAX_SIZE_REF, InfltWinSZ}), %% we use this counter because we might deal with batches as %% elements. - inflight_append(TableId, {?SIZE_REF, 0}, Id, Index), - inflight_append(TableId, {?BATCH_COUNT_REF, 0}, Id, Index), - inflight_append(TableId, {?INITIAL_TIME_REF, erlang:system_time()}, Id, Index), - inflight_append( - TableId, {?INITIAL_MONOTONIC_TIME_REF, make_request_ref()}, Id, Index - ), + inflight_append(TableId, {?SIZE_REF, 0}), + inflight_append(TableId, {?BATCH_COUNT_REF, 0}), + inflight_append(TableId, {?INITIAL_TIME_REF, erlang:system_time()}), + inflight_append(TableId, {?INITIAL_MONOTONIC_TIME_REF, make_request_ref()}), TableId. -spec inflight_get_first_retriable(ets:tid(), integer()) -> @@ -1290,7 +1408,7 @@ inflight_new(InfltWinSZ, Id, Index) -> inflight_get_first_retriable(InflightTID, Now) -> MatchSpec = ets:fun2ms( - fun(?INFLIGHT_ITEM(Ref, BatchOrQuery, IsRetriable, _WorkerMRef)) when + fun(?INFLIGHT_ITEM(Ref, BatchOrQuery, IsRetriable, _AsyncWorkerMRef)) when IsRetriable =:= true -> {Ref, BatchOrQuery} @@ -1331,38 +1449,32 @@ inflight_num_msgs(InflightTID) -> [{_, Size}] = ets:lookup(InflightTID, ?SIZE_REF), Size. -inflight_append(undefined, _InflightItem, _Id, _Index) -> +inflight_append(undefined, _InflightItem) -> ok; inflight_append( InflightTID, - ?INFLIGHT_ITEM(Ref, [?QUERY(_, _, _, _) | _] = Batch0, IsRetriable, WorkerMRef), - Id, - Index + ?INFLIGHT_ITEM(Ref, [?QUERY(_, _, _, _) | _] = Batch0, IsRetriable, AsyncWorkerMRef) ) -> Batch = mark_as_sent(Batch0), - InflightItem = ?INFLIGHT_ITEM(Ref, Batch, IsRetriable, WorkerMRef), + InflightItem = ?INFLIGHT_ITEM(Ref, Batch, IsRetriable, AsyncWorkerMRef), IsNew = ets:insert_new(InflightTID, InflightItem), BatchSize = length(Batch), IsNew andalso inc_inflight(InflightTID, BatchSize), - emqx_resource_metrics:inflight_set(Id, Index, inflight_num_msgs(InflightTID)), ?tp(buffer_worker_appended_to_inflight, #{item => InflightItem, is_new => IsNew}), ok; inflight_append( InflightTID, ?INFLIGHT_ITEM( - Ref, ?QUERY(_ReplyTo, _Req, _HasBeenSent, _ExpireAt) = Query0, IsRetriable, WorkerMRef - ), - Id, - Index + Ref, ?QUERY(_ReplyTo, _Req, _HasBeenSent, _ExpireAt) = Query0, IsRetriable, AsyncWorkerMRef + ) ) -> Query = mark_as_sent(Query0), - InflightItem = ?INFLIGHT_ITEM(Ref, Query, IsRetriable, WorkerMRef), + InflightItem = ?INFLIGHT_ITEM(Ref, Query, IsRetriable, AsyncWorkerMRef), IsNew = ets:insert_new(InflightTID, InflightItem), IsNew andalso inc_inflight(InflightTID, 1), - emqx_resource_metrics:inflight_set(Id, Index, inflight_num_msgs(InflightTID)), ?tp(buffer_worker_appended_to_inflight, #{item => InflightItem, is_new => IsNew}), ok; -inflight_append(InflightTID, {Ref, Data}, _Id, _Index) -> +inflight_append(InflightTID, {Ref, Data}) -> ets:insert(InflightTID, {Ref, Data}), %% this is a metadata row being inserted; therefore, we don't bump %% the inflight metric. @@ -1380,71 +1492,67 @@ mark_inflight_as_retriable(InflightTID, Ref) -> %% Track each worker pid only once. ensure_async_worker_monitored( - Data0 = #{async_workers := AsyncWorkers}, {async_return, {ok, WorkerPid}} = _Result + Data0 = #{async_workers := AsyncWorkers}, {async_return, {ok, AsyncWorkerPid}} = _Result ) when - is_pid(WorkerPid), is_map_key(WorkerPid, AsyncWorkers) + is_pid(AsyncWorkerPid), is_map_key(AsyncWorkerPid, AsyncWorkers) -> - WorkerMRef = maps:get(WorkerPid, AsyncWorkers), - {Data0, WorkerMRef}; + AsyncWorkerMRef = maps:get(AsyncWorkerPid, AsyncWorkers), + {Data0, AsyncWorkerMRef}; ensure_async_worker_monitored( - Data0 = #{async_workers := AsyncWorkers0}, {async_return, {ok, WorkerPid}} + Data0 = #{async_workers := AsyncWorkers0}, {async_return, {ok, AsyncWorkerPid}} ) when - is_pid(WorkerPid) + is_pid(AsyncWorkerPid) -> - WorkerMRef = monitor(process, WorkerPid), - AsyncWorkers = AsyncWorkers0#{WorkerPid => WorkerMRef}, + AsyncWorkerMRef = monitor(process, AsyncWorkerPid), + AsyncWorkers = AsyncWorkers0#{AsyncWorkerPid => AsyncWorkerMRef}, Data = Data0#{async_workers := AsyncWorkers}, - {Data, WorkerMRef}; + {Data, AsyncWorkerMRef}; ensure_async_worker_monitored(Data0, _Result) -> {Data0, undefined}. -store_async_worker_reference(undefined = _InflightTID, _Ref, _WorkerMRef) -> +-spec store_async_worker_reference(undefined | ets:tid(), inflight_key(), undefined | reference()) -> + ok. +store_async_worker_reference(undefined = _InflightTID, _Ref, _AsyncWorkerMRef) -> ok; store_async_worker_reference(_InflightTID, _Ref, undefined = _WorkerRef) -> ok; -store_async_worker_reference(InflightTID, Ref, WorkerMRef) when - is_reference(WorkerMRef) +store_async_worker_reference(InflightTID, Ref, AsyncWorkerMRef) when + is_reference(AsyncWorkerMRef) -> _ = ets:update_element( - InflightTID, Ref, {?WORKER_MREF_IDX, WorkerMRef} + InflightTID, Ref, {?WORKER_MREF_IDX, AsyncWorkerMRef} ), ok. -ack_inflight(undefined, _Ref, _Id, _Index, _WorkerPid) -> +ack_inflight(undefined, _Ref, _BufferWorkerPid) -> false; -ack_inflight(InflightTID, Ref, Id, Index, WorkerPid) -> +ack_inflight(InflightTID, Ref, BufferWorkerPid) -> {Count, Removed} = case ets:take(InflightTID, Ref) of - [?INFLIGHT_ITEM(Ref, ?QUERY(_, _, _, _), _IsRetriable, _WorkerMRef)] -> + [?INFLIGHT_ITEM(Ref, ?QUERY(_, _, _, _), _IsRetriable, _AsyncWorkerMRef)] -> {1, true}; - [?INFLIGHT_ITEM(Ref, [?QUERY(_, _, _, _) | _] = Batch, _IsRetriable, _WorkerMRef)] -> + [?INFLIGHT_ITEM(Ref, [?QUERY(_, _, _, _) | _] = Batch, _IsRetriable, _AsyncWorkerMRef)] -> {length(Batch), true}; [] -> {0, false} end, FlushCheck = dec_inflight_remove(InflightTID, Count, Removed), case FlushCheck of - continue -> ok; - flush -> ?MODULE:flush_worker(WorkerPid) + no_flush -> ok; + flush -> ?MODULE:flush_worker(BufferWorkerPid) end, IsKnownRef = (Count > 0), - case IsKnownRef of - true -> - emqx_resource_metrics:inflight_set(Id, Index, inflight_num_msgs(InflightTID)); - false -> - ok - end, IsKnownRef. -mark_inflight_items_as_retriable(Data, WorkerMRef) -> +mark_inflight_items_as_retriable(Data, AsyncWorkerMRef) -> #{inflight_tid := InflightTID} = Data, IsRetriable = true, MatchSpec = ets:fun2ms( - fun(?INFLIGHT_ITEM(Ref, BatchOrQuery, _IsRetriable, WorkerMRef0)) when - WorkerMRef =:= WorkerMRef0 + fun(?INFLIGHT_ITEM(Ref, BatchOrQuery, _IsRetriable, AsyncWorkerMRef0)) when + AsyncWorkerMRef =:= AsyncWorkerMRef0 -> - ?INFLIGHT_ITEM(Ref, BatchOrQuery, IsRetriable, WorkerMRef0) + ?INFLIGHT_ITEM(Ref, BatchOrQuery, IsRetriable, AsyncWorkerMRef0) end ), _NumAffected = ets:select_replace(InflightTID, MatchSpec), @@ -1462,9 +1570,9 @@ inc_inflight(InflightTID, Count) -> ok. -spec dec_inflight_remove(undefined | ets:tid(), non_neg_integer(), Removed :: boolean()) -> - continue | flush. + no_flush | flush. dec_inflight_remove(_InflightTID, _Count = 0, _Removed = false) -> - continue; + no_flush; dec_inflight_remove(InflightTID, _Count = 0, _Removed = true) -> NewValue = ets:update_counter(InflightTID, ?BATCH_COUNT_REF, {2, -1, 0, 0}), MaxValue = emqx_utils_ets:lookup_value(InflightTID, ?MAX_SIZE_REF, 0), @@ -1473,7 +1581,7 @@ dec_inflight_remove(InflightTID, _Count = 0, _Removed = true) -> %% make it continue flushing. case NewValue =:= MaxValue - 1 of true -> flush; - false -> continue + false -> no_flush end; dec_inflight_remove(InflightTID, Count, _Removed = true) when Count > 0 -> %% If Count > 0, it must have been removed @@ -1485,7 +1593,7 @@ dec_inflight_remove(InflightTID, Count, _Removed = true) when Count > 0 -> %% make it continue flushing. case NewValue =:= MaxValue - 1 of true -> flush; - false -> continue + false -> no_flush end. dec_inflight_update(_InflightTID, _Count = 0) -> @@ -1496,16 +1604,6 @@ dec_inflight_update(InflightTID, Count) when Count > 0 -> %%============================================================================== -inc_sent_failed(Id, _HasBeenSent = true) -> - emqx_resource_metrics:retried_failed_inc(Id); -inc_sent_failed(Id, _HasBeenSent) -> - emqx_resource_metrics:failed_inc(Id). - -inc_sent_success(Id, _HasBeenSent = true) -> - emqx_resource_metrics:retried_success_inc(Id); -inc_sent_success(Id, _HasBeenSent) -> - emqx_resource_metrics:success_inc(Id). - call_mode(force_sync, _) -> sync; call_mode(async_if_possible, always_sync) -> sync; call_mode(async_if_possible, async_if_possible) -> async. diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 6d70422cf..fa8d53903 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -504,8 +504,10 @@ stop_resource(#data{state = ResState, id = ResId} = Data) -> %% We don't care the return value of the Mod:on_stop/2. %% The callback mod should make sure the resource is stopped after on_stop/2 %% is returned. - case ResState /= undefined of + HasAllocatedResources = emqx_resource:has_allocated_resources(ResId), + case ResState =/= undefined orelse HasAllocatedResources of true -> + %% we clear the allocated resources after stop is successful emqx_resource:call_stop(Data#data.id, Data#data.mod, ResState); false -> ok diff --git a/apps/emqx_resource/src/emqx_resource_manager_sup.erl b/apps/emqx_resource/src/emqx_resource_manager_sup.erl index 73f1988c6..9e86e6363 100644 --- a/apps/emqx_resource/src/emqx_resource_manager_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_manager_sup.erl @@ -17,6 +17,8 @@ -behaviour(supervisor). +-include("emqx_resource.hrl"). + -export([ensure_child/5, delete_child/1]). -export([start_link/0]). @@ -36,6 +38,12 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> + %% Maps resource_id() to one or more allocated resources. + emqx_utils_ets:new(?RESOURCE_ALLOCATION_TAB, [ + bag, + public, + {read_concurrency, true} + ]), ChildSpecs = [ #{ id => emqx_resource_manager, diff --git a/apps/emqx_resource/src/emqx_resource_metrics.erl b/apps/emqx_resource/src/emqx_resource_metrics.erl index 28507e291..df28d893b 100644 --- a/apps/emqx_resource/src/emqx_resource_metrics.erl +++ b/apps/emqx_resource/src/emqx_resource_metrics.erl @@ -206,6 +206,8 @@ inflight_get(ID) -> dropped_inc(ID) -> dropped_inc(ID, 1). +dropped_inc(_ID, 0) -> + ok; dropped_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, dropped], #{counter_inc => Val}, #{resource_id => ID}). @@ -216,6 +218,8 @@ dropped_get(ID) -> dropped_other_inc(ID) -> dropped_other_inc(ID, 1). +dropped_other_inc(_ID, 0) -> + ok; dropped_other_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, dropped_other], #{counter_inc => Val}, #{ resource_id => ID @@ -228,6 +232,8 @@ dropped_other_get(ID) -> dropped_expired_inc(ID) -> dropped_expired_inc(ID, 1). +dropped_expired_inc(_ID, 0) -> + ok; dropped_expired_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, dropped_expired], #{counter_inc => Val}, #{ resource_id => ID @@ -240,6 +246,8 @@ dropped_expired_get(ID) -> late_reply_inc(ID) -> late_reply_inc(ID, 1). +late_reply_inc(_ID, 0) -> + ok; late_reply_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, late_reply], #{counter_inc => Val}, #{ resource_id => ID @@ -252,6 +260,8 @@ late_reply_get(ID) -> dropped_queue_full_inc(ID) -> dropped_queue_full_inc(ID, 1). +dropped_queue_full_inc(_ID, 0) -> + ok; dropped_queue_full_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, dropped_queue_full], #{counter_inc => Val}, #{ resource_id => ID @@ -264,6 +274,8 @@ dropped_queue_full_get(ID) -> dropped_resource_not_found_inc(ID) -> dropped_resource_not_found_inc(ID, 1). +dropped_resource_not_found_inc(_ID, 0) -> + ok; dropped_resource_not_found_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, dropped_resource_not_found], #{counter_inc => Val}, #{ resource_id => ID @@ -276,6 +288,8 @@ dropped_resource_not_found_get(ID) -> dropped_resource_stopped_inc(ID) -> dropped_resource_stopped_inc(ID, 1). +dropped_resource_stopped_inc(_ID, 0) -> + ok; dropped_resource_stopped_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, dropped_resource_stopped], #{counter_inc => Val}, #{ resource_id => ID @@ -288,6 +302,8 @@ dropped_resource_stopped_get(ID) -> matched_inc(ID) -> matched_inc(ID, 1). +matched_inc(_ID, 0) -> + ok; matched_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, matched], #{counter_inc => Val}, #{resource_id => ID}). @@ -298,6 +314,8 @@ matched_get(ID) -> received_inc(ID) -> received_inc(ID, 1). +received_inc(_ID, 0) -> + ok; received_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, received], #{counter_inc => Val}, #{resource_id => ID}). @@ -308,6 +326,8 @@ received_get(ID) -> retried_inc(ID) -> retried_inc(ID, 1). +retried_inc(_ID, 0) -> + ok; retried_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, retried], #{counter_inc => Val}, #{resource_id => ID}). @@ -318,6 +338,8 @@ retried_get(ID) -> failed_inc(ID) -> failed_inc(ID, 1). +failed_inc(_ID, 0) -> + ok; failed_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, failed], #{counter_inc => Val}, #{resource_id => ID}). @@ -328,6 +350,8 @@ failed_get(ID) -> retried_failed_inc(ID) -> retried_failed_inc(ID, 1). +retried_failed_inc(_ID, 0) -> + ok; retried_failed_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, retried_failed], #{counter_inc => Val}, #{ resource_id => ID @@ -340,6 +364,8 @@ retried_failed_get(ID) -> retried_success_inc(ID) -> retried_success_inc(ID, 1). +retried_success_inc(_ID, 0) -> + ok; retried_success_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, retried_success], #{counter_inc => Val}, #{ resource_id => ID @@ -352,6 +378,8 @@ retried_success_get(ID) -> success_inc(ID) -> success_inc(ID, 1). +success_inc(_ID, 0) -> + ok; success_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, success], #{counter_inc => Val}, #{resource_id => ID}). diff --git a/apps/emqx_resource/src/emqx_resource_pool.erl b/apps/emqx_resource/src/emqx_resource_pool.erl index 913b29c86..ea2240efd 100644 --- a/apps/emqx_resource/src/emqx_resource_pool.erl +++ b/apps/emqx_resource/src/emqx_resource_pool.erl @@ -25,7 +25,12 @@ -include_lib("emqx/include/logger.hrl"). +-ifndef(TEST). -define(HEALTH_CHECK_TIMEOUT, 15000). +-else. +%% make tests faster +-define(HEALTH_CHECK_TIMEOUT, 1000). +-endif. start(Name, Mod, Options) -> case ecpool:start_sup_pool(Name, Mod, Options) of diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 2dd2eda99..8b2a68c4b 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -50,6 +50,7 @@ fields("creation_opts") -> {worker_pool_size, fun worker_pool_size/1}, {health_check_interval, fun health_check_interval/1}, {resume_interval, fun resume_interval/1}, + {metrics_flush_interval, fun metrics_flush_interval/1}, {start_after_created, fun start_after_created/1}, {start_timeout, fun start_timeout/1}, {auto_restart_interval, fun auto_restart_interval/1}, @@ -83,6 +84,11 @@ resume_interval(desc) -> ?DESC("resume_interval"); resume_interval(required) -> false; resume_interval(_) -> undefined. +metrics_flush_interval(type) -> emqx_schema:duration_ms(); +metrics_flush_interval(importance) -> ?IMPORTANCE_HIDDEN; +metrics_flush_interval(required) -> false; +metrics_flush_interval(_) -> undefined. + health_check_interval(type) -> emqx_schema:duration_ms(); health_check_interval(desc) -> ?DESC("health_check_interval"); health_check_interval(default) -> ?HEALTHCHECK_INTERVAL_RAW; diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index fc338b512..b960b0526 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -316,7 +316,11 @@ t_query_counter_async_query(_) -> ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, #{name => test_resource, register => true}, - #{query_mode => async, batch_size => 1} + #{ + query_mode => async, + batch_size => 1, + metrics_flush_interval => 50 + } ), ?assertMatch({ok, 0}, emqx_resource:simple_sync_query(?ID, get_counter)), NMsgs = 1_000, @@ -350,7 +354,11 @@ t_query_counter_async_query(_) -> end ), #{counters := C} = emqx_resource:get_metrics(?ID), - ?assertMatch(#{matched := 1002, 'success' := 1002, 'failed' := 0}, C), + ?retry( + _Sleep = 300, + _Attempts0 = 20, + ?assertMatch(#{matched := 1002, 'success' := 1002, 'failed' := 0}, C) + ), ok = emqx_resource:remove_local(?ID). t_query_counter_async_callback(_) -> @@ -1171,6 +1179,7 @@ t_unblock_only_required_buffer_workers(_) -> #{ query_mode => async, batch_size => 5, + metrics_flush_interval => 50, batch_time => 100 } ), @@ -1219,6 +1228,7 @@ t_retry_batch(_Config) -> batch_size => 5, batch_time => 100, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => 1_000 } ), @@ -1318,6 +1328,7 @@ t_delete_and_re_create_with_same_name(_Config) -> worker_pool_size => NumBufferWorkers, buffer_mode => volatile_offload, buffer_seg_bytes => 100, + metrics_flush_interval => 50, resume_interval => 1_000 } ), @@ -1354,10 +1365,16 @@ t_delete_and_re_create_with_same_name(_Config) -> %% ensure that stuff got enqueued into disk tap_metrics(?LINE), - Queuing1 = emqx_resource_metrics:queuing_get(?ID), - Inflight1 = emqx_resource_metrics:inflight_get(?ID), - ?assert(Queuing1 > 0), - ?assertEqual(2, Inflight1), + ?retry( + _Sleep = 300, + _Attempts0 = 20, + ?assert(emqx_resource_metrics:queuing_get(?ID) > 0) + ), + ?retry( + _Sleep = 300, + _Attempts0 = 20, + ?assertEqual(2, emqx_resource_metrics:inflight_get(?ID)) + ), %% now, we delete the resource process_flag(trap_exit, true), @@ -1409,6 +1426,7 @@ t_always_overflow(_Config) -> batch_size => 1, worker_pool_size => 1, max_buffer_bytes => 1, + metrics_flush_interval => 50, resume_interval => 1_000 } ), @@ -1446,6 +1464,7 @@ t_retry_sync_inflight(_Config) -> query_mode => sync, batch_size => 1, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => ResumeInterval } ), @@ -1496,6 +1515,7 @@ t_retry_sync_inflight_batch(_Config) -> batch_size => 2, batch_time => 200, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => ResumeInterval } ), @@ -1546,6 +1566,7 @@ t_retry_async_inflight(_Config) -> query_mode => async, batch_size => 1, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => ResumeInterval } ), @@ -1590,6 +1611,7 @@ t_retry_async_inflight_full(_Config) -> inflight_window => AsyncInflightWindow, batch_size => 1, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => ResumeInterval } ), @@ -1653,6 +1675,7 @@ t_async_reply_multi_eval(_Config) -> batch_size => 3, batch_time => 10, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => ResumeInterval } ), @@ -1667,7 +1690,7 @@ t_async_reply_multi_eval(_Config) -> #{} ), ?retry( - ResumeInterval, + 2 * ResumeInterval, TotalTime div ResumeInterval, begin Metrics = tap_metrics(?LINE), @@ -1683,7 +1706,7 @@ t_async_reply_multi_eval(_Config) -> failed := Failed } = Counters, ?assertEqual(TotalQueries, Matched - 1), - ?assertEqual(Matched, Success + Dropped + LateReply + Failed) + ?assertEqual(Matched, Success + Dropped + LateReply + Failed, #{counters => Counters}) end ). @@ -1700,6 +1723,7 @@ t_retry_async_inflight_batch(_Config) -> batch_size => 2, batch_time => 200, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => ResumeInterval } ), @@ -1745,6 +1769,7 @@ t_async_pool_worker_death(_Config) -> query_mode => async, batch_size => 1, worker_pool_size => NumBufferWorkers, + metrics_refresh_interval => 50, resume_interval => ResumeInterval } ), @@ -1768,8 +1793,11 @@ t_async_pool_worker_death(_Config) -> inc_counter_in_parallel_increasing(NumReqs, 1, ReqOpts), {ok, _} = snabbkaffe:receive_events(SRef0), - Inflight0 = emqx_resource_metrics:inflight_get(?ID), - ?assertEqual(NumReqs, Inflight0), + ?retry( + _Sleep = 300, + _Attempts0 = 20, + ?assertEqual(NumReqs, emqx_resource_metrics:inflight_get(?ID)) + ), %% grab one of the worker pids and kill it {ok, #{pid := Pid0}} = emqx_resource:simple_sync_query(?ID, get_state), @@ -1820,6 +1848,7 @@ t_expiration_sync_before_sending(_Config) -> query_mode => sync, batch_size => 1, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => 1_000 } ), @@ -1837,6 +1866,7 @@ t_expiration_sync_batch_before_sending(_Config) -> batch_size => 2, batch_time => 100, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => 1_000 } ), @@ -1853,6 +1883,7 @@ t_expiration_async_before_sending(_Config) -> query_mode => async, batch_size => 1, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => 1_000 } ), @@ -1870,6 +1901,7 @@ t_expiration_async_batch_before_sending(_Config) -> batch_size => 2, batch_time => 100, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => 1_000 } ), @@ -1950,6 +1982,7 @@ t_expiration_sync_before_sending_partial_batch(_Config) -> batch_size => 2, batch_time => 100, worker_pool_size => 1, + metrics_flush_interval => 250, resume_interval => 1_000 } ), @@ -1968,6 +2001,7 @@ t_expiration_async_before_sending_partial_batch(_Config) -> batch_size => 2, batch_time => 100, worker_pool_size => 1, + metrics_flush_interval => 250, resume_interval => 1_000 } ), @@ -2057,7 +2091,14 @@ do_t_expiration_before_sending_partial_batch(QueryMode) -> ], ?of_kind(buffer_worker_flush_potentially_partial, Trace) ), - wait_until_gauge_is(inflight, 0, 500), + wait_until_gauge_is( + inflight, + #{ + expected_value => 0, + timeout => 500, + max_events => 10 + } + ), Metrics = tap_metrics(?LINE), case QueryMode of async -> @@ -2933,8 +2974,15 @@ install_telemetry_handler(TestCase) -> put({?MODULE, telemetry_table}, Tid), Tid. -wait_until_gauge_is(GaugeName, ExpectedValue, Timeout) -> - Events = receive_all_events(GaugeName, Timeout), +wait_until_gauge_is( + GaugeName, + #{ + expected_value := ExpectedValue, + timeout := Timeout, + max_events := MaxEvents + } +) -> + Events = receive_all_events(GaugeName, Timeout, MaxEvents), case length(Events) > 0 andalso lists:last(Events) of #{measurements := #{gauge_set := ExpectedValue}} -> ok; @@ -2948,12 +2996,18 @@ wait_until_gauge_is(GaugeName, ExpectedValue, Timeout) -> end. receive_all_events(EventName, Timeout) -> - receive_all_events(EventName, Timeout, []). + receive_all_events(EventName, Timeout, _MaxEvents = 50, _Count = 0, _Acc = []). -receive_all_events(EventName, Timeout, Acc) -> +receive_all_events(EventName, Timeout, MaxEvents) -> + receive_all_events(EventName, Timeout, MaxEvents, _Count = 0, _Acc = []). + +receive_all_events(_EventName, _Timeout, MaxEvents, Count, Acc) when Count >= MaxEvents -> + lists:reverse(Acc); +receive_all_events(EventName, Timeout, MaxEvents, Count, Acc) -> receive {telemetry, #{name := [_, _, EventName]} = Event} -> - receive_all_events(EventName, Timeout, [Event | Acc]) + ct:pal("telemetry event: ~p", [Event]), + receive_all_events(EventName, Timeout, MaxEvents, Count + 1, [Event | Acc]) after Timeout -> lists:reverse(Acc) end. diff --git a/apps/emqx_rule_engine/README.md b/apps/emqx_rule_engine/README.md index 2c2e43db3..3fdd9b6a7 100644 --- a/apps/emqx_rule_engine/README.md +++ b/apps/emqx_rule_engine/README.md @@ -1,5 +1,5 @@ -# Emqx Rule Engine +# EMQX Rule Engine The rule engine's goal is to provide a simple and flexible way to transform and reroute the messages coming to the EMQX broker. For example, one message diff --git a/apps/emqx_rule_engine/include/rule_engine.hrl b/apps/emqx_rule_engine/include/rule_engine.hrl index c69a24244..51ad6ab85 100644 --- a/apps/emqx_rule_engine/include/rule_engine.hrl +++ b/apps/emqx_rule_engine/include/rule_engine.hrl @@ -92,6 +92,8 @@ ?RAISE(EXP, _ = do_nothing, ERROR) ). +-define(RAISE_BAD_SQL(Detail), throw(Detail)). + -define(RAISE(EXP, EXP_ON_FAIL, ERROR), fun() -> try @@ -106,3 +108,16 @@ %% Tables -define(RULE_TAB, emqx_rule_engine). + +%% Allowed sql function provider modules +-define(DEFAULT_SQL_FUNC_PROVIDER, emqx_rule_funcs). +-define(IS_VALID_SQL_FUNC_PROVIDER_MODULE_NAME(Name), + (case Name of + <<"emqx_rule_funcs", _/binary>> -> + true; + <<"EmqxRuleFuncs", _/binary>> -> + true; + _ -> + false + end) +). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 8dc78958a..c6f94f5ea 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.17"}, + {vsn, "5.0.18"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index d36ec6a40..48711d4cc 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -410,17 +410,22 @@ param_path_id() -> {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} end; '/rules/:id'(delete, #{bindings := #{id := Id}}) -> - ConfPath = emqx_rule_engine:config_key_path() ++ [Id], - case emqx_conf:remove(ConfPath, #{override_to => cluster}) of - {ok, _} -> - {204}; - {error, Reason} -> - ?SLOG(error, #{ - msg => "delete_rule_failed", - id => Id, - reason => Reason - }), - {500, #{code => 'INTERNAL_ERROR', message => ?ERR_BADARGS(Reason)}} + case emqx_rule_engine:get_rule(Id) of + {ok, _Rule} -> + ConfPath = emqx_rule_engine:config_key_path() ++ [Id], + case emqx_conf:remove(ConfPath, #{override_to => cluster}) of + {ok, _} -> + {204}; + {error, Reason} -> + ?SLOG(error, #{ + msg => "delete_rule_failed", + id => Id, + reason => Reason + }), + {500, #{code => 'INTERNAL_ERROR', message => ?ERR_BADARGS(Reason)}} + end; + not_found -> + {404, #{code => 'NOT_FOUND', message => <<"Rule Id Not Found">>}} end. '/rules/:id/metrics'(get, #{bindings := #{id := Id}}) -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 5b7f962fb..602a7f91e 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -452,19 +452,23 @@ eval_switch_clauses(CaseOn, [{Cond, Clause} | CaseClauses], ElseClauses, Columns eval_switch_clauses(CaseOn, CaseClauses, ElseClauses, Columns) end. -apply_func(Name, Args, Columns) when is_atom(Name) -> - do_apply_func(Name, Args, Columns); apply_func(Name, Args, Columns) when is_binary(Name) -> - FunName = - try - binary_to_existing_atom(Name, utf8) - catch - error:badarg -> error({sql_function_not_supported, Name}) - end, - do_apply_func(FunName, Args, Columns). + FuncName = parse_function_name(?DEFAULT_SQL_FUNC_PROVIDER, Name), + apply_func(FuncName, Args, Columns); +apply_func([{key, ModuleName0}, {key, FuncName0}], Args, Columns) -> + ModuleName = parse_module_name(ModuleName0), + FuncName = parse_function_name(ModuleName, FuncName0), + do_apply_func(ModuleName, FuncName, Args, Columns); +apply_func(Name, Args, Columns) when is_atom(Name) -> + do_apply_func(?DEFAULT_SQL_FUNC_PROVIDER, Name, Args, Columns); +apply_func(Other, _, _) -> + ?RAISE_BAD_SQL(#{ + reason => bad_sql_function_reference, + reference => Other + }). -do_apply_func(Name, Args, Columns) -> - case erlang:apply(emqx_rule_funcs, Name, Args) of +do_apply_func(Module, Name, Args, Columns) -> + case erlang:apply(Module, Name, Args) of Func when is_function(Func) -> erlang:apply(Func, [Columns]); Result -> @@ -531,3 +535,39 @@ is_ok_result(R) when is_tuple(R) -> ok == erlang:element(1, R); is_ok_result(_) -> false. + +parse_module_name(Name) when is_binary(Name) -> + case ?IS_VALID_SQL_FUNC_PROVIDER_MODULE_NAME(Name) of + true -> + ok; + false -> + ?RAISE_BAD_SQL(#{ + reason => sql_function_provider_module_not_allowed, + module => Name + }) + end, + try + parse_module_name(binary_to_existing_atom(Name, utf8)) + catch + error:badarg -> + ?RAISE_BAD_SQL(#{ + reason => sql_function_provider_module_not_loaded, + module => Name + }) + end; +parse_module_name(Name) when is_atom(Name) -> + Name. + +parse_function_name(Module, Name) when is_binary(Name) -> + try + parse_function_name(Module, binary_to_existing_atom(Name, utf8)) + catch + error:badarg -> + ?RAISE_BAD_SQL(#{ + reason => sql_function_not_supported, + module => Module, + function => Name + }) + end; +parse_function_name(_Module, Name) when is_atom(Name) -> + Name. diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl b/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl index 9b6ed7eae..b6661684f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl @@ -44,11 +44,23 @@ -type alias() :: binary() | list(binary()). --type field() :: - const() - | variable() - | {as, field(), alias()} - | {'fun', atom(), list(field())}. +%% TODO: So far the SQL function module names and function names are as binary(), +%% binary_to_atom is called to convert to module and function name. +%% For better performance, the function references +%% can be converted to a fun Module:Function/N When compiling the SQL. +-type ext_module_name() :: atom() | binary(). +-type func_name() :: atom() | binary(). +-type func_args() :: [field()]. +%% Functions defiend in emqx_rule_funcs +-type builtin_func_ref() :: {var, func_name()}. +%% Functions defined in other modules, reference syntax: Module.Function(Arg1, Arg2, ...) +%% NOTE: it's '.' (Elixir style), but not ':' (Erlang style). +%% Parsed as a two element path-list: [{key, Module}, {key, Func}]. +-type external_func_ref() :: {path, [{key, ext_module_name() | func_name()}]}. +-type func_ref() :: builtin_func_ref() | external_func_ref(). +-type sql_func() :: {'fun', func_ref(), func_args()}. + +-type field() :: const() | variable() | {as, field(), alias()} | sql_func(). -export_type([select/0]). diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 9c3e5513a..e88529ad1 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -62,6 +62,9 @@ groups() -> t_match_atom_and_binary, t_sqlselect_0, t_sqlselect_00, + t_sqlselect_with_3rd_party_impl, + t_sqlselect_with_3rd_party_impl2, + t_sqlselect_with_3rd_party_funcs_unknown, t_sqlselect_001, t_sqlselect_inject_props, t_sqlselect_01, @@ -120,6 +123,8 @@ groups() -> %%------------------------------------------------------------------------------ init_per_suite(Config) -> + %% ensure module loaded + emqx_rule_funcs_demo:module_info(), application:load(emqx_conf), ok = emqx_common_test_helpers:start_apps( [emqx_conf, emqx_rule_engine, emqx_authz], @@ -1012,6 +1017,60 @@ t_sqlselect_00(_Config) -> ) ). +t_sqlselect_with_3rd_party_impl(_Config) -> + Sql = + "select * from \"t/#\" where emqx_rule_funcs_demo.is_my_topic(topic)", + T = fun(Topic) -> + emqx_rule_sqltester:test( + #{ + sql => Sql, + context => + #{ + payload => #{<<"what">> => 0}, + topic => Topic + } + } + ) + end, + ?assertMatch({ok, _}, T(<<"t/2/3/4/5">>)), + ?assertMatch({error, nomatch}, T(<<"t/1">>)). + +t_sqlselect_with_3rd_party_impl2(_Config) -> + Sql = fun(N) -> + "select emqx_rule_funcs_demo.duplicate_payload(payload," ++ integer_to_list(N) ++ + ") as payload_list from \"t/#\"" + end, + T = fun(Payload, N) -> + emqx_rule_sqltester:test( + #{ + sql => Sql(N), + context => + #{ + payload => Payload, + topic => <<"t/a">> + } + } + ) + end, + ?assertMatch({ok, #{<<"payload_list">> := [_, _]}}, T(<<"payload1">>, 2)), + ?assertMatch({ok, #{<<"payload_list">> := [_, _, _]}}, T(<<"payload1">>, 3)), + %% crash + ?assertMatch({error, {select_and_transform_error, _}}, T(<<"payload1">>, 4)). + +t_sqlselect_with_3rd_party_funcs_unknown(_Config) -> + Sql = "select emqx_rule_funcs_demo_no_such_module.foo(payload) from \"t/#\"", + ?assertMatch( + {error, + {select_and_transform_error, + {throw, #{reason := sql_function_provider_module_not_loaded}, _}}}, + emqx_rule_sqltester:test( + #{ + sql => Sql, + context => #{payload => <<"a">>, topic => <<"t/a">>} + } + ) + ). + t_sqlselect_001(_Config) -> %% Verify that the jq function can be called from SQL Sql = diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl index 8d7546fca..ccee05604 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl @@ -120,7 +120,14 @@ t_crud_rule_api(_Config) -> ) ), - %ct:pal("Show After Deleted: ~p", [NotFound]), + ?assertMatch( + {404, #{code := 'NOT_FOUND'}}, + emqx_rule_engine_api:'/rules/:id'( + delete, + #{bindings => #{id => RuleId}} + ) + ), + ?assertMatch( {404, #{code := _, message := _Message}}, emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleId}}) diff --git a/apps/emqx_statsd/src/emqx_statsd_app.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_demo.erl similarity index 62% rename from apps/emqx_statsd/src/emqx_statsd_app.erl rename to apps/emqx_rule_engine/test/emqx_rule_funcs_demo.erl index 0090c8465..b0d42b10e 100644 --- a/apps/emqx_statsd/src/emqx_statsd_app.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_demo.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -14,21 +14,19 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_statsd_app). - --behaviour(application). - --include("emqx_statsd.hrl"). +-module(emqx_rule_funcs_demo). -export([ - start/2, - stop/1 + is_my_topic/1, + duplicate_payload/2 ]). -start(_StartType, _StartArgs) -> - {ok, Sup} = emqx_statsd_sup:start_link(), - emqx_statsd_config:add_handler(), - {ok, Sup}. -stop(_) -> - emqx_statsd_config:remove_handler(), - ok. +%% check if the topic is of 5 levels. +is_my_topic(Topic) -> + emqx_topic:levels(Topic) =:= 5. + +%% duplicate the payload, but only supports 2 or 3 copies. +duplicate_payload(Payload, 2) -> + [Payload, Payload]; +duplicate_payload(Payload, 3) -> + [Payload, Payload, Payload]. diff --git a/apps/emqx_s3/src/emqx_s3.app.src b/apps/emqx_s3/src/emqx_s3.app.src index 7864ffb29..6d0518769 100644 --- a/apps/emqx_s3/src/emqx_s3.app.src +++ b/apps/emqx_s3/src/emqx_s3.app.src @@ -1,6 +1,6 @@ {application, emqx_s3, [ {description, "EMQX S3"}, - {vsn, "5.0.6"}, + {vsn, "5.0.7"}, {modules, []}, {registered, [emqx_s3_sup]}, {applications, [ diff --git a/apps/emqx_s3/src/emqx_s3.erl b/apps/emqx_s3/src/emqx_s3.erl index 0c2592736..cc48cdb93 100644 --- a/apps/emqx_s3/src/emqx_s3.erl +++ b/apps/emqx_s3/src/emqx_s3.erl @@ -44,7 +44,7 @@ -type profile_config() :: #{ bucket := string(), access_key_id => string(), - secret_access_key => string(), + secret_access_key => emqx_secret:t(string()), host := string(), port := pos_integer(), url_expire_time := pos_integer(), diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index 3bc5861c6..c062cf1ca 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -60,7 +60,7 @@ acl := emqx_s3:acl() | undefined, url_expire_time := pos_integer(), access_key_id := string() | undefined, - secret_access_key := string() | undefined, + secret_access_key := emqx_secret:t(string()) | undefined, http_pool := http_pool(), pool_type := pool_type(), request_timeout := timeout() | undefined, @@ -230,7 +230,7 @@ aws_config(#{ s3_bucket_after_host = true, access_key_id = AccessKeyId, - secret_access_key = SecretAccessKey, + secret_access_key = emqx_secret:unwrap(SecretAccessKey), http_client = request_fun( HttpPool, PoolType, with_default(MaxRetries, ?DEFAULT_MAX_RETRIES) diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index 87f006bcb..a449640a6 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -11,7 +11,7 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). --include("src/emqx_s3.hrl"). +-include("emqx_s3.hrl"). -export([ start_link/2, @@ -377,10 +377,10 @@ stop_http_pool(ProfileId, PoolName) -> ok = ?tp(debug, "s3_stop_http_pool", #{pool_name => PoolName}). do_start_http_pool(PoolName, HttpConfig) -> - ?SLOG(warning, #{msg => "s3_start_http_pool", pool_name => PoolName, config => HttpConfig}), + ?SLOG(debug, #{msg => "s3_starting_http_pool", pool_name => PoolName, config => HttpConfig}), case ehttpc_sup:start_pool(PoolName, HttpConfig) of {ok, _} -> - ?SLOG(warning, #{msg => "s3_start_http_pool_success", pool_name => PoolName}), + ?SLOG(info, #{msg => "s3_start_http_pool_success", pool_name => PoolName}), ok; {error, _} = Error -> ?SLOG(error, #{msg => "s3_start_http_pool_fail", pool_name => PoolName, error => Error}), diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index 5866f8c2b..f02364969 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -34,11 +34,12 @@ fields(s3) -> )}, {secret_access_key, mk( - string(), + hoconsc:union([string(), function()]), #{ desc => ?DESC("secret_access_key"), required => false, - sensitive => true + sensitive => true, + converter => fun secret/2 } )}, {bucket, @@ -124,7 +125,7 @@ fields(transport_options) -> mk( boolean(), #{ - default => true, + default => false, desc => ?DESC("ipv6_probe"), required => false } @@ -142,6 +143,14 @@ desc(s3) -> desc(transport_options) -> "Options for the HTTP transport layer used by the S3 client". +secret(undefined, #{}) -> + undefined; +secret(Secret, #{make_serializable := true}) -> + unicode:characters_to_binary(emqx_secret:unwrap(Secret)); +secret(Secret, #{}) -> + _ = is_binary(Secret) orelse throw({expected_type, string}), + emqx_secret:wrap(unicode:characters_to_list(Secret)). + translate(Conf) -> translate(Conf, #{}). diff --git a/apps/emqx_s3/src/emqx_s3_uploader.erl b/apps/emqx_s3/src/emqx_s3_uploader.erl index 595612f62..aa547c7cc 100644 --- a/apps/emqx_s3/src/emqx_s3_uploader.erl +++ b/apps/emqx_s3/src/emqx_s3_uploader.erl @@ -18,7 +18,9 @@ complete/2, abort/1, - abort/2 + abort/2, + + shutdown/1 ]). -export([ @@ -87,6 +89,11 @@ abort(Pid) -> abort(Pid, Timeout) -> gen_statem:call(Pid, abort, Timeout). +-spec shutdown(pid()) -> ok. +shutdown(Pid) -> + _ = erlang:exit(Pid, shutdown), + ok. + %%-------------------------------------------------------------------- %% gen_statem callbacks %%-------------------------------------------------------------------- diff --git a/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl b/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl index 63f659da0..3c7753857 100644 --- a/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl @@ -49,7 +49,7 @@ t_full_config(_Config) -> host := "s3.us-east-1.endpoint.com", min_part_size := 10485760, port := 443, - secret_access_key := "secret_access_key", + secret_access_key := Secret, transport_options := #{ connect_timeout := 30000, @@ -74,7 +74,7 @@ t_full_config(_Config) -> versions := ['tlsv1.2'] } } - }, + } when is_function(Secret), emqx_s3_schema:translate(#{ <<"access_key_id">> => <<"access_key_id">>, <<"secret_access_key">> => <<"secret_access_key">>, @@ -126,6 +126,26 @@ t_sensitive_config_hidden(_Config) -> ) ). +t_sensitive_config_no_leak(_Config) -> + ?assertThrow( + {emqx_s3_schema, [ + Error = #{ + kind := validation_error, + path := "s3.secret_access_key", + reason := {expected_type, string} + } + ]} when map_size(Error) == 3, + emqx_s3_schema:translate( + #{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"access_key_id">> => <<"access_key_id">>, + <<"secret_access_key">> => #{<<"1">> => <<"secret_access_key">>} + } + ) + ). + t_invalid_limits(_Config) -> ?assertException( throw, diff --git a/apps/emqx_statsd/README.md b/apps/emqx_statsd/README.md deleted file mode 100644 index f9097a58e..000000000 --- a/apps/emqx_statsd/README.md +++ /dev/null @@ -1,9 +0,0 @@ -emqx_statsd -===== - -An OTP application - -Build ------ - - $ rebar3 compile diff --git a/apps/emqx_statsd/rebar.config b/apps/emqx_statsd/rebar.config deleted file mode 100644 index a1383d920..000000000 --- a/apps/emqx_statsd/rebar.config +++ /dev/null @@ -1,15 +0,0 @@ -%% -*- mode: erlang -*- - -{erl_opts, [debug_info]}. -{deps, [ - {emqx, {path, "../emqx"}}, - {emqx_utils, {path, "../emqx_utils"}}, - {estatsd, {git, "https://github.com/emqx/estatsd", {tag, "0.1.0"}}} -]}. - -{shell, [ - % {config, "config/sys.config"}, - {apps, [emqx_statsd]} -]}. - -{project_plugins, [erlfmt]}. diff --git a/apps/emqx_statsd/src/emqx_statsd.app.src b/apps/emqx_statsd/src/emqx_statsd.app.src deleted file mode 100644 index 87fc8c596..000000000 --- a/apps/emqx_statsd/src/emqx_statsd.app.src +++ /dev/null @@ -1,18 +0,0 @@ -%% -*- mode: erlang -*- -{application, emqx_statsd, [ - {description, "EMQX Statsd"}, - {vsn, "5.0.9"}, - {registered, []}, - {mod, {emqx_statsd_app, []}}, - {applications, [ - kernel, - stdlib, - estatsd, - emqx, - emqx_management - ]}, - {env, []}, - {modules, []}, - {licenses, ["Apache 2.0"]}, - {links, []} -]}. diff --git a/apps/emqx_statsd/src/emqx_statsd.erl b/apps/emqx_statsd/src/emqx_statsd.erl deleted file mode 100644 index b2d726b07..000000000 --- a/apps/emqx_statsd/src/emqx_statsd.erl +++ /dev/null @@ -1,168 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_statsd). - --behaviour(gen_server). - --ifdef(TEST). --compile(export_all). --compile(nowarn_export_all). --endif. - --include("emqx_statsd.hrl"). - --include_lib("emqx/include/logger.hrl"). - --export([ - start/0, - stop/0, - restart/0, - %% for rpc: remove after 5.1.x - do_start/0, - do_stop/0, - do_restart/0 -]). - -%% Interface --export([start_link/1]). - -%% Internal Exports --export([ - init/1, - handle_call/3, - handle_cast/2, - handle_info/2, - code_change/3, - terminate/2 -]). - --define(SAMPLE_TIMEOUT, sample_timeout). - -%% Remove after 5.1.x -start() -> check_multicall_result(emqx_statsd_proto_v1:start(mria:running_nodes())). -stop() -> check_multicall_result(emqx_statsd_proto_v1:stop(mria:running_nodes())). -restart() -> check_multicall_result(emqx_statsd_proto_v1:restart(mria:running_nodes())). - -do_start() -> - emqx_statsd_sup:ensure_child_started(?APP). - -do_stop() -> - emqx_statsd_sup:ensure_child_stopped(?APP). - -do_restart() -> - ok = do_stop(), - ok = do_start(), - ok. - -start_link(Conf) -> - gen_server:start_link({local, ?MODULE}, ?MODULE, Conf, []). - -init(Conf) -> - process_flag(trap_exit, true), - #{ - tags := TagsRaw, - server := Server, - sample_time_interval := SampleTimeInterval, - flush_time_interval := FlushTimeInterval - } = Conf, - FlushTimeInterval1 = flush_interval(FlushTimeInterval, SampleTimeInterval), - #{hostname := Host, port := Port} = emqx_schema:parse_server(Server, ?SERVER_PARSE_OPTS), - Tags = maps:fold(fun(K, V, Acc) -> [{to_bin(K), to_bin(V)} | Acc] end, [], TagsRaw), - Opts = [{tags, Tags}, {host, Host}, {port, Port}, {prefix, <<"emqx">>}], - {ok, Pid} = estatsd:start_link(Opts), - {ok, - ensure_timer(#{ - sample_time_interval => SampleTimeInterval, - flush_time_interval => FlushTimeInterval1, - estatsd_pid => Pid - })}. - -handle_call(_Req, _From, State) -> - {reply, ignore, State}. - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info( - {timeout, Ref, ?SAMPLE_TIMEOUT}, - State = #{ - sample_time_interval := SampleTimeInterval, - flush_time_interval := FlushTimeInterval, - estatsd_pid := Pid, - timer := Ref - } -) -> - Metrics = emqx_metrics:all() ++ emqx_stats:getstats() ++ emqx_mgmt:vm_stats(), - SampleRate = SampleTimeInterval / FlushTimeInterval, - StatsdMetrics = [ - {gauge, Name, Value, SampleRate, []} - || {Name, Value} <- Metrics - ], - ok = estatsd:submit(Pid, StatsdMetrics), - {noreply, ensure_timer(State), hibernate}; -handle_info({'EXIT', Pid, Error}, State = #{estatsd_pid := Pid}) -> - {stop, {shutdown, Error}, State}; -handle_info(_Msg, State) -> - {noreply, State}. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -terminate(_Reason, #{estatsd_pid := Pid}) -> - estatsd:stop(Pid), - ok. - -%%------------------------------------------------------------------------------ -%% Internal function -%%------------------------------------------------------------------------------ - -flush_interval(FlushInterval, SampleInterval) when FlushInterval >= SampleInterval -> - FlushInterval; -flush_interval(_FlushInterval, SampleInterval) -> - ?SLOG( - warning, - #{ - msg => - "Configured flush_time_interval is lower than sample_time_interval, " - "setting: flush_time_interval = sample_time_interval." - } - ), - SampleInterval. - -ensure_timer(State = #{sample_time_interval := SampleTimeInterval}) -> - State#{timer => emqx_utils:start_timer(SampleTimeInterval, ?SAMPLE_TIMEOUT)}. - -check_multicall_result({Results, []}) -> - case - lists:all( - fun - (ok) -> true; - (_) -> false - end, - Results - ) - of - true -> ok; - false -> error({bad_result, Results}) - end; -check_multicall_result({_, _}) -> - error(multicall_failed). - -to_bin(B) when is_binary(B) -> B; -to_bin(I) when is_integer(I) -> integer_to_binary(I); -to_bin(L) when is_list(L) -> list_to_binary(L); -to_bin(A) when is_atom(A) -> atom_to_binary(A, utf8). diff --git a/apps/emqx_statsd/src/emqx_statsd_api.erl b/apps/emqx_statsd/src/emqx_statsd_api.erl deleted file mode 100644 index 4ee144d57..000000000 --- a/apps/emqx_statsd/src/emqx_statsd_api.erl +++ /dev/null @@ -1,97 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_statsd_api). - --behaviour(minirest_api). - --include("emqx_statsd.hrl"). - --include_lib("hocon/include/hoconsc.hrl"). --include_lib("typerefl/include/types.hrl"). - --import(hoconsc, [mk/2, ref/2]). - --export([statsd/2]). - --export([ - api_spec/0, - paths/0, - schema/1 -]). - --define(API_TAG_STATSD, [<<"Monitor">>]). --define(SCHEMA_MODULE, emqx_statsd_schema). - --define(INTERNAL_ERROR, 'INTERNAL_ERROR'). - -api_spec() -> - emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). - -paths() -> - ["/statsd"]. - -schema("/statsd") -> - #{ - 'operationId' => statsd, - get => - #{ - deprecated => true, - description => ?DESC(get_statsd_config_api), - tags => ?API_TAG_STATSD, - responses => - #{200 => statsd_config_schema()} - }, - put => - #{ - deprecated => true, - description => ?DESC(update_statsd_config_api), - tags => ?API_TAG_STATSD, - 'requestBody' => statsd_config_schema(), - responses => - #{200 => statsd_config_schema()} - } - }. - -%%-------------------------------------------------------------------- -%% Helper funcs -%%-------------------------------------------------------------------- - -statsd_config_schema() -> - emqx_dashboard_swagger:schema_with_example( - ref(?SCHEMA_MODULE, "statsd"), - statsd_example() - ). - -statsd_example() -> - #{ - enable => true, - flush_time_interval => <<"30s">>, - sample_time_interval => <<"30s">>, - server => <<"127.0.0.1:8125">>, - tags => #{} - }. - -statsd(get, _Params) -> - {200, emqx:get_raw_config([<<"statsd">>], #{})}; -statsd(put, #{body := Body}) -> - case emqx_statsd_config:update(Body) of - {ok, NewConfig} -> - {200, NewConfig}; - {error, Reason} -> - Message = list_to_binary(io_lib:format("Update config failed ~p", [Reason])), - {500, ?INTERNAL_ERROR, Message} - end. diff --git a/apps/emqx_statsd/src/emqx_statsd_config.erl b/apps/emqx_statsd/src/emqx_statsd_config.erl deleted file mode 100644 index 6bc430956..000000000 --- a/apps/emqx_statsd/src/emqx_statsd_config.erl +++ /dev/null @@ -1,54 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- --module(emqx_statsd_config). - --behaviour(emqx_config_handler). - --include("emqx_statsd.hrl"). - --export([add_handler/0, remove_handler/0]). --export([post_config_update/5]). --export([update/1]). - -update(Config) -> - case - emqx_conf:update( - ?STATSD, - Config, - #{rawconf_with_defaults => true, override_to => cluster} - ) - of - {ok, #{raw_config := NewConfigRows}} -> - {ok, NewConfigRows}; - {error, Reason} -> - {error, Reason} - end. - -add_handler() -> - ok = emqx_config_handler:add_handler(?STATSD, ?MODULE), - ok. - -remove_handler() -> - ok = emqx_config_handler:remove_handler(?STATSD), - ok. - -post_config_update(?STATSD, _Req, #{enable := true} = New, _Old, _AppEnvs) -> - emqx_statsd_sup:ensure_child_stopped(?APP), - emqx_statsd_sup:ensure_child_started(?APP, New); -post_config_update(?STATSD, _Req, #{enable := false}, _Old, _AppEnvs) -> - emqx_statsd_sup:ensure_child_stopped(?APP); -post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) -> - ok. diff --git a/apps/emqx_statsd/src/emqx_statsd_schema.erl b/apps/emqx_statsd/src/emqx_statsd_schema.erl deleted file mode 100644 index 01decc6f7..000000000 --- a/apps/emqx_statsd/src/emqx_statsd_schema.erl +++ /dev/null @@ -1,96 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_statsd_schema). - --include_lib("hocon/include/hoconsc.hrl"). --include_lib("typerefl/include/types.hrl"). --include("emqx_statsd.hrl"). - --behaviour(hocon_schema). - --export([ - namespace/0, - roots/0, - fields/1, - desc/1, - validations/0 -]). - -namespace() -> "statsd". - -roots() -> - [{"statsd", hoconsc:mk(hoconsc:ref(?MODULE, "statsd"), #{importance => ?IMPORTANCE_HIDDEN})}]. - -fields("statsd") -> - [ - {enable, - hoconsc:mk( - boolean(), - #{ - default => false, - desc => ?DESC(enable) - } - )}, - {server, server()}, - {sample_time_interval, fun sample_interval/1}, - {flush_time_interval, fun flush_interval/1}, - {tags, fun tags/1} - ]. - -desc("statsd") -> ?DESC(statsd); -desc(_) -> undefined. - -server() -> - Meta = #{ - default => <<"127.0.0.1:8125">>, - desc => ?DESC(?FUNCTION_NAME) - }, - emqx_schema:servers_sc(Meta, ?SERVER_PARSE_OPTS). - -sample_interval(type) -> emqx_schema:duration_ms(); -sample_interval(default) -> <<"30s">>; -sample_interval(desc) -> ?DESC(?FUNCTION_NAME); -sample_interval(_) -> undefined. - -flush_interval(type) -> emqx_schema:duration_ms(); -flush_interval(default) -> <<"30s">>; -flush_interval(desc) -> ?DESC(?FUNCTION_NAME); -flush_interval(_) -> undefined. - -tags(type) -> map(); -tags(default) -> #{}; -tags(desc) -> ?DESC(?FUNCTION_NAME); -tags(_) -> undefined. - -validations() -> - [ - {check_interval, fun check_interval/1} - ]. - -check_interval(Conf) -> - case hocon_maps:get("statsd.sample_time_interval", Conf) of - undefined -> - ok; - Sample -> - Flush = hocon_maps:get("statsd.flush_time_interval", Conf), - case Sample =< Flush of - true -> - true; - false -> - {bad_interval, #{sample_time_interval => Sample, flush_time_interval => Flush}} - end - end. diff --git a/apps/emqx_statsd/src/emqx_statsd_sup.erl b/apps/emqx_statsd/src/emqx_statsd_sup.erl deleted file mode 100644 index 35c1d332c..000000000 --- a/apps/emqx_statsd/src/emqx_statsd_sup.erl +++ /dev/null @@ -1,81 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- -%%%------------------------------------------------------------------- -%% @doc emqx_statsd top level supervisor. -%% @end -%%%------------------------------------------------------------------- - --module(emqx_statsd_sup). - --behaviour(supervisor). - --export([ - start_link/0, - ensure_child_started/1, - ensure_child_started/2, - ensure_child_stopped/1 -]). - --export([init/1]). - -%% Helper macro for declaring children of supervisor --define(CHILD(Mod, Opts), #{ - id => Mod, - start => {Mod, start_link, Opts}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [Mod] -}). - -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - --spec ensure_child_started(atom()) -> ok. -ensure_child_started(Mod) when is_atom(Mod) -> - ensure_child_started(Mod, emqx_conf:get([statsd], #{})). - --spec ensure_child_started(atom(), map()) -> ok. -ensure_child_started(Mod, Conf) when is_atom(Mod) -> - assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, [Conf]))). - -%% @doc Stop the child worker process. --spec ensure_child_stopped(any()) -> ok. -ensure_child_stopped(ChildId) -> - case supervisor:terminate_child(?MODULE, ChildId) of - ok -> - %% with terminate_child/2 returned 'ok', it's not possible - %% for supervisor:delete_child/2 to return {error, Reason} - ok = supervisor:delete_child(?MODULE, ChildId); - {error, not_found} -> - ok - end. - -init([]) -> - Children = - case emqx_conf:get([statsd], #{}) of - #{enable := true} = Conf -> [?CHILD(emqx_statsd, [Conf])]; - _ -> [] - end, - {ok, {{one_for_one, 100, 3600}, Children}}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -assert_started({ok, _Pid}) -> ok; -assert_started({error, {already_started, _Pid}}) -> ok; -assert_started({error, Reason}) -> erlang:error(Reason). diff --git a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl deleted file mode 100644 index 8b8709c27..000000000 --- a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl +++ /dev/null @@ -1,204 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_statsd_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("common_test/include/ct.hrl"). --include_lib("eunit/include/eunit.hrl"). --import(emqx_dashboard_api_test_helpers, [request/3, uri/1]). - --define(BASE_CONF, << - "\n" - "statsd {\n" - "enable = true\n" - "flush_time_interval = 4s\n" - "sample_time_interval = 4s\n" - "server = \"127.0.0.1:8126\"\n" - "tags {\"t1\" = \"good\", test = 100}\n" - "}\n" ->>). --define(BAD_CONF, << - "\n" - "statsd {\n" - "enable = true\n" - "flush_time_interval = 4s\n" - "sample_time_interval = 4s\n" - "server = \"\"\n" - "tags {\"t1\" = \"good\", test = 100}\n" - "}\n" ->>). - --define(DEFAULT_CONF, << - "\n" - "statsd {\n" - "enable = true\n" - "flush_time_interval = 4s\n" - "sample_time_interval = 4s\n" - "tags {\"t1\" = \"good\", test = 100}\n" - "}\n" ->>). - -init_per_suite(Config) -> - emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_dashboard, emqx_statsd], - fun set_special_configs/1 - ), - ok = emqx_common_test_helpers:load_config(emqx_statsd_schema, ?BASE_CONF), - Config. - -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([emqx_statsd, emqx_dashboard, emqx_conf]). - -set_special_configs(emqx_dashboard) -> - emqx_dashboard_api_test_helpers:set_default_config(); -set_special_configs(_) -> - ok. - -all() -> - emqx_common_test_helpers:all(?MODULE). - -t_server_validator(_) -> - Server0 = emqx_conf:get_raw([statsd, server]), - ?assertThrow( - #{ - kind := validation_error, - path := "statsd.server", - reason := "cannot_be_empty", - value := "" - }, - emqx_common_test_helpers:load_config(emqx_statsd_schema, ?BAD_CONF) - ), - %% default - ok = emqx_common_test_helpers:load_config(emqx_statsd_schema, ?DEFAULT_CONF), - DefaultServer = default_server(), - ?assertEqual(DefaultServer, emqx_conf:get_raw([statsd, server])), - DefaultServerStr = binary_to_list(DefaultServer), - ?assertEqual(DefaultServerStr, emqx_conf:get([statsd, server])), - %% recover - ok = emqx_common_test_helpers:load_config(emqx_statsd_schema, ?BASE_CONF), - Server2 = emqx_conf:get_raw([statsd, server]), - ?assertMatch(Server0, Server2), - ok. - -t_statsd(_) -> - {ok, Socket} = gen_udp:open(8126, [{active, true}]), - receive - {udp, Socket1, Host, Port, Data} -> - ct:pal("receive:~p~n", [{Socket, Socket1, Host, Port}]), - ?assert(length(Data) > 50), - ?assert(nomatch =/= string:find(Data, "\nemqx.cpu_use:")) - after 10 * 1000 -> - error(timeout) - end, - gen_udp:close(Socket). - -t_management(_) -> - ?assertMatch(ok, emqx_statsd:start()), - ?assertMatch(ok, emqx_statsd:start()), - ?assertMatch(ok, emqx_statsd:stop()), - ?assertMatch(ok, emqx_statsd:stop()), - ?assertMatch(ok, emqx_statsd:restart()). - -t_rest_http(_) -> - {ok, Res0} = request(get), - ?assertEqual( - #{ - <<"enable">> => true, - <<"flush_time_interval">> => <<"4s">>, - <<"sample_time_interval">> => <<"4s">>, - <<"server">> => <<"127.0.0.1:8126">>, - <<"tags">> => #{<<"t1">> => <<"good">>, <<"test">> => 100} - }, - Res0 - ), - {ok, Res1} = request(put, #{enable => false}), - ?assertMatch(#{<<"enable">> := false}, Res1), - ?assertEqual(maps:remove(<<"enable">>, Res0), maps:remove(<<"enable">>, Res1)), - {ok, Res2} = request(get), - ?assertEqual(Res1, Res2), - ?assertEqual( - error, request(put, #{sample_time_interval => "11s", flush_time_interval => "10s"}) - ), - {ok, _} = request(put, #{enable => true}), - ok. - -t_kill_exit(_) -> - {ok, _} = request(put, #{enable => true}), - Pid = erlang:whereis(emqx_statsd), - ?assertEqual(ignore, gen_server:call(Pid, whatever)), - ?assertEqual(ok, gen_server:cast(Pid, whatever)), - ?assertEqual(Pid, erlang:whereis(emqx_statsd)), - #{estatsd_pid := Estatsd} = sys:get_state(emqx_statsd), - ?assert(erlang:exit(Estatsd, kill)), - ?assertEqual(false, is_process_alive(Estatsd)), - ct:sleep(150), - Pid1 = erlang:whereis(emqx_statsd), - ?assertNotEqual(Pid, Pid1), - #{estatsd_pid := Estatsd1} = sys:get_state(emqx_statsd), - ?assertNotEqual(Estatsd, Estatsd1), - ok. - -t_config_update(_) -> - OldRawConf = emqx_conf:get_raw([statsd]), - {ok, _} = emqx_statsd_config:update(OldRawConf#{<<"enable">> => true}), - CommonKeys = [flush_time_interval, sample_time_interval], - OldConf = emqx_conf:get([statsd]), - OldStatsDState = sys:get_state(emqx_statsd), - OldPid = erlang:whereis(emqx_statsd), - ?assertEqual(maps:with(CommonKeys, OldConf), maps:with(CommonKeys, OldStatsDState)), - NewRawConfExpect = OldRawConf#{ - <<"flush_time_interval">> := <<"42s">>, - <<"sample_time_interval">> := <<"42s">> - }, - try - {ok, _} = emqx_statsd_config:update(NewRawConfExpect), - NewRawConf = emqx_conf:get_raw([statsd]), - NewConf = emqx_conf:get([statsd]), - NewStatsDState = sys:get_state(emqx_statsd), - NewPid = erlang:whereis(emqx_statsd), - ?assertNotEqual(OldRawConf, NewRawConf), - ?assertEqual(NewRawConfExpect, NewRawConf), - ?assertEqual(maps:with(CommonKeys, NewConf), maps:with(CommonKeys, NewStatsDState)), - ?assertNotEqual(OldPid, NewPid) - after - {ok, _} = emqx_statsd_config:update(OldRawConf) - end, - %% bad server url - BadRawConf = OldRawConf#{<<"server">> := <<"">>}, - {error, #{ - kind := validation_error, - path := "statsd.server", - reason := "cannot_be_empty", - value := "" - }} = emqx_statsd_config:update(BadRawConf), - ok. - -request(Method) -> request(Method, []). - -request(Method, Body) -> - case request(Method, uri(["statsd"]), Body) of - {ok, 200, Res} -> - {ok, emqx_utils_json:decode(Res, [return_maps])}; - {ok, _Status, _} -> - error - end. - -default_server() -> - {server, Schema} = lists:keyfind(server, 1, emqx_statsd_schema:fields("statsd")), - hocon_schema:field_schema(Schema, default). diff --git a/apps/emqx_telemetry/README.md b/apps/emqx_telemetry/README.md new file mode 100644 index 000000000..a608b6475 --- /dev/null +++ b/apps/emqx_telemetry/README.md @@ -0,0 +1,55 @@ +# emqx_telemetry + +In order for EMQ to understand how EMQX opensource edition is being used, +this app reports some telemetry data to [EMQ's telemetry server](https://telemetry.emqx.io/api/telemetry) + +To turn it off, you can set `telemetry.enable = false` in emqx.conf, +or start EMQX with environment variable `EMQX_TELEMETRY__ENABLE=false`. + +## Reported Data + +This application is only relesaed in EMQX opensource edition. +There is nothing in the reported data which can back-track the origin. +The report interval is 7 days, so there is no performance impact. + +Here is the full catalog of the reported data. + +### Installation + +- EMQX version string +- License (always stubed with "opensource") +- VM Stats (number of cores, total memory) +- Operating system name +- Operating system version +- Erlang/OTP version +- Number of enabled plugins +- Cluster UUID (auto-generated random ID) +- Node UUID (auto-generated random ID) + +### Provisioning + +- Advanced MQTT features + - Number of retained messages + - Number of topic-rewrite rules + - Number of delay-publish messages + - Number of auto-subscription rules +- Enabled authentication and authorization mechanisms +- Enabled gateway names +- Enabled bridge types +- Enabled exhooks (the number of endpoints and hookpoints) + +### Runtime + +- Uptime (how long the server has been running since last start/restart) +- Number of connected clients +- Number of messages sent +- Messaging rates + + +## More info + +More introduction about [Telemetry](https://www.emqx.io/docs/en/v5.0/telemetry/telemetry.html#telemetry). + +See HTTP API docs to [Enable/Disable telemetry](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Telemetry/paths/~1telemetry~1status/put), +[Get the enabled status](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Telemetry/paths/~1telemetry~1status/get) +and [Get the data of the module collected](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Telemetry/paths/~1telemetry~1data/get). diff --git a/apps/emqx_telemetry/rebar.config b/apps/emqx_telemetry/rebar.config new file mode 100644 index 000000000..ff542aed7 --- /dev/null +++ b/apps/emqx_telemetry/rebar.config @@ -0,0 +1,8 @@ +%% -*- mode: erlang -*- + +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_conf, {path, "../emqx_conf"}} +]}. +{project_plugins, [erlfmt]}. diff --git a/apps/emqx_telemetry/src/emqx_telemetry.app.src b/apps/emqx_telemetry/src/emqx_telemetry.app.src new file mode 100644 index 000000000..7a3ca3dcb --- /dev/null +++ b/apps/emqx_telemetry/src/emqx_telemetry.app.src @@ -0,0 +1,15 @@ +{application, emqx_telemetry, [ + {description, "Report telemetry data for EMQX Opensource edition"}, + {vsn, "0.1.0"}, + {registered, [emqx_telemetry_sup, emqx_telemetry]}, + {mod, {emqx_telemetry_app, []}}, + {applications, [ + kernel, + stdlib + ]}, + {env, []}, + {modules, []}, + + {licenses, ["Apache-2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_modules/src/emqx_telemetry.erl b/apps/emqx_telemetry/src/emqx_telemetry.erl similarity index 93% rename from apps/emqx_modules/src/emqx_telemetry.erl rename to apps/emqx_telemetry/src/emqx_telemetry.erl index 3b27302df..bf4004dce 100644 --- a/apps/emqx_modules/src/emqx_telemetry.erl +++ b/apps/emqx_telemetry/src/emqx_telemetry.erl @@ -18,13 +18,6 @@ -behaviour(gen_server). --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/logger.hrl"). - --include_lib("snabbkaffe/include/snabbkaffe.hrl"). - --include("emqx_modules.hrl"). - -export([ start_link/0, stop/0 @@ -41,9 +34,10 @@ code_change/3 ]). +%% config change hook points -export([ - enable/0, - disable/0 + start_reporting/0, + stop_reporting/0 ]). -export([ @@ -52,8 +46,6 @@ get_telemetry/0 ]). --export([official_version/1]). - %% Internal exports (RPC) -export([ do_ensure_uuids/0 @@ -72,6 +64,15 @@ get_value/3 ]). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% The destination URL for the telemetry data report +-define(TELEMETRY_URL, "https://telemetry.emqx.io/api/telemetry"). +-define(REPORT_INTERVAL, 604800). + -record(telemetry, { id :: atom(), uuid :: binary() @@ -112,11 +113,17 @@ start_link() -> stop() -> gen_server:stop(?MODULE). -enable() -> - gen_server:call(?MODULE, enable, 15_000). +%% @doc Re-start the reporting timer after disabled. +%% This is an async notification which never fails. +%% This is a no-op in enterprise edition. +start_reporting() -> + gen_server:cast(?MODULE, start_reporting). -disable() -> - gen_server:call(?MODULE, disable). +%% @doc Stop the reporting timer. +%% This is an async notification which never fails. +%% This is a no-op in enterprise eidtion. +stop_reporting() -> + gen_server:cast(?MODULE, stop_reporting). get_node_uuid() -> gen_server:call(?MODULE, get_node_uuid). @@ -132,7 +139,9 @@ get_telemetry() -> %%-------------------------------------------------------------------- init(_Opts) -> - {ok, undefined, {continue, init}}. + process_flag(trap_exit, true), + emqx_telemetry_config:on_server_start(), + {ok, "ignored", {continue, init}}. handle_continue(init, _) -> ok = mria:create_table( @@ -148,21 +157,12 @@ handle_continue(init, _) -> ok = mria:wait_for_tables([?TELEMETRY]), State0 = empty_state(), {NodeUUID, ClusterUUID} = ensure_uuids(), - {noreply, State0#state{node_uuid = NodeUUID, cluster_uuid = ClusterUUID}}; -handle_continue(Continue, State) -> - ?SLOG(error, #{msg => "unexpected_continue", continue => Continue}), - {noreply, State}. + case is_enabled() of + true -> ok = start_reporting(); + false -> ok + end, + {noreply, State0#state{node_uuid = NodeUUID, cluster_uuid = ClusterUUID}}. -handle_call(enable, _From, State) -> - %% Wait a few moments before reporting the first telemetry, as the - %% apps might still be starting up. Also, this avoids hanging - %% `emqx_modules_app' initialization in case the POST request - %% takes a lot of time. - FirstReportTimeoutMS = timer:seconds(10), - {reply, ok, ensure_report_timer(FirstReportTimeoutMS, State)}; -handle_call(disable, _From, State = #state{timer = Timer}) -> - emqx_utils:cancel_timer(Timer), - {reply, ok, State#state{timer = undefined}}; handle_call(get_node_uuid, _From, State = #state{node_uuid = UUID}) -> {reply, {ok, UUID}, State}; handle_call(get_cluster_uuid, _From, State = #state{cluster_uuid = UUID}) -> @@ -174,6 +174,14 @@ handle_call(Req, _From, State) -> ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. +handle_cast(start_reporting, State) -> + %% Wait a few moments before reporting the first telemetry, as the + %% apps might still be starting up. + FirstReportTimeoutMS = timer:seconds(10), + {noreply, ensure_report_timer(FirstReportTimeoutMS, State)}; +handle_cast(stop_reporting, State = #state{timer = Timer}) -> + emqx_utils:cancel_timer(Timer), + {noreply, State#state{timer = undefined}}; handle_cast(Msg, State) -> ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), {noreply, State}. @@ -188,7 +196,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, _State) -> - ok. + emqx_telemetry_config:on_server_stop(). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -197,9 +205,8 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%------------------------------------------------------------------------------ -official_version(Version) -> - Pt = "^\\d+\\.\\d+(?:\\.\\d+)?(?:(-(?:alpha|beta|rc)\\.[1-9][0-9]*))?$", - match =:= re:run(Version, Pt, [{capture, none}]). +is_enabled() -> + emqx_telemetry_config:is_enabled(). ensure_report_timer(State = #state{report_interval = ReportInterval}) -> ensure_report_timer(ReportInterval, State). diff --git a/apps/emqx_modules/src/emqx_telemetry_api.erl b/apps/emqx_telemetry/src/emqx_telemetry_api.erl similarity index 97% rename from apps/emqx_modules/src/emqx_telemetry_api.erl rename to apps/emqx_telemetry/src/emqx_telemetry_api.erl index b7209d146..c90ad6b38 100644 --- a/apps/emqx_modules/src/emqx_telemetry_api.erl +++ b/apps/emqx_telemetry/src/emqx_telemetry_api.erl @@ -220,7 +220,7 @@ status(get, _Params) -> {200, get_telemetry_status()}; status(put, #{body := Body}) -> Enable = maps:get(<<"enable">>, Body), - case Enable =:= emqx_modules_conf:is_telemetry_enabled() of + case Enable =:= is_enabled() of true -> Reason = case Enable of @@ -241,7 +241,7 @@ status(put, #{body := Body}) -> end. data(get, _Request) -> - case emqx_modules_conf:is_telemetry_enabled() of + case is_enabled() of true -> {200, emqx_utils_json:encode(get_telemetry_data())}; false -> @@ -256,11 +256,14 @@ data(get, _Request) -> %%-------------------------------------------------------------------- enable_telemetry(Enable) -> - emqx_modules_conf:set_telemetry_status(Enable). + emqx_telemetry_config:set_telemetry_status(Enable). get_telemetry_status() -> - #{enable => emqx_modules_conf:is_telemetry_enabled()}. + #{enable => is_enabled()}. get_telemetry_data() -> {ok, TelemetryData} = emqx_telemetry:get_telemetry(), TelemetryData. + +is_enabled() -> + emqx_telemetry_config:is_enabled(). diff --git a/apps/emqx_statsd/include/emqx_statsd.hrl b/apps/emqx_telemetry/src/emqx_telemetry_app.erl similarity index 71% rename from apps/emqx_statsd/include/emqx_statsd.hrl rename to apps/emqx_telemetry/src/emqx_telemetry_app.erl index 557ebc684..507b45d95 100644 --- a/apps/emqx_statsd/include/emqx_statsd.hrl +++ b/apps/emqx_telemetry/src/emqx_telemetry_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -14,6 +14,16 @@ %% limitations under the License. %%-------------------------------------------------------------------- --define(APP, emqx_statsd). --define(STATSD, [statsd]). --define(SERVER_PARSE_OPTS, #{default_port => 8125}). +-module(emqx_telemetry_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + emqx_telemetry_sup:start_link(). + +stop(_State) -> + ok. + +%% internal functions diff --git a/apps/emqx_telemetry/src/emqx_telemetry_config.erl b/apps/emqx_telemetry/src/emqx_telemetry_config.erl new file mode 100644 index 000000000..9419db939 --- /dev/null +++ b/apps/emqx_telemetry/src/emqx_telemetry_config.erl @@ -0,0 +1,79 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_telemetry_config). + +%% Public API +-export([ + set_telemetry_status/1, + is_enabled/0 +]). + +%% emqx_config_handler callback +-export([ + pre_config_update/3, + post_config_update/5 +]). + +%% internal use +-export([ + on_server_start/0, + on_server_stop/0, + is_official_version/1 +]). + +is_enabled() -> + IsOfficial = ?MODULE:is_official_version(emqx_release:version()), + emqx_conf:get([telemetry, enable], IsOfficial). + +on_server_start() -> + emqx_conf:add_handler([telemetry], ?MODULE). + +on_server_stop() -> + emqx_conf:remove_handler([telemetry]). + +-spec set_telemetry_status(boolean()) -> ok | {error, term()}. +set_telemetry_status(Status) -> + case cfg_update([telemetry], set_telemetry_status, Status) of + {ok, _} -> ok; + {error, _} = Error -> Error + end. + +pre_config_update(_, {set_telemetry_status, Status}, RawConf) -> + {ok, RawConf#{<<"enable">> => Status}}. + +post_config_update( + _, + {set_telemetry_status, Status}, + _NewConfig, + _OldConfig, + _AppEnvs +) -> + case Status of + true -> emqx_telemetry:start_reporting(); + false -> emqx_telemetry:stop_reporting() + end. + +cfg_update(Path, Action, Params) -> + emqx_conf:update( + Path, + {Action, Params}, + #{override_to => cluster} + ). + +is_official_version(Version) -> + Pt = "^\\d+\\.\\d+(?:\\.\\d+)?(?:(-(?:alpha|beta|rc)\\.[1-9][0-9]*))?$", + match =:= re:run(Version, Pt, [{capture, none}]). diff --git a/apps/emqx_statsd/src/proto/emqx_statsd_proto_v1.erl b/apps/emqx_telemetry/src/emqx_telemetry_schema.erl similarity index 57% rename from apps/emqx_statsd/src/proto/emqx_statsd_proto_v1.erl rename to apps/emqx_telemetry/src/emqx_telemetry_schema.erl index 7b696038b..1e1f547c5 100644 --- a/apps/emqx_statsd/src/proto/emqx_statsd_proto_v1.erl +++ b/apps/emqx_telemetry/src/emqx_telemetry_schema.erl @@ -14,31 +14,25 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_statsd_proto_v1). +-module(emqx_telemetry_schema). --behaviour(emqx_bpapi). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-behaviour(hocon_schema). -export([ - introduced_in/0, - - start/1, - stop/1, - restart/1 + roots/0, + fields/1, + desc/1 ]). --include_lib("emqx/include/bpapi.hrl"). +roots() -> ["telemetry"]. -introduced_in() -> - "5.0.0". +fields("telemetry") -> + [{enable, ?HOCON(boolean(), #{required => false, desc => ?DESC("enable")})}]. --spec start([node()]) -> emqx_rpc:multicall_result(). -start(Nodes) -> - rpc:multicall(Nodes, emqx_statsd, do_start, [], 5000). - --spec stop([node()]) -> emqx_rpc:multicall_result(). -stop(Nodes) -> - rpc:multicall(Nodes, emqx_statsd, do_stop, [], 5000). - --spec restart([node()]) -> emqx_rpc:multicall_result(). -restart(Nodes) -> - rpc:multicall(Nodes, emqx_statsd, do_restart, [], 5000). +desc("telemetry") -> + ?DESC("telemetry_root_doc"); +desc(_) -> + undefined. diff --git a/apps/emqx_telemetry/src/emqx_telemetry_sup.erl b/apps/emqx_telemetry/src/emqx_telemetry_sup.erl new file mode 100644 index 000000000..b85348aa6 --- /dev/null +++ b/apps/emqx_telemetry/src/emqx_telemetry_sup.erl @@ -0,0 +1,45 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_telemetry_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). +-define(CHILD(Mod), #{ + id => Mod, + start => {Mod, start_link, []}, + restart => transient, + shutdown => 5000, + type => worker, + modules => [Mod] +}). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +init([]) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 5 + }, + ChildSpecs = [?CHILD(emqx_telemetry)], + {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_modules/src/proto/emqx_telemetry_proto_v1.erl b/apps/emqx_telemetry/src/proto/emqx_telemetry_proto_v1.erl similarity index 100% rename from apps/emqx_modules/src/proto/emqx_telemetry_proto_v1.erl rename to apps/emqx_telemetry/src/proto/emqx_telemetry_proto_v1.erl diff --git a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl b/apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl similarity index 89% rename from apps/emqx_modules/test/emqx_telemetry_SUITE.erl rename to apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl index 86ea65620..65604c4e5 100644 --- a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl +++ b/apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl @@ -25,13 +25,21 @@ -import(proplists, [get_value/2]). --define(BASE_CONF, #{ +-define(MODULES_CONF, #{ <<"dealyed">> => <<"true">>, <<"max_delayed_messages">> => <<"0">> }). all() -> emqx_common_test_helpers:all(?MODULE). +suite() -> + [ + {timetrap, {minutes, 1}}, + {repeat, 1} + ]. + +apps() -> [emqx_conf, emqx_retainer, emqx_authn, emqx_authz, emqx_modules, emqx_telemetry]. + init_per_suite(Config) -> net_kernel:start(['master@127.0.0.1', longnames]), ok = meck:new(emqx_authz, [non_strict, passthrough, no_history, no_link]), @@ -42,12 +50,9 @@ init_per_suite(Config) -> emqx_common_test_helpers:deps_path(emqx_authz, "etc/acl.conf") end ), - ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF), + ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?MODULES_CONF), emqx_gateway_test_utils:load_all_gateway_apps(), - emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_authn, emqx_authz, emqx_modules], - fun set_special_configs/1 - ), + start_apps(), Config. end_per_suite(_Config) -> @@ -61,7 +66,7 @@ end_per_suite(_Config) -> ), mnesia:clear_table(cluster_rpc_commit), mnesia:clear_table(cluster_rpc_mfa), - emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authn, emqx_authz, emqx_modules]), + stop_apps(), meck:unload(emqx_authz), ok. @@ -100,13 +105,9 @@ init_per_testcase(t_get_telemetry, Config) -> Config; init_per_testcase(t_advanced_mqtt_features, Config) -> {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), - {ok, Retainer} = emqx_retainer:start_link(), {atomic, ok} = mria:clear_table(emqx_delayed), mock_advanced_mqtt_features(), - [ - {retainer, Retainer} - | Config - ]; + Config; init_per_testcase(t_authn_authz_info, Config) -> mock_httpc(), {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), @@ -116,13 +117,13 @@ init_per_testcase(t_authn_authz_info, Config) -> create_authz(postgresql), Config; init_per_testcase(t_enable, Config) -> - ok = meck:new(emqx_telemetry, [non_strict, passthrough, no_history, no_link]), - ok = meck:expect(emqx_telemetry, official_version, fun(_) -> true end), + ok = meck:new(emqx_telemetry_config, [non_strict, passthrough, no_history, no_link]), + ok = meck:expect(emqx_telemetry_config, is_official_version, fun(_) -> true end), mock_httpc(), Config; init_per_testcase(t_send_after_enable, Config) -> - ok = meck:new(emqx_telemetry, [non_strict, passthrough, no_history, no_link]), - ok = meck:expect(emqx_telemetry, official_version, fun(_) -> true end), + ok = meck:new(emqx_telemetry_config, [non_strict, passthrough, no_history, no_link]), + ok = meck:expect(emqx_telemetry_config, is_official_version, fun(_) -> true end), mock_httpc(), Config; init_per_testcase(t_rule_engine_and_data_bridge_info, Config) -> @@ -172,12 +173,9 @@ init_per_testcase(t_uuid_restored_from_file, Config) -> %% clear the UUIDs in the DB {atomic, ok} = mria:clear_table(emqx_telemetry), - emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authn, emqx_authz, emqx_modules]), - ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF), - emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_authn, emqx_authz, emqx_modules], - fun set_special_configs/1 - ), + stop_apps(), + ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?MODULES_CONF), + start_apps(), Node = start_slave(n1), [ {n1, Node}, @@ -205,11 +203,9 @@ end_per_testcase(t_get_telemetry, _Config) -> meck:unload([httpc, emqx_telemetry]), application:stop(emqx_gateway), ok; -end_per_testcase(t_advanced_mqtt_features, Config) -> - Retainer = ?config(retainer, Config), +end_per_testcase(t_advanced_mqtt_features, _Config) -> process_flag(trap_exit, true), ok = emqx_retainer:clean(), - exit(Retainer, kill), {ok, _} = emqx_auto_subscribe:update([]), ok = emqx_rewrite:update([]), {atomic, ok} = mria:clear_table(emqx_delayed), @@ -228,9 +224,9 @@ end_per_testcase(t_authn_authz_info, _Config) -> ), ok; end_per_testcase(t_enable, _Config) -> - meck:unload([httpc, emqx_telemetry]); + meck:unload([httpc, emqx_telemetry_config]); end_per_testcase(t_send_after_enable, _Config) -> - meck:unload([httpc, emqx_telemetry]); + meck:unload([httpc, emqx_telemetry_config]); end_per_testcase(t_rule_engine_and_data_bridge_info, _Config) -> meck:unload(httpc), lists:foreach( @@ -278,10 +274,10 @@ t_node_uuid(_) -> Parts = binary:split(UUID, <<"-">>, [global, trim]), ?assertEqual(5, length(Parts)), {ok, NodeUUID2} = emqx_telemetry:get_node_uuid(), - emqx_telemetry:disable(), - emqx_telemetry:enable(), - emqx_modules_conf:set_telemetry_status(false), - emqx_modules_conf:set_telemetry_status(true), + emqx_telemetry:stop_reporting(), + emqx_telemetry:start_reporting(), + emqx_telemetry_config:set_telemetry_status(false), + emqx_telemetry_config:set_telemetry_status(true), {ok, NodeUUID3} = emqx_telemetry:get_node_uuid(), {ok, NodeUUID4} = emqx_telemetry_proto_v1:get_node_uuid(node()), ?assertEqual(NodeUUID2, NodeUUID3), @@ -325,12 +321,9 @@ t_uuid_saved_to_file(_Config) -> %% clear the UUIDs in the DB {atomic, ok} = mria:clear_table(emqx_telemetry), - emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authn, emqx_authz, emqx_modules]), - ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF), - emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_authn, emqx_authz, emqx_modules], - fun set_special_configs/1 - ), + stop_apps(), + ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?MODULES_CONF), + start_apps(), {ok, NodeUUID} = emqx_telemetry:get_node_uuid(), {ok, ClusterUUID} = emqx_telemetry:get_cluster_uuid(), ?assertEqual( @@ -343,30 +336,35 @@ t_uuid_saved_to_file(_Config) -> ), ok. -t_official_version(_) -> - true = emqx_telemetry:official_version("0.0.0"), - true = emqx_telemetry:official_version("1.1.1"), - true = emqx_telemetry:official_version("10.10.10"), - false = emqx_telemetry:official_version("0.0.0.0"), - false = emqx_telemetry:official_version("1.1.a"), - true = emqx_telemetry:official_version("0.0-alpha.1"), - true = emqx_telemetry:official_version("1.1-alpha.1"), - true = emqx_telemetry:official_version("10.10-alpha.10"), - false = emqx_telemetry:official_version("1.1-alpha.0"), - true = emqx_telemetry:official_version("1.1-beta.1"), - true = emqx_telemetry:official_version("1.1-rc.1"), - false = emqx_telemetry:official_version("1.1-alpha.a"), - true = emqx_telemetry:official_version("5.0.0"), - true = emqx_telemetry:official_version("5.0.0-alpha.1"), - true = emqx_telemetry:official_version("5.0.0-beta.4"), - true = emqx_telemetry:official_version("5.0-rc.1"), - true = emqx_telemetry:official_version("5.0.0-rc.1"), - false = emqx_telemetry:official_version("5.0.0-alpha.a"), - false = emqx_telemetry:official_version("5.0.0-beta.a"), - false = emqx_telemetry:official_version("5.0.0-rc.a"), - false = emqx_telemetry:official_version("5.0.0-foo"), - false = emqx_telemetry:official_version("5.0.0-rc.1-ccdf7920"), - ok. +t_is_official_version(_) -> + Is = fun(V) -> is_official_version(V) end, + true = lists:all(Is, [ + "0.0.0", + "1.1.1", + "10.10.10", + "0.0-alpha.1", + "1.1-alpha.1", + "10.10-alpha.10", + "1.1-rc.1", + "1.1-beta.1", + "5.0.0", + "5.0.0-alpha.1", + "5.0.0-beta.4", + "5.0-rc.1", + "5.0.0-rc.1" + ]), + + false = lists:any(Is, [ + "1.1-alpha.a", + "1.1-alpha.0", + "0.0.0.0", + "1.1.a", + "5.0.0-alpha.a", + "5.0.0-beta.a", + "5.0.0-rc.a", + "5.0.0-foo", + "5.0.0-rc.1-ccdf7920" + ]). t_get_telemetry(_Config) -> {ok, TelemetryData} = emqx_telemetry:get_telemetry(), @@ -432,23 +430,25 @@ t_num_clients(_Config) -> {port, 1883}, {clean_start, false} ]), - ?wait_async_action( + {{ok, _}, _} = ?wait_async_action( {ok, _} = emqtt:connect(Client), #{ ?snk_kind := emqx_stats_setstat, count_stat := 'live_connections.count', value := 1 - } + }, + 2000 ), {ok, TelemetryData0} = emqx_telemetry:get_telemetry(), ?assertEqual(1, get_value(num_clients, TelemetryData0)), - ?wait_async_action( + {ok, _} = ?wait_async_action( ok = emqtt:disconnect(Client), #{ ?snk_kind := emqx_stats_setstat, count_stat := 'live_connections.count', value := 0 - } + }, + 2000 ), {ok, TelemetryData1} = emqx_telemetry:get_telemetry(), ?assertEqual(0, get_value(num_clients, TelemetryData1)), @@ -485,19 +485,19 @@ t_authn_authz_info(_) -> ). t_enable(_) -> - ok = emqx_telemetry:enable(), - ok = emqx_telemetry:disable(). + ok = emqx_telemetry:start_reporting(), + ok = emqx_telemetry:stop_reporting(). t_send_after_enable(_) -> - ok = emqx_telemetry:disable(), + ok = emqx_telemetry:stop_reporting(), ok = snabbkaffe:start_trace(), try - ok = emqx_telemetry:enable(), + ok = emqx_telemetry:start_reporting(), Timeout = 12_000, ?assertMatch( {ok, _}, ?wait_async_action( - ok = emqx_telemetry:enable(), + ok = emqx_telemetry:start_reporting(), #{?snk_kind := telemetry_data_reported}, Timeout ) @@ -818,11 +818,10 @@ start_slave(Name) -> (emqx) -> application:set_env(emqx, boot_modules, []), ekka:join(TestNode), - emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF), - + emqx_common_test_helpers:load_config(emqx_modules_schema, ?MODULES_CONF), ok; (_App) -> - emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF), + emqx_common_test_helpers:load_config(emqx_modules_schema, ?MODULES_CONF), ok end, Opts = #{ @@ -837,7 +836,7 @@ start_slave(Name) -> env_handler => Handler, load_apps => [gen_rpc, emqx], listener_ports => [], - apps => [emqx_conf, emqx_modules] + apps => [emqx, emqx_conf, emqx_retainer, emqx_modules, emqx_telemetry] }, emqx_common_test_helpers:start_slave(Name, Opts). @@ -861,3 +860,12 @@ leave_cluster() -> application:set_env(mria, db_backend, mnesia), ekka:leave() end. + +is_official_version(V) -> + emqx_telemetry_config:is_official_version(V). + +start_apps() -> + emqx_common_test_helpers:start_apps(apps(), fun set_special_configs/1). + +stop_apps() -> + emqx_common_test_helpers:stop_apps(lists:reverse(apps())). diff --git a/apps/emqx_modules/test/emqx_telemetry_SUITE_data/BUILD_INFO b/apps/emqx_telemetry/test/emqx_telemetry_SUITE_data/BUILD_INFO similarity index 100% rename from apps/emqx_modules/test/emqx_telemetry_SUITE_data/BUILD_INFO rename to apps/emqx_telemetry/test/emqx_telemetry_SUITE_data/BUILD_INFO diff --git a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl b/apps/emqx_telemetry/test/emqx_telemetry_api_SUITE.erl similarity index 97% rename from apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl rename to apps/emqx_telemetry/test/emqx_telemetry_api_SUITE.erl index c375810b5..e8289ff90 100644 --- a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl +++ b/apps/emqx_telemetry/test/emqx_telemetry_api_SUITE.erl @@ -31,7 +31,7 @@ all() -> init_per_suite(Config) -> ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF), ok = emqx_mgmt_api_test_util:init_suite( - [emqx_conf, emqx_authn, emqx_authz, emqx_modules], + [emqx_conf, emqx_authn, emqx_authz, emqx_telemetry], fun set_special_configs/1 ), @@ -47,7 +47,7 @@ end_per_suite(_Config) -> } ), emqx_mgmt_api_test_util:end_suite([ - emqx_conf, emqx_authn, emqx_authz, emqx_modules + emqx_conf, emqx_authn, emqx_authz, emqx_telemetry ]), ok. diff --git a/bin/emqx b/bin/emqx index 3b7212c99..802e4b1dd 100755 --- a/bin/emqx +++ b/bin/emqx @@ -612,7 +612,7 @@ check_license() { set +x logerr "License not found." logerr "Please specify one via the EMQX_LICENSE__KEY variable" - logerr "or via license.key in emqx-enterprise.conf." + logerr "or via license.key in emqx.conf." return 1 fi } @@ -875,16 +875,16 @@ tr_log_to_env() { unset EMQX_LOG__TO case "${log_to}" in console) - export EMQX_LOG__CONSOLE_HANDLER__ENABLE='true' - export EMQX_LOG__FILE_HANDLERS__DEFAULT__ENABLE='false' + export EMQX_LOG__CONSOLE__ENABLE='true' + export EMQX_LOG__FILE__ENABLE='false' ;; file) - export EMQX_LOG__CONSOLE_HANDLER__ENABLE='false' - export EMQX_LOG__FILE_HANDLERS__DEFAULT__ENABLE='true' + export EMQX_LOG__CONSOLE__ENABLE='false' + export EMQX_LOG__FILE__ENABLE='true' ;; both) - export EMQX_LOG__CONSOLE_HANDLER__ENABLE='true' - export EMQX_LOG__FILE_HANDLERS__DEFAULT__ENABLE='true' + export EMQX_LOG__CONSOLE__ENABLE='true' + export EMQX_LOG__FILE__ENABLE='true' ;; default) # want to use config file defaults, do nothing diff --git a/bin/emqx.cmd b/bin/emqx.cmd index ede51a56b..2871bcc20 100644 --- a/bin/emqx.cmd +++ b/bin/emqx.cmd @@ -46,6 +46,7 @@ @set "RUNNER_ROOT_DIR=%rel_root_dir%" :: hard code etc dir @set "EMQX_ETC_DIR=%rel_root_dir%\etc" +@set "EMQX_LOG_DIR=%rel_root_dir%\log" @set "etc_dir=%rel_root_dir%\etc" @set "lib_dir=%rel_root_dir%\lib" @set "emqx_conf=%etc_dir%\emqx.conf" diff --git a/build b/build index 0846d6057..a5f63f403 100755 --- a/build +++ b/build @@ -91,37 +91,65 @@ log() { echo "===< $msg" } +prepare_erl_libs() { + local libs_dir="$1" + local erl_libs="${ERL_LIBS:-}" + local sep + if [ "${SYSTEM}" = 'windows' ]; then + sep=';' + else + sep=':' + fi + for app in "${libs_dir}"/*; do + if [ -d "${app}/ebin" ]; then + if [ -n "$erl_libs" ]; then + erl_libs="${erl_libs}${sep}${app}" + else + erl_libs="${app}" + fi + fi + done + export ERL_LIBS="$erl_libs" +} + make_docs() { - local libs_dir1 libs_dir2 libs_dir3 docdir - libs_dir1="$("$FIND" "_build/$PROFILE/lib/" -maxdepth 2 -name ebin -type d)" - if [ -d "_build/default/lib/" ]; then - libs_dir2="$("$FIND" "_build/default/lib/" -maxdepth 2 -name ebin -type d)" - else - libs_dir2='' - fi - if [ -d "_build/$PROFILE/checkouts" ]; then - libs_dir3="$("$FIND" "_build/$PROFILE/checkouts/" -maxdepth 2 -name ebin -type d 2>/dev/null || true)" - else - libs_dir3='' - fi case "$(is_enterprise "$PROFILE")" in 'yes') - SCHEMA_MODULE='emqx_ee_conf_schema' + SCHEMA_MODULE='emqx_enterprise_schema' ;; 'no') SCHEMA_MODULE='emqx_conf_schema' ;; esac - docdir="_build/docgen/$PROFILE" + prepare_erl_libs "_build/$PROFILE/checkouts" + prepare_erl_libs "_build/$PROFILE/lib" + local docdir="_build/docgen/$PROFILE" mkdir -p "$docdir" # shellcheck disable=SC2086 - erl -noshell -pa $libs_dir1 $libs_dir2 $libs_dir3 -eval \ + erl -noshell -eval \ "ok = emqx_conf:dump_schema('$docdir', $SCHEMA_MODULE), \ halt(0)." } -assert_no_compile_time_only_deps() { - : +## arg1 is the profile for which the following args (as app names) should be excluded +assert_no_excluded_deps() { + local profile="$1" + shift 1 + if [ "$PROFILE" != "$profile" ]; then + # not currently building the profile which has apps to be excluded + return 0 + fi + local rel_dir="_build/$PROFILE/rel/emqx/lib" + local excluded_apps=( "$@" ) + local found + for app in "${excluded_apps[@]}"; do + found="$($FIND "$rel_dir" -maxdepth 1 -type d -name "$app-*")" + if [ -n "${found}" ]; then + echo "ERROR: ${app} should not be included in ${PROFILE}" + echo "ERROR: found ${app} in ${rel_dir}" + exit 1 + fi + done } just_compile() { @@ -139,20 +167,20 @@ make_rel() { just_compile # now assemble the release tar ./rebar3 as "$PROFILE" "$release_or_tar" - assert_no_compile_time_only_deps + assert_no_excluded_deps emqx-enterprise emqx_telemetry } make_elixir_rel() { - ./scripts/pre-compile.sh "$PROFILE" - export_elixir_release_vars "$PROFILE" - # for some reason, this has to be run outside "do"... - mix local.rebar --if-missing --force - # shellcheck disable=SC1010 - mix do local.hex --if-missing --force, \ - local.rebar rebar3 "${PWD}/rebar3" --if-missing --force, \ - deps.get - mix release --overwrite - assert_no_compile_time_only_deps + ./scripts/pre-compile.sh "$PROFILE" + export_elixir_release_vars "$PROFILE" + # for some reason, this has to be run outside "do"... + mix local.rebar --if-missing --force + # shellcheck disable=SC1010 + mix do local.hex --if-missing --force, \ + local.rebar rebar3 "${PWD}/rebar3" --if-missing --force, \ + deps.get + mix release --overwrite + assert_no_excluded_deps emqx-enterprise emqx_telemetry } ## extract previous version .tar.gz files to _build/$PROFILE/rel/emqx before making relup diff --git a/changes/ce/feat-10584.en.md b/changes/ce/feat-10584.en.md new file mode 100644 index 000000000..abb514cbb --- /dev/null +++ b/changes/ce/feat-10584.en.md @@ -0,0 +1 @@ +Add log level configuration to SSL communication diff --git a/changes/ce/feat-10702.en.md b/changes/ce/feat-10702.en.md new file mode 100644 index 000000000..936103848 --- /dev/null +++ b/changes/ce/feat-10702.en.md @@ -0,0 +1,4 @@ +Introduce a more straightforward configuration option `keepalive_multiplier` and +deprecate the old `keepalive_backoff` configuration. +After this enhancement, EMQX checks the client's keepalive timeout status +period by multiplying the "Client Requested Keepalive Interval" with `keepalive_multiplier`. diff --git a/changes/ce/fix-10563.en.md b/changes/ce/fix-10563.en.md new file mode 100644 index 000000000..f902fb57b --- /dev/null +++ b/changes/ce/fix-10563.en.md @@ -0,0 +1,2 @@ +Corrected an issue where the no_local flag was not functioning correctly. + diff --git a/changes/ce/fix-10600.en.md b/changes/ce/fix-10600.en.md new file mode 100644 index 000000000..522ef96a9 --- /dev/null +++ b/changes/ce/fix-10600.en.md @@ -0,0 +1,2 @@ +Deleted emqx_statsd application. + diff --git a/changes/ce/fix-10653.en.md b/changes/ce/fix-10653.en.md new file mode 100644 index 000000000..c18ea9ed0 --- /dev/null +++ b/changes/ce/fix-10653.en.md @@ -0,0 +1 @@ +Store gateway authentication TLS certificates and keys in the data directory. diff --git a/changes/ce/fix-10677.en.md b/changes/ce/fix-10677.en.md new file mode 100644 index 000000000..c669606e7 --- /dev/null +++ b/changes/ce/fix-10677.en.md @@ -0,0 +1 @@ +In Rule API, reapond with 404 HTTP error code when trying to delete a rule that does not exist. diff --git a/changes/ce/fix-10682.en.md b/changes/ce/fix-10682.en.md new file mode 100644 index 000000000..df8d93116 --- /dev/null +++ b/changes/ce/fix-10682.en.md @@ -0,0 +1 @@ +Fix the timestamp for the will message is incorrectly assigned at the session creation time, now this timestamp is the disconnected time of the session. diff --git a/changes/ce/fix-10701.en.md b/changes/ce/fix-10701.en.md new file mode 100644 index 000000000..c153e01e1 --- /dev/null +++ b/changes/ce/fix-10701.en.md @@ -0,0 +1 @@ +RPM package for Amazon Linux 2 did not support TLS v1.3 as it was assembled with Erlang/OTP built with openssl 1.0. diff --git a/changes/ce/fix-10715.en.md b/changes/ce/fix-10715.en.md new file mode 100644 index 000000000..68df92df6 --- /dev/null +++ b/changes/ce/fix-10715.en.md @@ -0,0 +1 @@ +Postpone trimming the connection information structure until after `client.connected` hooks have been executed. These hooks once again have access to the client's peer certificate. diff --git a/changes/ce/fix-10737.en.md b/changes/ce/fix-10737.en.md new file mode 100644 index 000000000..b5826bac2 --- /dev/null +++ b/changes/ce/fix-10737.en.md @@ -0,0 +1,2 @@ +Fix the issue where the HTTP API interface of Gateway cannot handle ClientIDs with +special characters, such as: `!@#$%^&*()_+{}:"<>?/`. diff --git a/changes/ce/fix-10785.en.md b/changes/ce/fix-10785.en.md new file mode 100644 index 000000000..14075593f --- /dev/null +++ b/changes/ce/fix-10785.en.md @@ -0,0 +1,3 @@ +Ensure `EMQX_LOG_DIR` is set by Windows boot script. + +The environment variable `EMQX_LOG_DIR` was missing in v5.0.25, caused EMQX Windows package fail to boot unless set by sysadmin. diff --git a/changes/ce/fix-10809.en.md b/changes/ce/fix-10809.en.md new file mode 100644 index 000000000..e7e15e5ca --- /dev/null +++ b/changes/ce/fix-10809.en.md @@ -0,0 +1,2 @@ +Address `** ERROR ** Mnesia post_commit hook failed: error:badarg` error messages happening during node shutdown or restart. +Mria pull request: https://github.com/emqx/mria/pull/142 diff --git a/changes/ce/fix-10818.en.md b/changes/ce/fix-10818.en.md new file mode 100644 index 000000000..068fab4d4 --- /dev/null +++ b/changes/ce/fix-10818.en.md @@ -0,0 +1 @@ +Fixing `emqx_ctl traces` command. diff --git a/changes/ce/perf-10625.en.md b/changes/ce/perf-10625.en.md index 42e712648..a3271d739 100644 --- a/changes/ce/perf-10625.en.md +++ b/changes/ce/perf-10625.en.md @@ -1,4 +1,4 @@ Simplify limiter configuration. - Reduce the complexity of the limiter's configuration. -e.g. now users can use `limiter.messages_rate = 1000/s` to quickly set the node-level limit for the message publish. +e.g. now users can use `limiter.messages_rate = "1000/s"` to quickly set the node-level limit for the message publish. - Update the `configs/limiter` API to suit this refactor. diff --git a/changes/ce/perf-10678.en.md b/changes/ce/perf-10678.en.md new file mode 100644 index 000000000..67090cf1d --- /dev/null +++ b/changes/ce/perf-10678.en.md @@ -0,0 +1 @@ +Optimized counter increment calls to avoid work if increment is zero. diff --git a/changes/ce/perf-10690.en.md b/changes/ce/perf-10690.en.md new file mode 100644 index 000000000..4f7c49e53 --- /dev/null +++ b/changes/ce/perf-10690.en.md @@ -0,0 +1,3 @@ +Added a retry mechanism to webhook bridge that attempts to improve throughput. + +This optimization retries request failures without blocking the buffering layer, which can improve throughput in situations of high messaging rate. diff --git a/changes/ce/perf-10698.en.md b/changes/ce/perf-10698.en.md new file mode 100644 index 000000000..db398b919 --- /dev/null +++ b/changes/ce/perf-10698.en.md @@ -0,0 +1 @@ +Optimize memory usage when accessing the configuration during runtime. diff --git a/changes/ce/perf-10698.zh.md b/changes/ce/perf-10698.zh.md new file mode 100644 index 000000000..ea7f88259 --- /dev/null +++ b/changes/ce/perf-10698.zh.md @@ -0,0 +1,5 @@ +在运行时降低读取配置的内存占用。 + + + + diff --git a/changes/ee/feat-10778.en.md b/changes/ee/feat-10778.en.md new file mode 100644 index 000000000..3084d2959 --- /dev/null +++ b/changes/ee/feat-10778.en.md @@ -0,0 +1 @@ +Refactored Pulsar Producer bridge to avoid leaking resources during crashes. diff --git a/changes/ee/fix-10807.en.md b/changes/ee/fix-10807.en.md new file mode 100644 index 000000000..8cd5da0c8 --- /dev/null +++ b/changes/ee/fix-10807.en.md @@ -0,0 +1 @@ +Removed license check debug logs. diff --git a/changes/ee/refactor-10654.en.md b/changes/ee/refactor-10654.en.md new file mode 100644 index 000000000..229d002da --- /dev/null +++ b/changes/ee/refactor-10654.en.md @@ -0,0 +1 @@ +The clickhouse bridge has been refactored so it is located in its own OTP application. diff --git a/changes/feat-10607.en.md b/changes/feat-10607.en.md new file mode 100644 index 000000000..a7fef73a3 --- /dev/null +++ b/changes/feat-10607.en.md @@ -0,0 +1,3 @@ +Simplified log configuration: +- Replaced `log.console_handler` with `log.console`, keeping only `enable, level, formatter, and time_offset`. +- Replaced `log.file_handlers` with `log.file`, keeping only `enable, level, formatter, time_offset, rotation_count, rotation_size, to`. diff --git a/changes/v5.0.26.en.md b/changes/v5.0.26.en.md new file mode 100644 index 000000000..c79131bec --- /dev/null +++ b/changes/v5.0.26.en.md @@ -0,0 +1,107 @@ +# + +## Enhancements + +- [#10584](https://github.com/emqx/emqx/pull/10584) Add log level configuration to SSL communication + +- [#10702](https://github.com/emqx/emqx/pull/10702) Introduce a more straightforward configuration option `keepalive_multiplier` and + deprecate the old `keepalive_backoff` configuration. + After this enhancement, EMQX checks the client's keepalive timeout status + period by multiplying the "Client Requested Keepalive Interval" with `keepalive_multiplier`. + +- [#10713](https://github.com/emqx/emqx/pull/10713) We hide the request_timeout in resource_option of the webhook to keep it consistent with the http request_timeout of the webhook. + From now on, when configuring a webhook through API or configuration files, + it is no longer necessary to configure the request_timeout of the resource. Only configuring the http request_timeout is sufficient, and the request_timeout in the resource will automatically be consistent with the http request_timeout. + +- [#10511](https://github.com/emqx/emqx/pull/10511) Improve the security and privacy of some resource logs by masking sensitive information in the data. + +- [#10678](https://github.com/emqx/emqx/pull/10678) Optimized counter increment calls to avoid work if increment is zero. + +- [#10690](https://github.com/emqx/emqx/pull/10690) Added a retry mechanism to webhook bridge that attempts to improve throughput. + + This optimization retries request failures without blocking the buffering layer, which can improve throughput in situations of high messaging rate. + +- [#10698](https://github.com/emqx/emqx/pull/10698) Optimize memory usage when accessing the configuration during runtime. + +## Bug Fixes + +- [#10340](https://github.com/emqx/emqx/pull/10340) Fixed the issue that could lead to crash logs being printed when stopping EMQX via systemd. + ``` + 2023-03-29T16:43:25.915761+08:00 [error] Generic server memsup terminating. Reason: {port_died,normal}. Last message: {'EXIT',<0.2117.0>,{port_died,normal}}. State: [{data,[{"Timeout",60000}]},{items,{"Memory Usage",[{"Allocated",929959936},{"Total",3832242176}]}},{items,{"Worst Memory User",[{"Pid",<0.2031.0>},{"Memory",4720472}]}}]. + 2023-03-29T16:43:25.924764+08:00 [error] crasher: initial call: memsup:init/1, pid: <0.2116.0>, registered_name: memsup, exit: {{port_died,normal},[{gen_server,handle_common_reply,8,[{file,"gen_server.erl"},{line,811}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,226}]}]}, ancestors: [os_mon_sup,<0.2114.0>], message_queue_len: 0, messages: [], links: [<0.2115.0>], dictionary: [], trap_exit: true, status: running, heap_size: 4185, stack_size: 29, reductions: 187637; neighbours: + 2023-03-29T16:43:25.924979+08:00 [error] Supervisor: {local,os_mon_sup}. Context: child_terminated. Reason: {port_died,normal}. Offender: id=memsup,pid=<0.2116.0>. + ``` + +- [#10563](https://github.com/emqx/emqx/pull/10563) Corrected an issue where the no_local flag was not functioning correctly. + + +- [#10600](https://github.com/emqx/emqx/pull/10600) Deleted emqx_statsd application. + + +- [#10653](https://github.com/emqx/emqx/pull/10653) Store gateway authentication TLS certificates and keys in the data directory. + +- [#10677](https://github.com/emqx/emqx/pull/10677) In Rule API, reapond with 404 HTTP error code when trying to delete a rule that does not exist. + +- [#10682](https://github.com/emqx/emqx/pull/10682) Fix the timestamp for the will message is incorrectly assigned at the session creation time, now this timestamp is the disconnected time of the session. + +- [#10701](https://github.com/emqx/emqx/pull/10701) RPM package for Amazon Linux 2 did not support TLS v1.3 as it was assembled with Erlang/OTP built with openssl 1.0. + +- [#10715](https://github.com/emqx/emqx/pull/10715) Postpone trimming the connection information structure until after `client.connected` hooks have been executed. These hooks once again have access to the client's peer certificate. + +- [#10717](https://github.com/emqx/emqx/pull/10717) Fixed an issue where the buffering layer processes could use a lot of CPU when inflight window is full. + +- [#10724](https://github.com/emqx/emqx/pull/10724) A summary has been added for all endpoints in the HTTP API documentation (accessible at "http://emqx_host_name:18083/api-docs"). + +- [#10726](https://github.com/emqx/emqx/pull/10726) Validate Health Check Interval and Auto Restart Interval against the range from 1ms to 1 hour. + +- [#10728](https://github.com/emqx/emqx/pull/10728) Fixed an issue where the rule engine was unable to access variables exported by `FOREACH` in the `DO` clause. + + Given a payload: `{"date": "2023-05-06", "array": ["a"]}`, as well as the following SQL statement: + ``` + FOREACH payload.date as date, payload.array as elem + DO date, elem + FROM "t/#" + ``` + Prior to the fix, the `date` variable exported by `FOREACH` could not be accessed in the `DO` clause of the above SQL, resulting in the following output for the SQL statement: + `[{"elem": "a","date": "undefined"}]`. + After the fix, the output of the SQL statement is: `[{"elem": "a","date": "2023-05-06"}]` + +- [#10737](https://github.com/emqx/emqx/pull/10737) Fix the issue where the HTTP API interface of Gateway cannot handle ClientIDs with + special characters, such as: `!@#$%^&*()_+{}:"<>?/`. + +- [#10742](https://github.com/emqx/emqx/pull/10742) Check the correctness of the rules before saving the authorization file source. + Previously, Saving wrong rules could lead to restart failure. + +- [#10743](https://github.com/emqx/emqx/pull/10743) Fixes an issue where trying to get a bridge info or metrics could result in a crash when a node is joining a cluster. + +- [#10746](https://github.com/emqx/emqx/pull/10746) Add missing support of the event `$events/delivery_dropped` into the rule engine test API `rule_test`. + +- [#10747](https://github.com/emqx/emqx/pull/10747) Refactor date and time functions, `format_date` and `date_to_unix_ts`, in the rule engine to fix the implementation problem. + +- [#10755](https://github.com/emqx/emqx/pull/10755) Fixed data bridge resource update race condition. + + In the 'delete + create' process for EMQX resource updates, + long bridge creation times could cause dashboard request timeouts. + If a bridge resource update was initiated before completion of its creation, + it led to an erroneous deletion from the runtime, despite being present in the config file. + + This fix addresses the race condition in bridge resource updates, + ensuring the accurate identification and addition of new resources, + maintaining consistency between runtime and configuration file statuses. + +- [#10760](https://github.com/emqx/emqx/pull/10760) Fix Internal Error 500 that occurred sometimes when bridge statistics page was updated while a node was (re)joining the cluster. + +- [#10761](https://github.com/emqx/emqx/pull/10761) Fixing the issue where the default value of SSL certificate for Dashboard Listener was not correctly interpolated, which caused HTTPS to be inaccessible when verify_peer and cacertfile were using the default configuration. + +- [#10785](https://github.com/emqx/emqx/pull/10785) Ensure `EMQX_LOG_DIR` is set by Windows boot script. + + The environment variable `EMQX_LOG_DIR` was missing in v5.0.25, caused EMQX Windows package fail to boot unless set by sysadmin. + +- [#10801](https://github.com/emqx/emqx/pull/10801) Avoid duplicated percent decode the topic name in API `/topics/{topic}` and `/topics`. + +- [#10809](https://github.com/emqx/emqx/pull/10809) Address `** ERROR ** Mnesia post_commit hook failed: error:badarg` error messages happening during node shutdown or restart. + Mria pull request: https://github.com/emqx/mria/pull/142 + +- [#10817](https://github.com/emqx/emqx/pull/10817) Fix the error of not being able to configure `auto_restart_interval` as infinity + +- [#10818](https://github.com/emqx/emqx/pull/10818) Fixing `emqx_ctl traces` command. diff --git a/dev b/dev index 485235432..5087cc30f 100755 --- a/dev +++ b/dev @@ -43,8 +43,8 @@ OPTIONS: ENVIRONMENT VARIABLES: - PROFILE: Overriden by '-p|--profile' option, defaults to 'emqx'. - EMQX_NODE_NAME: Overriden by '-n|--name' or '-r|--remsh' option. + PROFILE: Overridden by '-p|--profile' option, defaults to 'emqx'. + EMQX_NODE_NAME: Overridden by '-n|--name' or '-r|--remsh' option. The node name of the EMQX node. Default to emqx@127.0.0.1'. EMQX_NODE_COOKIE: Erlang cookie, defaults to ~/.erlang.cookie @@ -56,6 +56,9 @@ if [ -n "${DEBUG:-}" ]; then fi export HOCON_ENV_OVERRIDE_PREFIX='EMQX_' +export EMQX_LOG__FILE__DEFAULT__ENABLE='false' +export EMQX_LOG__CONSOLE__ENABLE='true' +SYSTEM="$(./scripts/get-distro.sh)" EMQX_NODE_NAME="${EMQX_NODE_NAME:-emqx@127.0.0.1}" PROFILE="${PROFILE:-emqx}" FORCE_COMPILE=0 @@ -129,10 +132,10 @@ export PROFILE case "${PROFILE}" in emqx) - SCHEMA_MOD='emqx_conf_schema' + export SCHEMA_MOD='emqx_conf_schema' ;; emqx-enterprise) - SCHEMA_MOD='emqx_ee_conf_schema' + export SCHEMA_MOD='emqx_enterprise_schema' ;; esac @@ -155,14 +158,24 @@ fi prepare_erl_libs() { local profile="$1" local libs_dir="_build/${profile}/lib" - local erl_libs='' + local erl_libs="${ERL_LIBS:-}" + local sep + if [ "${SYSTEM}" = 'windows' ]; then + sep=';' + else + sep=':' + fi if [ $FORCE_COMPILE -eq 1 ] || [ ! -d "$libs_dir" ]; then make "compile-${PROFILE}" else echo "Running from code in $libs_dir" fi for app in "${libs_dir}"/*; do - erl_libs="${erl_libs}:${app}" + if [ -n "$erl_libs" ]; then + erl_libs="${erl_libs}${sep}${app}" + else + erl_libs="${app}" + fi done export ERL_LIBS="$erl_libs" } @@ -332,7 +345,7 @@ boot() { BOOT_SEQUENCE=" Apps=[${APPS}], ok=lists:foreach(fun application:load/1, Apps), - io:format(user, \"~nLoaded ~p apps~n\", [length(Apps)]), + io:format(user, \"~nLoaded: ~p~n\", [Apps]), {ok, _} = application:ensure_all_started(emqx_machine). " diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct index de5d8c3b1..7cbc30567 100644 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ b/lib-ee/emqx_ee_bridge/docker-ct @@ -4,4 +4,3 @@ mongo_rs_sharded mysql redis redis_cluster -clickhouse diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index bff42dd98..4a52b2c35 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_bridge, [ {description, "EMQX Enterprise data bridges"}, - {vsn, "0.1.13"}, + {vsn, "0.1.14"}, {registered, []}, {applications, [ kernel, @@ -17,7 +17,8 @@ emqx_bridge_rocketmq, emqx_bridge_rabbitmq, emqx_bridge_tdengine, - emqx_bridge_influxdb + emqx_bridge_influxdb, + emqx_bridge_clickhouse ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 804e7d814..3636e3eb2 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -36,7 +36,7 @@ api_schemas(Method) -> api_ref(emqx_bridge_timescale, <<"timescale">>, Method), api_ref(emqx_bridge_matrix, <<"matrix">>, Method), api_ref(emqx_bridge_tdengine, <<"tdengine">>, Method), - api_ref(emqx_ee_bridge_clickhouse, <<"clickhouse">>, Method), + api_ref(emqx_bridge_clickhouse, <<"clickhouse">>, Method), api_ref(emqx_bridge_dynamo, <<"dynamo">>, Method), api_ref(emqx_bridge_rocketmq, <<"rocketmq">>, Method), api_ref(emqx_bridge_sqlserver, <<"sqlserver">>, Method), @@ -61,7 +61,7 @@ schema_modules() -> emqx_bridge_timescale, emqx_bridge_matrix, emqx_bridge_tdengine, - emqx_ee_bridge_clickhouse, + emqx_bridge_clickhouse, emqx_bridge_dynamo, emqx_bridge_rocketmq, emqx_bridge_sqlserver, @@ -105,7 +105,7 @@ resource_type(pgsql) -> emqx_connector_pgsql; resource_type(timescale) -> emqx_connector_pgsql; resource_type(matrix) -> emqx_connector_pgsql; resource_type(tdengine) -> emqx_bridge_tdengine_connector; -resource_type(clickhouse) -> emqx_ee_connector_clickhouse; +resource_type(clickhouse) -> emqx_bridge_clickhouse_connector; resource_type(dynamo) -> emqx_bridge_dynamo_connector; resource_type(rocketmq) -> emqx_bridge_rocketmq_connector; resource_type(sqlserver) -> emqx_bridge_sqlserver_connector; @@ -301,7 +301,7 @@ clickhouse_structs() -> [ {clickhouse, mk( - hoconsc:map(name, ref(emqx_ee_bridge_clickhouse, "config")), + hoconsc:map(name, ref(emqx_bridge_clickhouse, "config")), #{ desc => <<"Clickhouse Bridge Config">>, required => false diff --git a/lib-ee/emqx_ee_conf/.gitignore b/lib-ee/emqx_ee_conf/.gitignore deleted file mode 100644 index f1c455451..000000000 --- a/lib-ee/emqx_ee_conf/.gitignore +++ /dev/null @@ -1,19 +0,0 @@ -.rebar3 -_* -.eunit -*.o -*.beam -*.plt -*.swp -*.swo -.erlang.cookie -ebin -log -erl_crash.dump -.rebar -logs -_build -.idea -*.iml -rebar3.crashdump -*~ diff --git a/lib-ee/emqx_ee_conf/README.md b/lib-ee/emqx_ee_conf/README.md deleted file mode 100644 index 701d285cc..000000000 --- a/lib-ee/emqx_ee_conf/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# emqx_ee_conf - -EMQX Enterprise configuration schema diff --git a/lib-ee/emqx_ee_conf/etc/emqx-enterprise.conf b/lib-ee/emqx_ee_conf/etc/emqx-enterprise.conf deleted file mode 100644 index 8da63dad9..000000000 --- a/lib-ee/emqx_ee_conf/etc/emqx-enterprise.conf +++ /dev/null @@ -1 +0,0 @@ -telemetry.enable = false diff --git a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl b/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl deleted file mode 100644 index 941c3e4d1..000000000 --- a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl +++ /dev/null @@ -1,53 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- - --module(emqx_ee_conf_schema_SUITE). - --compile(nowarn_export_all). --compile(export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - -all() -> - emqx_common_test_helpers:all(?MODULE). - -%%------------------------------------------------------------------------------ -%% Tests -%%------------------------------------------------------------------------------ - -t_namespace(_Config) -> - ?assertEqual( - emqx_conf_schema:namespace(), - emqx_ee_conf_schema:namespace() - ). - -t_roots(_Config) -> - BaseRoots = emqx_conf_schema:roots(), - EnterpriseRoots = emqx_ee_conf_schema:roots(), - - ?assertEqual([], BaseRoots -- EnterpriseRoots), - - ?assert( - lists:any( - fun - ({license, _}) -> true; - (_) -> false - end, - EnterpriseRoots - ) - ). - -t_fields(_Config) -> - ?assertEqual( - emqx_conf_schema:fields("node"), - emqx_ee_conf_schema:fields("node") - ). - -t_translations(_Config) -> - [Root | _] = emqx_ee_conf_schema:translations(), - ?assertEqual( - emqx_conf_schema:translation(Root), - emqx_ee_conf_schema:translation(Root) - ). diff --git a/lib-ee/emqx_ee_connector/docker-ct b/lib-ee/emqx_ee_connector/docker-ct index 3db090939..ef579c036 100644 --- a/lib-ee/emqx_ee_connector/docker-ct +++ b/lib-ee/emqx_ee_connector/docker-ct @@ -1,3 +1,2 @@ toxiproxy influxdb -clickhouse diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index 3414c80b5..ee1d4e500 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -2,7 +2,6 @@ {erl_opts, [debug_info]}. {deps, [ {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, - {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, {emqx, {path, "../../apps/emqx"}}, {emqx_utils, {path, "../../apps/emqx_utils"}} ]}. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 9a4f36cf3..3ed460492 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -1,13 +1,12 @@ {application, emqx_ee_connector, [ {description, "EMQX Enterprise connectors"}, - {vsn, "0.1.12"}, + {vsn, "0.1.13"}, {registered, []}, {applications, [ kernel, stdlib, ecpool, - hstreamdb_erl, - clickhouse + hstreamdb_erl ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src index aa43cf248..b74e9fa7d 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_schema_registry, [ {description, "EMQX Schema Registry"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, [emqx_ee_schema_registry_sup]}, {mod, {emqx_ee_schema_registry_app, []}}, {applications, [ diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl index b1453914b..5ffcb2ba6 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl @@ -58,7 +58,11 @@ get_serde(SchemaName) -> -spec get_schema(schema_name()) -> {ok, map()} | {error, not_found}. get_schema(SchemaName) -> - case emqx_config:get([?CONF_KEY_ROOT, schemas, SchemaName], undefined) of + case + emqx_config:get( + [?CONF_KEY_ROOT, schemas, binary_to_atom(SchemaName)], undefined + ) + of undefined -> {error, not_found}; Config -> diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl index 1f53766e3..99c4fa155 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl @@ -353,7 +353,7 @@ cluster(Config) -> {priv_data_dir, PrivDataDir}, {load_schema, true}, {start_autocluster, true}, - {schema_mod, emqx_ee_conf_schema}, + {schema_mod, emqx_enterprise_schema}, %% need to restart schema registry app in the tests so %% that it re-registers the config handler that is lost %% when emqx_conf restarts during join. diff --git a/lib-ee/emqx_license/src/emqx_license.app.src b/lib-ee/emqx_license/src/emqx_license.app.src index fcdcbc05b..354385faf 100644 --- a/lib-ee/emqx_license/src/emqx_license.app.src +++ b/lib-ee/emqx_license/src/emqx_license.app.src @@ -1,6 +1,6 @@ {application, emqx_license, [ {description, "EMQX License"}, - {vsn, "5.0.9"}, + {vsn, "5.0.10"}, {modules, []}, {registered, [emqx_license_sup]}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/lib-ee/emqx_license/src/emqx_license.erl b/lib-ee/emqx_license/src/emqx_license.erl index ef285b937..3e29dcf25 100644 --- a/lib-ee/emqx_license/src/emqx_license.erl +++ b/lib-ee/emqx_license/src/emqx_license.erl @@ -1,6 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- + -module(emqx_license). -include("emqx_license.hrl"). diff --git a/lib-ee/emqx_license/src/emqx_license_checker.erl b/lib-ee/emqx_license/src/emqx_license_checker.erl index 2ebb96004..da777ff84 100644 --- a/lib-ee/emqx_license/src/emqx_license_checker.erl +++ b/lib-ee/emqx_license/src/emqx_license_checker.erl @@ -117,7 +117,7 @@ handle_cast(_Msg, State) -> handle_info(check_license, #{license := License} = State) -> #{} = check_license(License), NewState = ensure_check_license_timer(State), - ?tp(debug, emqx_license_checked, #{}), + ?tp(emqx_license_checked, #{}), {noreply, NewState}; handle_info(check_expiry_alarm, #{license := License} = State) -> ok = expiry_early_alarm(License), diff --git a/lib-ee/emqx_license/src/emqx_license_installer.erl b/lib-ee/emqx_license/src/emqx_license_installer.erl deleted file mode 100644 index 58ee6ebcc..000000000 --- a/lib-ee/emqx_license/src/emqx_license_installer.erl +++ /dev/null @@ -1,86 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- --module(emqx_license_installer). - --include_lib("snabbkaffe/include/snabbkaffe.hrl"). - --behaviour(gen_server). - --export([ - start_link/1, - start_link/4 -]). - -%% gen_server callbacks --export([ - init/1, - handle_call/3, - handle_cast/2, - handle_info/2 -]). - --define(NAME, emqx). --define(INTERVAL, 5000). - -%%------------------------------------------------------------------------------ -%% API -%%------------------------------------------------------------------------------ - -start_link(Callback) -> - start_link(?NAME, ?MODULE, ?INTERVAL, Callback). - -start_link(Name, ServerName, Interval, Callback) -> - gen_server:start_link({local, ServerName}, ?MODULE, [Name, Interval, Callback], []). - -%%------------------------------------------------------------------------------ -%% gen_server callbacks -%%------------------------------------------------------------------------------ - -init([Name, Interval, Callback]) -> - Pid = whereis(Name), - State = #{ - interval => Interval, - name => Name, - pid => Pid, - callback => Callback - }, - {ok, ensure_timer(State)}. - -handle_call(_Req, _From, State) -> - {reply, unknown, State}. - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info({timeout, Timer, check_pid}, #{timer := Timer} = State) -> - NewState = check_pid(State), - {noreply, ensure_timer(NewState)}; -handle_info(_Msg, State) -> - {noreply, State}. - -%%------------------------------------------------------------------------------ -%% Private functions -%%------------------------------------------------------------------------------ - -ensure_timer(#{interval := Interval} = State) -> - _ = - case State of - #{timer := Timer} -> erlang:cancel_timer(Timer); - _ -> ok - end, - State#{timer => erlang:start_timer(Interval, self(), check_pid)}. - -check_pid(#{name := Name, pid := OldPid, callback := Callback} = State) -> - case whereis(Name) of - undefined -> - ?tp(debug, emqx_license_installer_noproc, #{old_pid => OldPid}), - State; - OldPid -> - ?tp(debug, emqx_license_installer_nochange, #{old_pid => OldPid}), - State; - NewPid -> - _ = Callback(), - ?tp(debug, emqx_license_installer_called, #{old_pid => OldPid}), - State#{pid => NewPid} - end. diff --git a/lib-ee/emqx_license/src/emqx_license_resources.erl b/lib-ee/emqx_license/src/emqx_license_resources.erl index 2cc62b8a3..9a63e5e06 100644 --- a/lib-ee/emqx_license/src/emqx_license_resources.erl +++ b/lib-ee/emqx_license/src/emqx_license_resources.erl @@ -76,7 +76,7 @@ handle_cast(_Msg, State) -> handle_info(update_resources, State) -> true = update_resources(), connection_quota_early_alarm(), - ?tp(debug, emqx_license_resources_updated, #{}), + ?tp(emqx_license_resources_updated, #{}), {noreply, ensure_timer(State)}. terminate(_Reason, _State) -> diff --git a/lib-ee/emqx_license/src/emqx_license_sup.erl b/lib-ee/emqx_license/src/emqx_license_sup.erl index 6b8f73953..304fac313 100644 --- a/lib-ee/emqx_license/src/emqx_license_sup.erl +++ b/lib-ee/emqx_license/src/emqx_license_sup.erl @@ -41,15 +41,6 @@ init([]) -> shutdown => 5000, type => worker, modules => [emqx_license_resources] - }, - - #{ - id => license_installer, - start => {emqx_license_installer, start_link, [fun emqx_license:load/0]}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_license_installer] } ] }}. diff --git a/lib-ee/emqx_license/test/emqx_license_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_SUITE.erl index 4a0f6d91b..69adabe76 100644 --- a/lib-ee/emqx_license/test/emqx_license_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_SUITE.erl @@ -73,10 +73,10 @@ setup_test(TestCase, Config) when [ {apps, [emqx_conf, emqx_license]}, {load_schema, false}, - {schema_mod, emqx_ee_conf_schema}, + {schema_mod, emqx_enterprise_schema}, {env_handler, fun (emqx) -> - emqx_config:save_schema_mod_and_names(emqx_ee_conf_schema), + emqx_config:save_schema_mod_and_names(emqx_enterprise_schema), %% emqx_config:save_schema_mod_and_names(emqx_license_schema), application:set_env(emqx, boot_modules, []), application:set_env( @@ -90,7 +90,7 @@ setup_test(TestCase, Config) when ), ok; (emqx_conf) -> - emqx_config:save_schema_mod_and_names(emqx_ee_conf_schema), + emqx_config:save_schema_mod_and_names(emqx_enterprise_schema), %% emqx_config:save_schema_mod_and_names(emqx_license_schema), application:set_env( emqx, diff --git a/lib-ee/emqx_license/test/emqx_license_installer_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_installer_SUITE.erl deleted file mode 100644 index 5d5e27489..000000000 --- a/lib-ee/emqx_license/test/emqx_license_installer_SUITE.erl +++ /dev/null @@ -1,89 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- - --module(emqx_license_installer_SUITE). - --compile(nowarn_export_all). --compile(export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). - -all() -> - emqx_common_test_helpers:all(?MODULE). - -init_per_suite(Config) -> - _ = application:load(emqx_conf), - emqx_common_test_helpers:start_apps([emqx_license], fun set_special_configs/1), - Config. - -end_per_suite(_) -> - emqx_common_test_helpers:stop_apps([emqx_license]), - ok. - -init_per_testcase(_Case, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), - Config. - -end_per_testcase(_Case, _Config) -> - ok. - -set_special_configs(emqx_license) -> - Config = #{key => emqx_license_test_lib:default_license()}, - emqx_config:put([license], Config); -set_special_configs(_) -> - ok. - -%%------------------------------------------------------------------------------ -%% Tests -%%------------------------------------------------------------------------------ - -t_update(_Config) -> - ?check_trace( - begin - ?wait_async_action( - begin - Pid0 = spawn_link(fun() -> - receive - exit -> ok - end - end), - register(installer_test, Pid0), - - {ok, _} = emqx_license_installer:start_link( - installer_test, - ?MODULE, - 10, - fun() -> ok end - ), - - {ok, _} = ?block_until( - #{?snk_kind := emqx_license_installer_nochange}, - 100 - ), - - Pid0 ! exit, - - {ok, _} = ?block_until( - #{?snk_kind := emqx_license_installer_noproc}, - 100 - ), - - Pid1 = spawn_link(fun() -> timer:sleep(100) end), - register(installer_test, Pid1) - end, - #{?snk_kind := emqx_license_installer_called}, - 1000 - ) - end, - fun(Trace) -> - ?assertMatch([_ | _], ?of_kind(emqx_license_installer_called, Trace)) - end - ). - -t_unknown_calls(_Config) -> - ok = gen_server:cast(emqx_license_installer, some_cast), - some_msg = erlang:send(emqx_license_installer, some_msg), - ?assertEqual(unknown, gen_server:call(emqx_license_installer, some_request)). diff --git a/mix.exs b/mix.exs index 52b49f246..3eab0d70d 100644 --- a/mix.exs +++ b/mix.exs @@ -55,10 +55,10 @@ defmodule EMQXUmbrella.MixProject do {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.6", override: true}, {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.7.2-emqx-9", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.15.1", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.15.2", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true}, - {:minirest, github: "emqx/minirest", tag: "1.3.9", override: true}, + {:minirest, github: "emqx/minirest", tag: "1.3.10", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.39.4", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.39.7", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, @@ -102,11 +102,11 @@ defmodule EMQXUmbrella.MixProject do end defp emqx_apps(profile_info, version) do - apps = umbrella_apps() ++ enterprise_apps(profile_info) + apps = umbrella_apps(profile_info) ++ enterprise_apps(profile_info) set_emqx_app_system_env(apps, profile_info, version) end - defp umbrella_apps() do + defp umbrella_apps(profile_info) do enterprise_apps = enterprise_umbrella_apps() "apps/*" @@ -124,6 +124,15 @@ defmodule EMQXUmbrella.MixProject do |> elem(0) |> then(&MapSet.member?(enterprise_apps, &1)) end) + |> Enum.reject(fn {app, _} -> + case profile_info do + %{edition_type: :enterprise} -> + app == :emqx_telemetry + + _ -> + false + end + end) end defp enterprise_apps(_profile_info = %{edition_type: :enterprise}) do @@ -158,7 +167,6 @@ defmodule EMQXUmbrella.MixProject do :emqx_bridge_gcp_pubsub, :emqx_bridge_cassandra, :emqx_bridge_opents, - :emqx_bridge_clickhouse, :emqx_bridge_dynamo, :emqx_bridge_hstreamdb, :emqx_bridge_influxdb, @@ -176,6 +184,7 @@ defmodule EMQXUmbrella.MixProject do :emqx_oracle, :emqx_bridge_oracle, :emqx_bridge_rabbitmq, + :emqx_bridge_clickhouse, :emqx_ft, :emqx_s3 ]) @@ -303,7 +312,6 @@ defmodule EMQXUmbrella.MixProject do :emqx_bridge, :emqx_modules, :emqx_management, - :emqx_statsd, :emqx_retainer, :emqx_prometheus, :emqx_auto_subscribe, @@ -369,7 +377,6 @@ defmodule EMQXUmbrella.MixProject do emqx_management: :permanent, emqx_dashboard: :permanent, emqx_retainer: :permanent, - emqx_statsd: :permanent, emqx_prometheus: :permanent, emqx_psk: :permanent, emqx_slow_subs: :permanent, @@ -386,7 +393,7 @@ defmodule EMQXUmbrella.MixProject do if(edition_type == :enterprise, do: [ emqx_license: :permanent, - emqx_ee_conf: :load, + emqx_enterprise: :load, emqx_ee_connector: :permanent, emqx_ee_bridge: :permanent, emqx_bridge_kafka: :permanent, @@ -416,7 +423,9 @@ defmodule EMQXUmbrella.MixProject do emqx_node_rebalance: :permanent, emqx_ft: :permanent ], - else: [] + else: [ + emqx_telemetry: :permanent + ] ) end @@ -555,14 +564,6 @@ defmodule EMQXUmbrella.MixProject do Path.join(etc, "emqx.conf") ) - if edition_type == :enterprise do - render_template( - "apps/emqx_conf/etc/emqx-enterprise.conf.all", - assigns, - Path.join(etc, "emqx-enterprise.conf") - ) - end - render_template( "rel/emqx_vars", assigns, @@ -788,7 +789,7 @@ defmodule EMQXUmbrella.MixProject do end end - defp emqx_schema_mod(:enterprise), do: :emqx_ee_conf_schema + defp emqx_schema_mod(:enterprise), do: :emqx_enterprise_schema defp emqx_schema_mod(:community), do: :emqx_conf_schema defp bcrypt_dep() do diff --git a/rebar.config b/rebar.config index bceac3005..0aab04892 100644 --- a/rebar.config +++ b/rebar.config @@ -62,10 +62,10 @@ , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}} , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-9"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.1"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.2"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.9"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.10"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.4"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.7"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} diff --git a/rebar.config.erl b/rebar.config.erl index 3a8ff6d84..a8474a703 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -348,7 +348,7 @@ overlay_vars_edition(ce) -> ]; overlay_vars_edition(ee) -> [ - {emqx_schema_mod, emqx_ee_conf_schema}, + {emqx_schema_mod, emqx_enterprise_schema}, {is_enterprise, "yes"} ]. @@ -431,7 +431,6 @@ relx_apps(ReleaseType, Edition) -> emqx_management, emqx_dashboard, emqx_retainer, - emqx_statsd, emqx_prometheus, emqx_psk, emqx_slow_subs, @@ -455,7 +454,7 @@ is_app(Name) -> relx_apps_per_edition(ee) -> [ emqx_license, - {emqx_ee_conf, load}, + {emqx_enterprise, load}, emqx_ee_connector, emqx_ee_bridge, emqx_bridge_kafka, @@ -486,7 +485,7 @@ relx_apps_per_edition(ee) -> emqx_ft ]; relx_apps_per_edition(ce) -> - []. + [emqx_telemetry]. relx_overlay(ReleaseType, Edition) -> [ @@ -515,8 +514,8 @@ relx_overlay(ReleaseType, Edition) -> {copy, "bin/nodetool", "bin/nodetool-{{release_version}}"} ] ++ etc_overlay(ReleaseType, Edition). -etc_overlay(ReleaseType, Edition) -> - Templates = emqx_etc_overlay(ReleaseType, Edition), +etc_overlay(ReleaseType, _Edition) -> + Templates = emqx_etc_overlay(ReleaseType), [ {mkdir, "etc/"}, {copy, "{{base_dir}}/lib/emqx/etc/certs", "etc/"}, @@ -530,24 +529,16 @@ etc_overlay(ReleaseType, Edition) -> Templates ). -emqx_etc_overlay(ReleaseType, Edition) -> +emqx_etc_overlay(ReleaseType) -> emqx_etc_overlay_per_rel(ReleaseType) ++ - emqx_etc_overlay_per_edition(Edition) ++ - emqx_etc_overlay_common(). + emqx_etc_overlay(). emqx_etc_overlay_per_rel(cloud) -> [{"{{base_dir}}/lib/emqx/etc/vm.args.cloud", "etc/vm.args"}]. -emqx_etc_overlay_common() -> - [{"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"}]. - -emqx_etc_overlay_per_edition(ce) -> +emqx_etc_overlay() -> [ - {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"} - ]; -emqx_etc_overlay_per_edition(ee) -> - [ - {"{{base_dir}}/lib/emqx_conf/etc/emqx-enterprise.conf.all", "etc/emqx-enterprise.conf"}, + {"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"}, {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"} ]. diff --git a/rel/i18n/emqx_ee_bridge_clickhouse.hocon b/rel/i18n/emqx_bridge_clickhouse.hocon similarity index 98% rename from rel/i18n/emqx_ee_bridge_clickhouse.hocon rename to rel/i18n/emqx_bridge_clickhouse.hocon index 6735aee22..726d1eb7c 100644 --- a/rel/i18n/emqx_ee_bridge_clickhouse.hocon +++ b/rel/i18n/emqx_bridge_clickhouse.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_clickhouse { +emqx_bridge_clickhouse { batch_value_separator.desc: """The default value ',' works for the VALUES format. You can also use other separator if other format is specified. See [INSERT INTO Statement](https://clickhouse.com/docs/en/sql-reference/statements/insert-into).""" diff --git a/rel/i18n/emqx_ee_connector_clickhouse.hocon b/rel/i18n/emqx_bridge_clickhouse_connector.hocon similarity index 89% rename from rel/i18n/emqx_ee_connector_clickhouse.hocon rename to rel/i18n/emqx_bridge_clickhouse_connector.hocon index cebba5aef..26746a52d 100644 --- a/rel/i18n/emqx_ee_connector_clickhouse.hocon +++ b/rel/i18n/emqx_bridge_clickhouse_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_clickhouse { +emqx_bridge_clickhouse_connector { base_url.desc: """The HTTP URL to the Clickhouse server that you want to connect to (for example http://myhostname:8123)""" diff --git a/rel/i18n/emqx_connector_http.hocon b/rel/i18n/emqx_connector_http.hocon index 70c644e33..b511d007a 100644 --- a/rel/i18n/emqx_connector_http.hocon +++ b/rel/i18n/emqx_connector_http.hocon @@ -1,14 +1,5 @@ emqx_connector_http { -base_url.desc: -"""The base URL is the URL includes only the scheme, host and port.
-When send an HTTP request, the real URL to be used is the concatenation of the base URL and the -path parameter
-For example: `http://localhost:9901/`""" - -base_url.label: -"""Base Url""" - body.desc: """HTTP request body.""" diff --git a/rel/i18n/emqx_mgmt_api_configs.hocon b/rel/i18n/emqx_mgmt_api_configs.hocon index 42bda7899..3255806d0 100644 --- a/rel/i18n/emqx_mgmt_api_configs.hocon +++ b/rel/i18n/emqx_mgmt_api_configs.hocon @@ -22,19 +22,19 @@ get_global_zone_configs.desc: get_global_zone_configs.label: """Get the global zone configs""" -update_globar_zone_configs.desc: -"""Update globbal zone configs""" -update_globar_zone_configs.label: -"""Update globbal zone configs""" +update_global_zone_configs.desc: +"""Update global zone configs""" +update_global_zone_configs.label: +"""Update global zone configs""" -get_node_level_limiter_congigs.desc: +get_node_level_limiter_configs.desc: """Get the node-level limiter configs""" -get_node_level_limiter_congigs.label: +get_node_level_limiter_configs.label: """Get the node-level limiter configs""" -update_node_level_limiter_congigs.desc: +update_node_level_limiter_configs.desc: """Update the node-level limiter configs""" -update_node_level_limiter_congigs.label: +update_node_level_limiter_configs.label: """Update the node-level limiter configs""" } diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index 5d6977b47..54d866014 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -796,7 +796,7 @@ fields_tcp_opts_high_watermark.desc: by the VM socket implementation reaches this limit.""" fields_tcp_opts_high_watermark.label: -"""TCP 高水位线""" +"""TCP high watermark""" fields_mqtt_quic_listener_stateless_operation_expiration_ms.desc: """The time limit between operations for the same endpoint, in milliseconds. Default: 100""" @@ -882,11 +882,12 @@ and an MQTT message is published to the system topic $SYS/sysmon/long_sche sysmon_vm_long_schedule.label: """Enable Long Schedule monitoring.""" -mqtt_keepalive_backoff.desc: -"""The coefficient EMQX uses to confirm whether the keep alive duration of the client expires. Formula: Keep Alive * Backoff * 2""" +mqtt_keepalive_multiplier.desc: +"""Keep-Alive Timeout = Keep-Alive interval × Keep-Alive Multiplier. +The default value 1.5 is following the MQTT 5.0 specification. This multiplier is adjustable, providing system administrators flexibility for tailoring to their specific needs. For instance, if a client's 10-second Keep-Alive interval PINGREQ gets delayed by an extra 10 seconds, changing the multiplier to 2 lets EMQX tolerate this delay.""" -mqtt_keepalive_backoff.label: -"""Keep Alive Backoff""" +mqtt_keepalive_multiplier.label: +"""Keep Alive Multiplier""" force_gc_bytes.desc: """GC the process after specified number of bytes have passed through.""" @@ -1318,6 +1319,11 @@ you drop support for the insecure renegotiation, prone to MitM attacks.""" common_ssl_opts_schema_secure_renegotiate.label: """SSL renegotiate""" +common_ssl_opts_schema_log_level.desc: +"""Log level for SSL communication. Default is 'notice'. Set to 'debug' to inspect TLS handshake messages.""" +common_ssl_opts_schema_log_level.label: +"""SSL log level""" + sysmon_vm_busy_port.desc: """When a port (e.g. TCP socket) is overloaded, there will be a busy_port warning log, and an MQTT message is published to the system topic $SYS/sysmon/busy_port.""" diff --git a/rel/i18n/emqx_statsd_api.hocon b/rel/i18n/emqx_statsd_api.hocon deleted file mode 100644 index d8bab13a7..000000000 --- a/rel/i18n/emqx_statsd_api.hocon +++ /dev/null @@ -1,9 +0,0 @@ -emqx_statsd_api { - -get_statsd_config_api.desc: -"""List the configuration of StatsD metrics collection and push service.""" - -update_statsd_config_api.desc: -"""Update the configuration of StatsD metrics collection and push service.""" - -} diff --git a/rel/i18n/emqx_statsd_schema.hocon b/rel/i18n/emqx_statsd_schema.hocon deleted file mode 100644 index fc21710c4..000000000 --- a/rel/i18n/emqx_statsd_schema.hocon +++ /dev/null @@ -1,30 +0,0 @@ -emqx_statsd_schema { - -enable.desc: -"""Enable or disable StatsD metrics collection and push service.""" - -flush_interval.desc: -"""The push interval for metrics.""" - -get_statsd_config_api.desc: -"""List the configuration of StatsD metrics collection and push service.""" - -sample_interval.desc: -"""The sampling interval for metrics.""" - -server.desc: -"""StatsD server address.""" - -statsd.desc: -"""StatsD metrics collection and push configuration.""" - -statsd.label: -"""StatsD""" - -tags.desc: -"""The tags for metrics.""" - -update_statsd_config_api.desc: -"""Update the configuration of StatsD metrics collection and push service.""" - -} diff --git a/rel/i18n/emqx_telemetry_schema.hocon b/rel/i18n/emqx_telemetry_schema.hocon new file mode 100644 index 000000000..b49a58a62 --- /dev/null +++ b/rel/i18n/emqx_telemetry_schema.hocon @@ -0,0 +1,8 @@ +emqx_telemetry_schema { +telemetry_root_doc.desc: +"""Configure telemetry data report from this EMQX node to EMQ's telemetry data collection server. +See https://www.emqx.io/docs/en/v5.0/telemetry/telemetry.html for more details.""" + +enable.desc: +"""Set to `false` disable telemetry data report""" +} diff --git a/rel/i18n/zh/emqx_ee_bridge_clickhouse.hocon b/rel/i18n/zh/emqx_bridge_clickhouse.hocon similarity index 97% rename from rel/i18n/zh/emqx_ee_bridge_clickhouse.hocon rename to rel/i18n/zh/emqx_bridge_clickhouse.hocon index a3ede08ba..ff291f667 100644 --- a/rel/i18n/zh/emqx_ee_bridge_clickhouse.hocon +++ b/rel/i18n/zh/emqx_bridge_clickhouse.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_clickhouse { +emqx_bridge_clickhouse { batch_value_separator.desc: """默认为逗号 ','，适用于 VALUE 格式。您也可以使用其他分隔符， 请参考 [INSERT INTO 语句](https://clickhouse.com/docs/en/sql-reference/statements/insert-into)。""" diff --git a/rel/i18n/zh/emqx_ee_connector_clickhouse.hocon b/rel/i18n/zh/emqx_bridge_clickhouse_connector.hocon similarity index 88% rename from rel/i18n/zh/emqx_ee_connector_clickhouse.hocon rename to rel/i18n/zh/emqx_bridge_clickhouse_connector.hocon index f1457a1f6..b6fd7f3b7 100644 --- a/rel/i18n/zh/emqx_ee_connector_clickhouse.hocon +++ b/rel/i18n/zh/emqx_bridge_clickhouse_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_clickhouse { +emqx_bridge_clickhouse_connector { base_url.desc: """你想连接到的Clickhouse服务器的HTTP URL（例如http://myhostname:8123）。""" diff --git a/rel/i18n/zh/emqx_connector_http.hocon b/rel/i18n/zh/emqx_connector_http.hocon index 5d6398b2e..af7869e12 100644 --- a/rel/i18n/zh/emqx_connector_http.hocon +++ b/rel/i18n/zh/emqx_connector_http.hocon @@ -1,13 +1,5 @@ emqx_connector_http { -base_url.desc: -"""base URL 只包含host和port。
-发送HTTP请求时，真实的URL是由base URL 和 path parameter连接而成。
-示例：`http://localhost:9901/`""" - -base_url.label: -"""Base Url""" - body.desc: """HTTP请求报文主体。""" diff --git a/rel/i18n/zh/emqx_schema.hocon b/rel/i18n/zh/emqx_schema.hocon index 0e329eac9..4c8e1f81e 100644 --- a/rel/i18n/zh/emqx_schema.hocon +++ b/rel/i18n/zh/emqx_schema.hocon @@ -843,10 +843,10 @@ sysmon_vm_long_schedule.desc: sysmon_vm_long_schedule.label: """启用长调度监控""" -mqtt_keepalive_backoff.desc: +mqtt_keepalive_multiplier.desc: """EMQX 判定客户端保活超时使用的阈值系数。计算公式为：Keep Alive * Backoff * 2""" -mqtt_keepalive_backoff.label: +mqtt_keepalive_multiplier.label: """保活超时阈值系数""" force_gc_bytes.desc: diff --git a/rel/i18n/zh/emqx_statsd_api.hocon b/rel/i18n/zh/emqx_statsd_api.hocon deleted file mode 100644 index 5761e2431..000000000 --- a/rel/i18n/zh/emqx_statsd_api.hocon +++ /dev/null @@ -1,9 +0,0 @@ -emqx_statsd_api { - -get_statsd_config_api.desc: -"""列出 StatsD 指标采集和推送服务的的配置。""" - -update_statsd_config_api.desc: -"""更新 StatsD 指标采集和推送服务的配置。""" - -} diff --git a/rel/i18n/zh/emqx_statsd_schema.hocon b/rel/i18n/zh/emqx_statsd_schema.hocon deleted file mode 100644 index 548ad33bc..000000000 --- a/rel/i18n/zh/emqx_statsd_schema.hocon +++ /dev/null @@ -1,30 +0,0 @@ -emqx_statsd_schema { - -enable.desc: -"""启用或禁用 StatsD 指标采集和推送服务。""" - -flush_interval.desc: -"""指标的推送间隔。""" - -get_statsd_config_api.desc: -"""列出 StatsD 指标采集和推送服务的的配置。""" - -sample_interval.desc: -"""指标的采样间隔。""" - -server.desc: -"""StatsD 服务器地址。""" - -statsd.desc: -"""StatsD 指标采集与推送配置。""" - -statsd.label: -"""StatsD""" - -tags.desc: -"""指标的标签。""" - -update_statsd_config_api.desc: -"""更新 StatsD 指标采集和推送服务的配置。""" - -} diff --git a/rel/i18n/zh/emqx_telemetry_schema.hocon b/rel/i18n/zh/emqx_telemetry_schema.hocon new file mode 100644 index 000000000..0fa238ee1 --- /dev/null +++ b/rel/i18n/zh/emqx_telemetry_schema.hocon @@ -0,0 +1,8 @@ +emqx_telemetry_schema { +telemetry_root_doc.desc: +"""配置 EMQX 节点向EMQ 的遥测服务器发送要测数据。 +详情请参考 https://www.emqx.io/docs/zh/v5.0/telemetry/telemetry.html。""" + +enable.desc: +"""设置为 `false` 可以关闭数据发送。""" +} diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index 2918ff5f7..92dbc2e7c 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -30,6 +30,10 @@ for app in ${APPS}; do else old_app_version='not_found' fi + if [ ! -f "$src_file" ]; then + # app is deleted + continue + fi now_app_version=$(grep -E 'vsn' "$src_file" | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') if [ "$old_app_version" = 'not_found' ]; then diff --git a/scripts/merge-config.escript b/scripts/merge-config.escript index 14ec979f2..812bfea2c 100755 --- a/scripts/merge-config.escript +++ b/scripts/merge-config.escript @@ -14,35 +14,15 @@ main(_) -> {ok, BaseConf} = file:read_file("apps/emqx_conf/etc/emqx_conf.conf"), Cfgs = get_all_cfgs("apps/"), - IsEnterprise = is_enterprise(), - Enterprise = - case IsEnterprise of - false -> []; - true -> [io_lib:nl(), "include emqx-enterprise.conf", io_lib:nl()] - end, Conf = [ merge(BaseConf, Cfgs), - io_lib:nl(), - Enterprise + io_lib:nl() ], ok = file:write_file("apps/emqx_conf/etc/emqx.conf.all", Conf), - - case IsEnterprise of - true -> - EnterpriseCfgs = get_all_cfgs("lib-ee"), - EnterpriseConf = merge(<<"">>, EnterpriseCfgs), - ok = file:write_file("apps/emqx_conf/etc/emqx-enterprise.conf.all", EnterpriseConf); - false -> - ok - end, merge_desc_files_per_lang("en"), %% TODO: remove this when we have zh translation moved to dashboard package merge_desc_files_per_lang("zh"). -is_enterprise() -> - Profile = os:getenv("PROFILE", "emqx"), - nomatch =/= string:find(Profile, "enterprise"). - merge(BaseConf, Cfgs) -> Confs = [BaseConf | lists:map(fun read_conf/1, Cfgs)], infix(lists:filter(fun(I) -> iolist_size(I) > 0 end, Confs), [io_lib:nl(), io_lib:nl()]). diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index bdbd77b3b..ea1c76d96 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -23,6 +23,7 @@ DevOps Dialyzer Diffie EIP +EMQ EMQX EPMD ERL diff --git a/scripts/spellcheck/spellcheck.sh b/scripts/spellcheck/spellcheck.sh index a21fc93ec..57ea21c55 100755 --- a/scripts/spellcheck/spellcheck.sh +++ b/scripts/spellcheck/spellcheck.sh @@ -7,7 +7,7 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." PROJ_ROOT="$(pwd)" if [ -z "${1:-}" ]; then - SCHEMA="${PROJ_ROOT}/_build/emqx/lib/emqx_dashboard/priv/www/static/schema.json" + SCHEMA="${PROJ_ROOT}/_build/docgen/emqx/schema-en.json" else SCHEMA="$(realpath "$1")" fi diff --git a/scripts/test/influx/emqx.conf b/scripts/test/influx/emqx.conf index cd8571f98..325b1e106 100644 --- a/scripts/test/influx/emqx.conf +++ b/scripts/test/influx/emqx.conf @@ -90,5 +90,4 @@ authorization { ] } -include emqx-enterprise.conf include influx-bridge.conf