diff --git a/.ci/docker-compose-file/docker-compose-ldap.yaml b/.ci/docker-compose-file/docker-compose-ldap.yaml index 03627c763..6804f6b01 100644 --- a/.ci/docker-compose-file/docker-compose-ldap.yaml +++ b/.ci/docker-compose-file/docker-compose-ldap.yaml @@ -6,6 +6,8 @@ services: build: context: ../.. dockerfile: .ci/docker-compose-file/openldap/Dockerfile + ulimits: + nofile: 1024 image: openldap #ports: # - 389:389 diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index 7de9a64fd..f2688d8d1 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -16,7 +16,7 @@ env: jobs: sanity-checks: - runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} + runs-on: ${{ fromJSON(github.repository_owner == 'emqx' && '["self-hosted","ephemeral"]' || '["ubuntu-22.04"]') }} container: "ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04" outputs: ct-matrix: ${{ steps.matrix.outputs.ct-matrix }} @@ -24,7 +24,7 @@ jobs: ct-docker: ${{ steps.matrix.outputs.ct-docker }} version-emqx: ${{ steps.matrix.outputs.version-emqx }} version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }} - runner: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} + runner_labels: ${{ github.repository_owner == 'emqx' && '["self-hosted", "ephemeral"]' || '["ubuntu-22.04"]' }} builder: "ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04" builder_vsn: "5.1-4" otp_vsn: "25.3.2-2" @@ -115,7 +115,7 @@ jobs: echo "version-emqx-enterprise=$(./pkg-vsn.sh emqx-enterprise)" | tee -a $GITHUB_OUTPUT compile: - runs-on: ${{ needs.sanity-checks.outputs.runner }} + runs-on: ${{ fromJSON(needs.sanity-checks.outputs.runner_labels) }} container: ${{ needs.sanity-checks.outputs.builder }} needs: - sanity-checks @@ -153,7 +153,7 @@ jobs: - compile uses: ./.github/workflows/run_emqx_app_tests.yaml with: - runner: ${{ needs.sanity-checks.outputs.runner }} + runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }} builder: ${{ needs.sanity-checks.outputs.builder }} before_ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }} after_ref: ${{ github.sha }} @@ -164,7 +164,7 @@ jobs: - compile uses: ./.github/workflows/run_test_cases.yaml with: - runner: ${{ needs.sanity-checks.outputs.runner }} + runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }} builder: ${{ needs.sanity-checks.outputs.builder }} ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }} ct-host: ${{ needs.sanity-checks.outputs.ct-host }} @@ -176,7 +176,7 @@ jobs: - compile uses: ./.github/workflows/static_checks.yaml with: - runner: ${{ needs.sanity-checks.outputs.runner }} + runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }} builder: ${{ needs.sanity-checks.outputs.builder }} ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }} @@ -185,7 +185,7 @@ jobs: - sanity-checks uses: ./.github/workflows/build_slim_packages.yaml with: - runner: ${{ needs.sanity-checks.outputs.runner }} + runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }} builder: ${{ needs.sanity-checks.outputs.builder }} builder_vsn: ${{ needs.sanity-checks.outputs.builder_vsn }} otp_vsn: ${{ needs.sanity-checks.outputs.otp_vsn }} @@ -196,6 +196,7 @@ jobs: - sanity-checks uses: ./.github/workflows/build_docker_for_test.yaml with: + runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }} otp_vsn: ${{ needs.sanity-checks.outputs.otp_vsn }} elixir_vsn: ${{ needs.sanity-checks.outputs.elixir_vsn }} version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} @@ -207,7 +208,7 @@ jobs: - build_slim_packages uses: ./.github/workflows/spellcheck.yaml with: - runner: ${{ needs.sanity-checks.outputs.runner }} + runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }} run_conf_tests: needs: @@ -215,7 +216,7 @@ jobs: - compile uses: ./.github/workflows/run_conf_tests.yaml with: - runner: ${{ needs.sanity-checks.outputs.runner }} + runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }} builder: ${{ needs.sanity-checks.outputs.builder }} check_deps_integrity: @@ -223,7 +224,7 @@ jobs: - sanity-checks uses: ./.github/workflows/check_deps_integrity.yaml with: - runner: ${{ needs.sanity-checks.outputs.runner }} + runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }} builder: ${{ needs.sanity-checks.outputs.builder }} run_jmeter_tests: @@ -232,6 +233,7 @@ jobs: - build_docker_for_test uses: ./.github/workflows/run_jmeter_tests.yaml with: + runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }} version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} run_docker_tests: @@ -240,7 +242,7 @@ jobs: - build_docker_for_test uses: ./.github/workflows/run_docker_tests.yaml with: - runner: ${{ needs.sanity-checks.outputs.runner }} + runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }} version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }} @@ -250,5 +252,6 @@ jobs: - build_docker_for_test uses: ./.github/workflows/run_helm_tests.yaml with: + runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }} version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }} diff --git a/.github/workflows/_push-entrypoint.yaml b/.github/workflows/_push-entrypoint.yaml index 4a9dbee24..a8a01fb45 100644 --- a/.github/workflows/_push-entrypoint.yaml +++ b/.github/workflows/_push-entrypoint.yaml @@ -11,9 +11,7 @@ on: - 'e*' branches: - 'master' - - 'release-51' - - 'release-52' - - 'release-53' + - 'release-5?' - 'ci/**' env: @@ -21,7 +19,7 @@ env: jobs: prepare: - runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} + runs-on: ${{ fromJSON(github.repository_owner == 'emqx' && '["self-hosted", "ephemeral"]' || '["ubuntu-22.04"]') }} container: 'ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04' outputs: profile: ${{ steps.parse-git-ref.outputs.profile }} @@ -31,7 +29,7 @@ jobs: ct-matrix: ${{ steps.matrix.outputs.ct-matrix }} ct-host: ${{ steps.matrix.outputs.ct-host }} ct-docker: ${{ steps.matrix.outputs.ct-docker }} - runner: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} + runner_labels: ${{ github.repository_owner == 'emqx' && '["self-hosted", "ephemeral"]' || '["ubuntu-22.04"]' }} builder: 'ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04' builder_vsn: '5.1-4' otp_vsn: '25.3.2-2' @@ -95,7 +93,6 @@ jobs: otp_vsn: ${{ needs.prepare.outputs.otp_vsn }} elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }} builder_vsn: ${{ needs.prepare.outputs.builder_vsn }} - runner: ${{ needs.prepare.outputs.runner }} secrets: inherit build_and_push_docker_images: @@ -111,7 +108,7 @@ jobs: otp_vsn: ${{ needs.prepare.outputs.otp_vsn }} elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }} builder_vsn: ${{ needs.prepare.outputs.builder_vsn }} - runner: ${{ needs.prepare.outputs.runner }} + runner_labels: ${{ needs.prepare.outputs.runner_labels }} secrets: inherit build_slim_packages: @@ -120,7 +117,7 @@ jobs: - prepare uses: ./.github/workflows/build_slim_packages.yaml with: - runner: ${{ needs.prepare.outputs.runner }} + runner_labels: ${{ needs.prepare.outputs.runner_labels }} builder: ${{ needs.prepare.outputs.builder }} builder_vsn: ${{ needs.prepare.outputs.builder_vsn }} otp_vsn: ${{ needs.prepare.outputs.otp_vsn }} @@ -128,7 +125,7 @@ jobs: compile: if: needs.prepare.outputs.release != 'true' - runs-on: ${{ needs.prepare.outputs.runner }} + runs-on: ${{ fromJSON(needs.prepare.outputs.runner_labels) }} container: ${{ needs.prepare.outputs.builder }} needs: - prepare @@ -165,7 +162,7 @@ jobs: - compile uses: ./.github/workflows/run_emqx_app_tests.yaml with: - runner: ${{ needs.prepare.outputs.runner }} + runner_labels: ${{ needs.prepare.outputs.runner_labels }} builder: ${{ needs.prepare.outputs.builder }} before_ref: ${{ github.event.before }} after_ref: ${{ github.sha }} @@ -177,7 +174,7 @@ jobs: - compile uses: ./.github/workflows/run_test_cases.yaml with: - runner: ${{ needs.prepare.outputs.runner }} + runner_labels: ${{ needs.prepare.outputs.runner_labels }} builder: ${{ needs.prepare.outputs.builder }} ct-matrix: ${{ needs.prepare.outputs.ct-matrix }} ct-host: ${{ needs.prepare.outputs.ct-host }} @@ -190,7 +187,7 @@ jobs: - compile uses: ./.github/workflows/run_conf_tests.yaml with: - runner: ${{ needs.prepare.outputs.runner }} + runner_labels: ${{ needs.prepare.outputs.runner_labels }} builder: ${{ needs.prepare.outputs.builder }} static_checks: @@ -200,6 +197,6 @@ jobs: - compile uses: ./.github/workflows/static_checks.yaml with: - runner: ${{ needs.prepare.outputs.runner }} + runner_labels: ${{ needs.prepare.outputs.runner_labels }} builder: ${{ needs.prepare.outputs.builder }} ct-matrix: ${{ needs.prepare.outputs.ct-matrix }} diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 3f568e430..62399575c 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -28,7 +28,7 @@ on: builder_vsn: required: true type: string - runner: + runner_labels: required: true type: string secrets: @@ -70,14 +70,14 @@ on: required: false type: string default: '5.1-4' - runner: + runner_labels: required: false type: string - default: 'ubuntu-22.04' + default: '["self-hosted","ephemeral", "linux"]' jobs: docker: - runs-on: ${{ inputs.runner }} + runs-on: ${{ fromJSON(inputs.runner_labels) }} strategy: fail-fast: false diff --git a/.github/workflows/build_docker_for_test.yaml b/.github/workflows/build_docker_for_test.yaml index 758cdd1cf..a71bb96d9 100644 --- a/.github/workflows/build_docker_for_test.yaml +++ b/.github/workflows/build_docker_for_test.yaml @@ -7,6 +7,9 @@ concurrency: on: workflow_call: inputs: + runner_labels: + required: true + type: string otp_vsn: required: true type: string @@ -22,7 +25,7 @@ on: jobs: docker: - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(inputs.runner_labels) }} env: EMQX_NAME: ${{ matrix.profile }} PKG_VSN: ${{ startsWith(matrix.profile, 'emqx-enterprise') && inputs.version-emqx-enterprise || inputs.version-emqx }} diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 2292eb545..26e5b5cab 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -19,9 +19,6 @@ on: elixir_vsn: required: true type: string - runner: - required: true - type: string builder_vsn: required: true type: string @@ -62,10 +59,6 @@ on: required: false type: string default: '1.14.5' - runner: - required: false - type: string - default: 'ubuntu-22.04' builder_vsn: required: false type: string @@ -158,7 +151,7 @@ jobs: path: _packages/${{ matrix.profile }}/ linux: - runs-on: ${{ matrix.build_machine }} + runs-on: ['self-hosted', 'linux', "${{ matrix.arch }}"] # always run in builder container because the host might have the wrong OTP version etc. # otherwise buildx.sh does not run docker if arch and os matches the target arch and os. container: @@ -172,7 +165,7 @@ jobs: otp: - ${{ inputs.otp_vsn }} arch: - - amd64 + - x64 - arm64 os: - ubuntu22.04 @@ -186,26 +179,17 @@ jobs: - el7 - amzn2 - amzn2023 - build_machine: - - aws-arm64 - - aws-amd64 builder: - ${{ inputs.builder_vsn }} elixir: - ${{ inputs.elixir_vsn }} with_elixir: - 'no' - exclude: - - arch: arm64 - build_machine: aws-amd64 - - arch: amd64 - build_machine: aws-arm64 include: - profile: emqx otp: ${{ inputs.otp_vsn }} - arch: amd64 + arch: x64 os: ubuntu22.04 - build_machine: aws-amd64 builder: ${{ inputs.builder_vsn }} elixir: ${{ inputs.elixir_vsn }} with_elixir: 'yes' diff --git a/.github/workflows/build_packages_cron.yaml b/.github/workflows/build_packages_cron.yaml index d14e41ff6..f59200d0f 100644 --- a/.github/workflows/build_packages_cron.yaml +++ b/.github/workflows/build_packages_cron.yaml @@ -12,7 +12,7 @@ on: jobs: linux: if: github.repository_owner == 'emqx' - runs-on: aws-${{ matrix.arch }} + runs-on: ['self-hosted', 'linux', "${{ matrix.arch }}", 'ephemeral'] container: image: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" @@ -26,7 +26,7 @@ jobs: otp: - 25.3.2-2 arch: - - amd64 + - x64 os: - debian10 - ubuntu22.04 @@ -41,7 +41,6 @@ jobs: shell: bash steps: - - uses: emqx/self-hosted-cleanup-action@v1.0.3 - uses: actions/checkout@v3 with: ref: ${{ matrix.profile[1] }} @@ -105,7 +104,6 @@ jobs: - macos-12-arm64 steps: - - uses: emqx/self-hosted-cleanup-action@v1.0.3 - uses: actions/checkout@v3 with: ref: ${{ matrix.branch }} diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 8e13ec0a1..8a53b402f 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -7,7 +7,7 @@ concurrency: on: workflow_call: inputs: - runner: + runner_labels: required: true type: string builder: @@ -27,10 +27,10 @@ on: inputs: ref: required: false - runner: + runner_labels: required: false type: string - default: 'ubuntu-22.04' + default: '["self-hosted","ephemeral"]' builder: required: false type: string @@ -50,7 +50,7 @@ on: jobs: linux: - runs-on: ${{ inputs.runner }} + runs-on: ${{ fromJSON(inputs.runner_labels) }} env: EMQX_NAME: ${{ matrix.profile[0] }} @@ -64,7 +64,6 @@ jobs: container: "ghcr.io/emqx/emqx-builder/${{ inputs.builder_vsn }}:${{ inputs.elixir_vsn }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}" steps: - - uses: AutoModality/action-clean@v1 - uses: actions/checkout@v3 with: fetch-depth: 0 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 8ece0746b..1b260fae9 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -3,7 +3,7 @@ name: Check integrity of rebar and mix dependencies on: workflow_call: inputs: - runner: + runner_labels: required: true type: string builder: @@ -12,7 +12,7 @@ on: jobs: check_deps_integrity: - runs-on: ${{ inputs.runner }} + runs-on: ${{ fromJSON(inputs.runner_labels) }} container: ${{ inputs.builder }} steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/green_master.yaml b/.github/workflows/green_master.yaml index 1161ca7d4..d7d4517d3 100644 --- a/.github/workflows/green_master.yaml +++ b/.github/workflows/green_master.yaml @@ -10,8 +10,8 @@ on: jobs: rerun-failed-jobs: - runs-on: ubuntu-22.04 if: github.repository_owner == 'emqx' + runs-on: ['self-hosted', 'linux', 'x64', 'ephemeral'] permissions: checks: read actions: write diff --git a/.github/workflows/run_conf_tests.yaml b/.github/workflows/run_conf_tests.yaml index cf696e0c6..a69746d76 100644 --- a/.github/workflows/run_conf_tests.yaml +++ b/.github/workflows/run_conf_tests.yaml @@ -7,7 +7,7 @@ concurrency: on: workflow_call: inputs: - runner: + runner_labels: required: true type: string builder: @@ -16,7 +16,7 @@ on: jobs: run_conf_tests: - runs-on: ${{ inputs.runner }} + runs-on: ${{ fromJSON(inputs.runner_labels) }} container: ${{ inputs.builder }} env: PROFILE: ${{ matrix.profile }} @@ -27,7 +27,6 @@ jobs: - emqx - emqx-enterprise steps: - - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: name: ${{ matrix.profile }} diff --git a/.github/workflows/run_docker_tests.yaml b/.github/workflows/run_docker_tests.yaml index 4188378d0..18dd9ac38 100644 --- a/.github/workflows/run_docker_tests.yaml +++ b/.github/workflows/run_docker_tests.yaml @@ -7,7 +7,7 @@ concurrency: on: workflow_call: inputs: - runner: + runner_labels: required: true type: string version-emqx: @@ -19,7 +19,7 @@ on: jobs: basic-tests: - runs-on: ${{ inputs.runner }} + runs-on: ${{ fromJSON(inputs.runner_labels) }} defaults: run: shell: bash @@ -63,7 +63,7 @@ jobs: docker compose rm -fs paho-mqtt-testing: - runs-on: ${{ inputs.runner }} + runs-on: ${{ fromJSON(inputs.runner_labels) }} defaults: run: shell: bash @@ -83,7 +83,6 @@ jobs: - mnesia - rlog steps: - - uses: AutoModality/action-clean@v1 - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: @@ -110,9 +109,8 @@ jobs: echo "DUMP_CONTAINER_LOGS_END" exit 1 fi - # node_dump requires netstat, which is not available in the container # simple smoke test for node_dump - # - name: test node_dump - # run: | - # docker exec -u root node1.emqx.io apt update && apt install -y net-tools - # docker exec node1.emqx.io node_dump + - name: test node_dump + run: | + docker exec -t -u root node1.emqx.io bash -c 'apt-get -y update && apt-get -y install net-tools' + docker exec node1.emqx.io node_dump diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index e062cef2e..c5554805a 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -10,7 +10,7 @@ concurrency: on: workflow_call: inputs: - runner: + runner_labels: required: true type: string builder: @@ -28,7 +28,7 @@ env: jobs: run_emqx_app_tests: - runs-on: ${{ inputs.runner }} + runs-on: ${{ fromJSON(inputs.runner_labels) }} container: ${{ inputs.builder }} defaults: @@ -61,5 +61,5 @@ jobs: - uses: actions/upload-artifact@v3 if: failure() with: - name: logs-${{ inputs.runner }} + name: logs-emqx-app-tests path: apps/emqx/_build/test/logs diff --git a/.github/workflows/run_helm_tests.yaml b/.github/workflows/run_helm_tests.yaml index ba3ebee5a..9c314afde 100644 --- a/.github/workflows/run_helm_tests.yaml +++ b/.github/workflows/run_helm_tests.yaml @@ -7,6 +7,9 @@ concurrency: on: workflow_call: inputs: + runner_labels: + required: true + type: string version-emqx: required: true type: string @@ -16,7 +19,7 @@ on: jobs: helm_test: - runs-on: ubuntu-22.04 + runs-on: ${{ fromJSON(inputs.runner_labels) }} defaults: run: shell: bash diff --git a/.github/workflows/run_jmeter_tests.yaml b/.github/workflows/run_jmeter_tests.yaml index 9a36d8796..18fc226a6 100644 --- a/.github/workflows/run_jmeter_tests.yaml +++ b/.github/workflows/run_jmeter_tests.yaml @@ -3,13 +3,16 @@ name: JMeter integration tests on: workflow_call: inputs: + runner_labels: + required: true + type: string version-emqx: required: true type: string jobs: jmeter_artifact: - runs-on: ubuntu-22.04 + runs-on: ${{ fromJSON(inputs.runner_labels) }} steps: - name: Cache Jmeter id: cache-jmeter @@ -38,7 +41,7 @@ jobs: path: /tmp/apache-jmeter.tgz advanced_feat: - runs-on: ubuntu-22.04 + runs-on: ${{ fromJSON(inputs.runner_labels) }} strategy: fail-fast: false @@ -89,7 +92,7 @@ jobs: path: ./jmeter_logs pgsql_authn_authz: - runs-on: ubuntu-22.04 + runs-on: ${{ fromJSON(inputs.runner_labels) }} strategy: fail-fast: false @@ -115,6 +118,7 @@ jobs: env: PGSQL_TAG: ${{ matrix.pgsql_tag }} run: | + docker pull postgres:${PGSQL_TAG} docker compose \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ -f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \ @@ -155,7 +159,7 @@ jobs: path: ./jmeter_logs mysql_authn_authz: - runs-on: ubuntu-22.04 + runs-on: ${{ fromJSON(inputs.runner_labels) }} strategy: fail-fast: false @@ -214,7 +218,7 @@ jobs: path: ./jmeter_logs JWT_authn: - runs-on: ubuntu-22.04 + runs-on: ${{ fromJSON(inputs.runner_labels) }} strategy: fail-fast: false @@ -265,7 +269,7 @@ jobs: path: ./jmeter_logs built_in_database_authn_authz: - runs-on: ubuntu-22.04 + runs-on: ${{ fromJSON(inputs.runner_labels) }} strategy: fail-fast: false diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 3958d5b00..a38ccae4f 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -16,7 +16,7 @@ on: jobs: relup_test_plan: - runs-on: ${{ inputs.runner }} + runs-on: ["${{ inputs.runner }}", 'linux', 'x64', 'ephemeral'] container: ${{ inputs.builder }} outputs: CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }} @@ -25,7 +25,6 @@ jobs: run: shell: bash steps: - - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: name: emqx-enterprise @@ -60,7 +59,7 @@ jobs: needs: - relup_test_plan if: needs.relup_test_plan.outputs.OLD_VERSIONS != '[]' - runs-on: ubuntu-22.04 + runs-on: ["${{ inputs.runner }}", 'linux', 'x64', 'ephemeral'] strategy: fail-fast: false matrix: diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 82b2bbeb9..54d250c24 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -7,7 +7,7 @@ concurrency: on: workflow_call: inputs: - runner: + runner_labels: required: true type: string builder: @@ -28,7 +28,7 @@ env: jobs: eunit_and_proper: - runs-on: ${{ inputs.runner }} + runs-on: ${{ fromJSON(inputs.runner_labels) }} name: "eunit_and_proper (${{ matrix.profile }})" strategy: fail-fast: false @@ -41,7 +41,6 @@ jobs: container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" steps: - - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: name: ${{ matrix.profile }} @@ -69,7 +68,7 @@ jobs: path: _build/test/cover ct_docker: - runs-on: ${{ inputs.runner }} + runs-on: ${{ fromJSON(inputs.runner_labels) }} name: "ct_docker (${{ matrix.app }}-${{ matrix.suitegroup }})" strategy: fail-fast: false @@ -81,7 +80,6 @@ jobs: shell: bash steps: - - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: name: ${{ matrix.profile }} @@ -110,14 +108,17 @@ jobs: with: name: coverdata path: _build/test/cover + - name: compress logs + if: failure() + run: tar -czf logs.tar.gz _build/test/logs - uses: actions/upload-artifact@v3 if: failure() with: name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }} - path: _build/test/logs + path: logs.tar.gz ct: - runs-on: ${{ inputs.runner }} + runs-on: ${{ fromJSON(inputs.runner_labels) }} name: "ct (${{ matrix.app }}-${{ matrix.suitegroup }})" strategy: fail-fast: false @@ -130,7 +131,6 @@ jobs: shell: bash steps: - - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: name: ${{ matrix.profile }} @@ -152,18 +152,21 @@ jobs: name: coverdata path: _build/test/cover if-no-files-found: warn # do not fail if no coverdata found + - name: compress logs + if: failure() + run: tar -czf logs.tar.gz _build/test/logs - uses: actions/upload-artifact@v3 if: failure() with: name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }} - path: _build/test/logs + path: logs.tar.gz tests_passed: needs: - eunit_and_proper - ct - ct_docker - runs-on: ${{ inputs.runner }} + runs-on: ${{ fromJSON(inputs.runner_labels) }} strategy: fail-fast: false steps: @@ -174,7 +177,7 @@ jobs: - eunit_and_proper - ct - ct_docker - runs-on: ${{ inputs.runner }} + runs-on: ${{ fromJSON(inputs.runner_labels) }} container: ${{ inputs.builder }} strategy: fail-fast: false @@ -182,7 +185,6 @@ jobs: profile: - emqx-enterprise steps: - - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: name: ${{ matrix.profile }} @@ -215,7 +217,7 @@ jobs: # do this in a separate job upload_coverdata: needs: make_cover - runs-on: ${{ inputs.runner }} + runs-on: ${{ fromJSON(inputs.runner_labels) }} steps: - name: Coveralls Finished env: diff --git a/.github/workflows/spellcheck.yaml b/.github/workflows/spellcheck.yaml index 42a464ee8..08f6c2c38 100644 --- a/.github/workflows/spellcheck.yaml +++ b/.github/workflows/spellcheck.yaml @@ -7,7 +7,7 @@ concurrency: on: workflow_call: inputs: - runner: + runner_labels: required: true type: string @@ -18,7 +18,7 @@ jobs: profile: - emqx - emqx-enterprise - runs-on: ${{ inputs.runner }} + runs-on: ${{ fromJSON(inputs.runner_labels) }} steps: - uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index 6b67c6f3b..eb8038dcc 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -10,8 +10,8 @@ on: jobs: stale: - runs-on: ubuntu-22.04 if: github.repository_owner == 'emqx' + runs-on: ['self-hosted', 'linux', 'x64', 'ephemeral'] permissions: issues: write pull-requests: none diff --git a/.github/workflows/static_checks.yaml b/.github/workflows/static_checks.yaml index 21a753a37..758c3712b 100644 --- a/.github/workflows/static_checks.yaml +++ b/.github/workflows/static_checks.yaml @@ -7,7 +7,7 @@ concurrency: on: workflow_call: inputs: - runner: + runner_labels: required: true type: string builder: @@ -22,7 +22,7 @@ env: jobs: static_checks: - runs-on: ${{ inputs.runner }} + runs-on: ${{ fromJSON(inputs.runner_labels) }} name: "static_checks (${{ matrix.profile }})" strategy: fail-fast: false @@ -30,7 +30,6 @@ jobs: include: ${{ fromJson(inputs.ct-matrix) }} container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" steps: - - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: name: ${{ matrix.profile }} diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl index c518cf85a..a4a85b49b 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl @@ -240,7 +240,8 @@ user_seeds() -> %% Disabled New(<<"mqttuser0006">>, <<"mqttuser0006">>, {error, user_disabled}), %% IsSuperuser - New(<<"mqttuser0007">>, <<"mqttuser0007">>, {ok, #{is_superuser => true}}) + New(<<"mqttuser0007">>, <<"mqttuser0007">>, {ok, #{is_superuser => true}}), + New(<<"mqttuser0008 (test)">>, <<"mqttuser0008 (test)">>, {ok, #{is_superuser => true}}) | Valid ]. diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl index efe433b4c..5773b4f49 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl @@ -166,7 +166,8 @@ t_update(_Config) -> CorrectConfig = raw_ldap_auth_config(), IncorrectConfig = CorrectConfig#{ - <<"base_dn">> => <<"ou=testdevice,dc=emqx,dc=io">> + <<"base_dn">> => <<"ou=testdevice,dc=emqx,dc=io">>, + <<"filter">> => <<"(objectClass=mqttUser)">> }, {ok, _} = emqx:update_config( @@ -207,7 +208,8 @@ raw_ldap_auth_config() -> <<"mechanism">> => <<"password_based">>, <<"backend">> => <<"ldap_bind">>, <<"server">> => ldap_server(), - <<"base_dn">> => <<"uid=${username},ou=testdevice,dc=emqx,dc=io">>, + <<"base_dn">> => <<"ou=testdevice,dc=emqx,dc=io">>, + <<"filter">> => <<"(uid=${username})">>, <<"username">> => <<"cn=root,dc=emqx,dc=io">>, <<"password">> => <<"public">>, <<"pool_size">> => 8, @@ -225,13 +227,22 @@ user_seeds() -> result => Result } end, + + Normal = lists:map( + fun(Idx) -> + erlang:iolist_to_binary(io_lib:format("mqttuser000~b", [Idx])) + end, + lists:seq(1, 5) + ), + + Specials = [<<"mqttuser0008 (test)">>], + Valid = lists:map( - fun(Idx) -> - Username = erlang:iolist_to_binary(io_lib:format("mqttuser000~b", [Idx])), + fun(Username) -> New(Username, Username, {ok, #{is_superuser => false}}) end, - lists:seq(1, 5) + Normal ++ Specials ), [ %% Not exists diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index 3485ac752..362ead229 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_kafka_impl_producer). +-behaviour(emqx_resource). + -include_lib("emqx_resource/include/emqx_resource.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). diff --git a/apps/emqx_ctl/src/emqx_ctl.app.src b/apps/emqx_ctl/src/emqx_ctl.app.src index db7e4e682..8abf67eb9 100644 --- a/apps/emqx_ctl/src/emqx_ctl.app.src +++ b/apps/emqx_ctl/src/emqx_ctl.app.src @@ -1,6 +1,6 @@ {application, emqx_ctl, [ {description, "Backend for emqx_ctl script"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {mod, {emqx_ctl_app, []}}, {applications, [ diff --git a/apps/emqx_ctl/src/emqx_ctl.erl b/apps/emqx_ctl/src/emqx_ctl.erl index ad51de837..ed6aa7bb1 100644 --- a/apps/emqx_ctl/src/emqx_ctl.erl +++ b/apps/emqx_ctl/src/emqx_ctl.erl @@ -331,13 +331,14 @@ safe_to_existing_atom(Str) -> is_initialized() -> ets:info(?CMD_TAB) =/= undefined. -audit_log(Level, From, Log) -> +audit_log(Level, From, Log = #{args := Args}) -> case lookup_command(audit) of {error, _} -> ignore; {ok, {Mod, Fun}} -> try - apply(Mod, Fun, [Level, From, Log]) + Log1 = Log#{args => [unicode:characters_to_binary(A) || A <- Args]}, + apply(Mod, Fun, [Level, From, Log1]) catch _:Reason:Stacktrace -> ?LOG_ERROR(#{ diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index 0c74d77ea..2e3eb1d32 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.28"}, + {vsn, "5.0.29"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [ diff --git a/apps/emqx_dashboard/src/emqx_dashboard_cli.erl b/apps/emqx_dashboard/src/emqx_dashboard_cli.erl index ec197fdb0..4712b151e 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_cli.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_cli.erl @@ -70,7 +70,7 @@ admins(_) -> unload() -> emqx_ctl:unregister_command(admins). -bin(S) -> iolist_to_binary(S). +bin(S) -> unicode:characters_to_binary(S). print_error(Reason) when is_binary(Reason) -> emqx_ctl:print("Error: ~s~n", [Reason]). diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src b/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src index b7055e18c..4f4cf5722 100644 --- a/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src +++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src @@ -1,6 +1,6 @@ {application, emqx_eviction_agent, [ {description, "EMQX Eviction Agent"}, - {vsn, "5.1.3"}, + {vsn, "5.1.4"}, {registered, [ emqx_eviction_agent_sup, emqx_eviction_agent, diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl index d8c1d7645..214aae2a1 100644 --- a/apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl +++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl @@ -23,6 +23,8 @@ roots/0 ]). +-define(TAGS, [<<"Node Eviction">>]). + %% API callbacks -export([ '/node_eviction/status'/2 @@ -44,7 +46,7 @@ schema("/node_eviction/status") -> #{ 'operationId' => '/node_eviction/status', get => #{ - tags => [<<"node_eviction">>], + tags => ?TAGS, summary => <<"Get node eviction status">>, description => ?DESC("node_eviction_status_get"), responses => #{ diff --git a/apps/emqx_ft/src/emqx_ft.app.src b/apps/emqx_ft/src/emqx_ft.app.src index bfeae56cc..2dd33479c 100644 --- a/apps/emqx_ft/src/emqx_ft.app.src +++ b/apps/emqx_ft/src/emqx_ft.app.src @@ -1,6 +1,6 @@ {application, emqx_ft, [ {description, "EMQX file transfer over MQTT"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {registered, []}, {mod, {emqx_ft_app, []}}, {applications, [ diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index be99618ca..180528f1f 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -110,7 +110,7 @@ schema("/file_transfer") -> #{ 'operationId' => '/file_transfer', get => #{ - tags => [<<"file_transfer">>], + tags => ?TAGS, summary => <<"Get current File Transfer configuration">>, description => ?DESC("file_transfer_get_config"), responses => #{ @@ -118,7 +118,7 @@ schema("/file_transfer") -> } }, put => #{ - tags => [<<"file_transfer">>], + tags => ?TAGS, summary => <<"Update File Transfer configuration">>, description => ?DESC("file_transfer_update_config"), 'requestBody' => ?SCHEMA_CONFIG, diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index 82f749967..2db801b9b 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -226,7 +226,9 @@ on_query( #{base_tokens := BaseTks, filter_tokens := FilterTks} = State ) -> Base = emqx_placeholder:proc_tmpl(BaseTks, Data), - FilterBin = emqx_placeholder:proc_tmpl(FilterTks, Data), + FilterBin = emqx_placeholder:proc_tmpl(FilterTks, Data, #{ + return => full_binary, var_trans => fun filter_escape/1 + }), case emqx_ldap_filter_parser:scan_and_parse(FilterBin) of {ok, Filter} -> do_ldap_query( @@ -278,7 +280,7 @@ do_ldap_query( error, LogMeta#{ msg => "ldap_query_found_more_than_one_match", - count => length(Entries) + count => Count } ), {error, {unrecoverable_error, Msg}} @@ -314,3 +316,20 @@ do_prepare_template([{filter, V} | T], State) -> do_prepare_template(T, State#{filter_tokens => emqx_placeholder:preproc_tmpl(V)}); do_prepare_template([], State) -> State. + +filter_escape(Binary) when is_binary(Binary) -> + filter_escape(erlang:binary_to_list(Binary)); +filter_escape([$\\ | T]) -> + [$\\, $\\ | filter_escape(T)]; +filter_escape([Char | T]) -> + case lists:member(Char, filter_control_chars()) of + true -> + [$\\, Char | filter_escape(T)]; + _ -> + [Char | filter_escape(T)] + end; +filter_escape([]) -> + []. + +filter_control_chars() -> + [$(, $), $&, $|, $=, $!, $~, $>, $<, $:, $*, $\t, $\n, $\r]. diff --git a/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl b/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl index a82a3ee3e..7c4d62bc9 100644 --- a/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl +++ b/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl @@ -2,8 +2,9 @@ Definitions. Control = [()&|!=~><:*] White = [\s\t\n\r]+ -NonString = [^()&|!=~><:*\s\t\n\r] -String = {NonString}+ +StringChars = [^()&|!=~><:*\t\n\r] +Escape = \\{Control}|\\{White} +String = ({Escape}|{StringChars})+ Rules. @@ -17,10 +18,10 @@ Rules. >= : {token, {greaterOrEqual, TokenLine}}. <= : {token, {lessOrEqual, TokenLine}}. \* : {token, {asterisk, TokenLine}}. +\:dn : {token, {dn, TokenLine}}. \: : {token, {colon, TokenLine}}. -dn : {token, {dn, TokenLine}}. {White} : skip_token. -{String} : {token, {string, TokenLine, TokenChars}}. +{String} : {token, {string, TokenLine, to_string(TokenChars)}}. %% Leex will hang if a composite operation is missing a character {Control} : {error, lists:flatten(io_lib:format("Unexpected Tokens:~ts", [TokenChars]))}. @@ -29,3 +30,8 @@ Erlang code. %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- +%% eldap does not support neither the '\28value\29' nor '\(value\)' +%% so after the tokenization we should remove all escape character +to_string(TokenChars) -> + String = string:trim(TokenChars), + lists:flatten(string:replace(String, "\\", "", all)). diff --git a/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl b/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl index e1b1ed98e..a400132f8 100644 --- a/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl +++ b/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl @@ -92,7 +92,7 @@ type -> value: {type, '$1'}. dnattrs -> - colon dn: {dnAttributes, true}. + dn: {dnAttributes, true}. matchingrule -> colon value: {matchingRule, '$2'}. diff --git a/apps/emqx_ldap/test/data/emqx.io.ldif b/apps/emqx_ldap/test/data/emqx.io.ldif index 71a1bb3fc..62ed4c3d5 100644 --- a/apps/emqx_ldap/test/data/emqx.io.ldif +++ b/apps/emqx_ldap/test/data/emqx.io.ldif @@ -157,6 +157,15 @@ uid: mqttuser0007 isSuperuser: TRUE userPassword: {SHA}axpQGbl00j3jvOG058y313ocnBk= +objectClass: top +dn:uid=mqttuser0008 (test),ou=testdevice,dc=emqx,dc=io +objectClass: mqttUser +objectClass: mqttDevice +objectClass: mqttSecurity +uid: mqttuser0008 (test) +isSuperuser: TRUE +userPassword: {SHA}FCzJLOp66OwsZ9DQzXSxdTd9c0U= + ## Try to test with base DN 'ou=dashboard,dc=emqx,dc=io' ## with a filter ugroup=group1 ## this should return 2 users in the query and fail the test diff --git a/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl b/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl index 1a7e970a8..eb2b6161c 100644 --- a/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl +++ b/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl @@ -223,6 +223,23 @@ t_error(_Config) -> ?assertMatch({error, _}, scan_and_parse("attr=value")), ?assertMatch({error, _}, scan_and_parse("(a=b)(c=d)")). +t_escape(_Config) -> + ?assertEqual( + 'and'([equalityMatch("a", "(value)")]), + parse("(&(a=\\(value\\)))") + ), + ?assertEqual( + 'or'([equalityMatch("a", "name (1)")]), + parse("(|(a=name \\(1\\)))") + ), + ?assertEqual( + 'or'([equalityMatch("a", "name (1) *")]), + parse("(|(a=name\\ \\(1\\) \\*))") + ). + +t_value_eql_dn(_Config) -> + ?assertEqual('and'([equalityMatch("a", "dn")]), parse("(&(a=dn))")). + % %%------------------------------------------------------------------------------ % %% Helpers % %%------------------------------------------------------------------------------ diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src b/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src index f6b619d39..c66ec9f23 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src @@ -1,6 +1,6 @@ {application, emqx_node_rebalance, [ {description, "EMQX Node Rebalance"}, - {vsn, "5.0.5"}, + {vsn, "5.0.6"}, {registered, [ emqx_node_rebalance_sup, emqx_node_rebalance, diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl index a9dc535ad..44ac0c291 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl @@ -50,6 +50,7 @@ -define(NODE_EVACUATING, 'NODE_EVACUATING'). -define(RPC_ERROR, 'RPC_ERROR'). -define(NOT_FOUND, 'NOT_FOUND'). +-define(TAGS, [<<"Load Rebalance">>]). %%-------------------------------------------------------------------- %% API Spec @@ -78,7 +79,7 @@ schema("/load_rebalance/status") -> #{ 'operationId' => '/load_rebalance/status', get => #{ - tags => [<<"load_rebalance">>], + tags => ?TAGS, summary => <<"Get rebalance status">>, description => ?DESC("load_rebalance_status"), responses => #{ @@ -90,7 +91,7 @@ schema("/load_rebalance/global_status") -> #{ 'operationId' => '/load_rebalance/global_status', get => #{ - tags => [<<"load_rebalance">>], + tags => ?TAGS, summary => <<"Get global rebalance status">>, description => ?DESC("load_rebalance_global_status"), responses => #{ @@ -102,7 +103,7 @@ schema("/load_rebalance/availability_check") -> #{ 'operationId' => '/load_rebalance/availability_check', get => #{ - tags => [<<"load_rebalance">>], + tags => ?TAGS, summary => <<"Node rebalance availability check">>, description => ?DESC("load_rebalance_availability_check"), responses => #{ @@ -115,7 +116,7 @@ schema("/load_rebalance/:node/start") -> #{ 'operationId' => '/load_rebalance/:node/start', post => #{ - tags => [<<"load_rebalance">>], + tags => ?TAGS, summary => <<"Start rebalancing with the node as coordinator">>, description => ?DESC("load_rebalance_start"), parameters => [param_node()], @@ -135,7 +136,7 @@ schema("/load_rebalance/:node/stop") -> #{ 'operationId' => '/load_rebalance/:node/stop', post => #{ - tags => [<<"load_rebalance">>], + tags => ?TAGS, summary => <<"Stop rebalancing coordinated by the node">>, description => ?DESC("load_rebalance_stop"), parameters => [param_node()], @@ -150,7 +151,7 @@ schema("/load_rebalance/:node/evacuation/start") -> #{ 'operationId' => '/load_rebalance/:node/evacuation/start', post => #{ - tags => [<<"load_rebalance">>], + tags => ?TAGS, summary => <<"Start evacuation on a node">>, description => ?DESC("load_rebalance_evacuation_start"), parameters => [param_node()], @@ -170,7 +171,7 @@ schema("/load_rebalance/:node/evacuation/stop") -> #{ 'operationId' => '/load_rebalance/:node/evacuation/stop', post => #{ - tags => [<<"load_rebalance">>], + tags => ?TAGS, summary => <<"Stop evacuation on a node">>, description => ?DESC("load_rebalance_evacuation_stop"), parameters => [param_node()], @@ -186,7 +187,7 @@ schema("/load_rebalance/:node/evacuation/stop") -> %% #{ %% 'operationId' => '/load_rebalance/:node/purge/start', %% post => #{ -%% tags => [<<"load_rebalance">>], +%% tags => ?TAGS, %% summary => <<"Start purge on the whole cluster">>, %% description => ?DESC("cluster_purge_start"), %% parameters => [param_node()], @@ -206,7 +207,7 @@ schema("/load_rebalance/:node/evacuation/stop") -> %% #{ %% 'operationId' => '/load_rebalance/:node/purge/stop', %% post => #{ -%% tags => [<<"load_rebalance">>], +%% tags => ?TAGS, %% summary => <<"Stop purge on the whole cluster">>, %% description => ?DESC("cluster_purge_stop"), %% parameters => [param_node()], diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 532c769e2..8092fadc8 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.23"}, + {vsn, "0.1.24"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index ffb03209b..92701a8c4 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -306,8 +306,7 @@ query(ResId, Request, Opts) -> {simple_async, _} -> %% TODO(5.1.1): pass Resource instead of ResId to simple APIs %% so the buffer worker does not need to lookup the cache again - Opts1 = Opts#{is_buffer_supported => true}, - emqx_resource_buffer_worker:simple_async_query(ResId, Request, Opts1); + emqx_resource_buffer_worker:simple_async_query(ResId, Request, Opts); {simple_sync, _} -> %% TODO(5.1.1): pass Resource instead of ResId to simple APIs %% so the buffer worker does not need to lookup the cache again diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index c75770673..9026fe65c 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -1048,7 +1048,9 @@ call_query(QM, Id, Index, Ref, Query, QueryOpts) -> ?RESOURCE_ERROR(not_found, "resource not found") end. -do_call_query(QM, Id, Index, Ref, Query, #{is_buffer_supported := true} = QueryOpts, Resource) -> +do_call_query(QM, Id, Index, Ref, Query, QueryOpts, #{query_mode := ResQM} = Resource) when + ResQM =:= simple_async; ResQM =:= simple_sync +-> %% The connector supports buffer, send even in disconnected state #{mod := Mod, state := ResSt, callback_mode := CBM} = Resource, CallMode = call_mode(QM, CBM), @@ -1059,7 +1061,7 @@ do_call_query(QM, Id, Index, Ref, Query, QueryOpts, #{status := connected} = Res #{mod := Mod, state := ResSt, callback_mode := CBM} = Resource, CallMode = call_mode(QM, CBM), apply_query_fun(CallMode, Mod, Id, Index, Ref, Query, ResSt, QueryOpts); -do_call_query(_QM, _Id, _Index, _Ref, _Query, _QueryOpts, _Data) -> +do_call_query(_QM, _Id, _Index, _Ref, _Query, _QueryOpts, _Resource) -> ?RESOURCE_ERROR(not_connected, "resource not connected"). -define(APPLY_RESOURCE(NAME, EXPR, REQ), diff --git a/apps/emqx_schema_registry/src/emqx_schema_registry.app.src b/apps/emqx_schema_registry/src/emqx_schema_registry.app.src index 9145f5dc0..e64d104f7 100644 --- a/apps/emqx_schema_registry/src/emqx_schema_registry.app.src +++ b/apps/emqx_schema_registry/src/emqx_schema_registry.app.src @@ -1,6 +1,6 @@ {application, emqx_schema_registry, [ {description, "EMQX Schema Registry"}, - {vsn, "0.1.6"}, + {vsn, "0.1.7"}, {registered, [emqx_schema_registry_sup]}, {mod, {emqx_schema_registry_app, []}}, {included_applications, [ diff --git a/apps/emqx_schema_registry/src/emqx_schema_registry_http_api.erl b/apps/emqx_schema_registry/src/emqx_schema_registry_http_api.erl index 5d170ef70..7c4df8053 100644 --- a/apps/emqx_schema_registry/src/emqx_schema_registry_http_api.erl +++ b/apps/emqx_schema_registry/src/emqx_schema_registry_http_api.erl @@ -22,6 +22,8 @@ '/schema_registry/:name'/2 ]). +-define(TAGS, [<<"Schema Registry">>]). + %%------------------------------------------------------------------------------------------------- %% `minirest' and `minirest_trails' API %%------------------------------------------------------------------------------------------------- @@ -41,7 +43,7 @@ schema("/schema_registry") -> #{ 'operationId' => '/schema_registry', get => #{ - tags => [<<"schema_registry">>], + tags => ?TAGS, summary => <<"List registered schemas">>, description => ?DESC("desc_schema_registry_api_list"), responses => @@ -57,7 +59,7 @@ schema("/schema_registry") -> } }, post => #{ - tags => [<<"schema_registry">>], + tags => ?TAGS, summary => <<"Register a new schema">>, description => ?DESC("desc_schema_registry_api_post"), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( @@ -79,7 +81,7 @@ schema("/schema_registry/:name") -> #{ 'operationId' => '/schema_registry/:name', get => #{ - tags => [<<"schema_registry">>], + tags => ?TAGS, summary => <<"Get registered schema">>, description => ?DESC("desc_schema_registry_api_get"), parameters => [param_path_schema_name()], @@ -94,7 +96,7 @@ schema("/schema_registry/:name") -> } }, put => #{ - tags => [<<"schema_registry">>], + tags => ?TAGS, summary => <<"Update a schema">>, description => ?DESC("desc_schema_registry_api_put"), parameters => [param_path_schema_name()], @@ -113,7 +115,7 @@ schema("/schema_registry/:name") -> } }, delete => #{ - tags => [<<"schema_registry">>], + tags => ?TAGS, summary => <<"Delete registered schema">>, description => ?DESC("desc_schema_registry_api_delete"), parameters => [param_path_schema_name()], diff --git a/changes/ee/fix-11722.en.md b/changes/ee/fix-11722.en.md new file mode 100644 index 000000000..3a8977a7a --- /dev/null +++ b/changes/ee/fix-11722.en.md @@ -0,0 +1 @@ +Fixed an issue where a Kafka Producer bridge with `sync` query mode would not buffer messages when in the `connecting` state. diff --git a/changes/ee/fix-11728.en.md b/changes/ee/fix-11728.en.md new file mode 100644 index 000000000..564606fea --- /dev/null +++ b/changes/ee/fix-11728.en.md @@ -0,0 +1,3 @@ +Improved the LDAP filter string parser: +1. Automatically escape special characters in filter strings. +2. Fixed a bug that the filter value can't be `dn`. diff --git a/deploy/packages/rpm/emqx.spec b/deploy/packages/rpm/emqx.spec index 4839d1a7d..0a6b75ef1 100644 --- a/deploy/packages/rpm/emqx.spec +++ b/deploy/packages/rpm/emqx.spec @@ -93,7 +93,7 @@ exit 0 %defattr(-,root,root) %{_service_dst} %attr(-,%{_user},%{_group}) %{_lib_home}/* -%attr(-,%{_user},%{_group}) %dir %{_var_home} +%attr(750,%{_user},%{_group}) %dir %{_var_home} %attr(-,%{_user},%{_group}) %config(noreplace) %{_var_home}/* %attr(-,%{_user},%{_group}) %dir %{_log_dir} %attr(-,%{_user},%{_group}) %config(noreplace) %{_conf_dir}/* diff --git a/lib-ee/emqx_license/src/emqx_license_cli.erl b/lib-ee/emqx_license/src/emqx_license_cli.erl index 26c3a074e..8ee498b13 100644 --- a/lib-ee/emqx_license/src/emqx_license_cli.erl +++ b/lib-ee/emqx_license/src/emqx_license_cli.erl @@ -41,7 +41,7 @@ license(_) -> emqx_ctl:usage( [ {"license info", "Show license info"}, - {"license update License", "Update license given as a string"} + {"license update ", "Update license given as a string"} ] ).