Merge remote-tracking branch 'origin/master' into 0919-auth-refactor

This commit is contained in:
Zaiming (Stone) Shi 2023-10-09 14:18:10 +02:00
commit 9ab49a7ae3
47 changed files with 216 additions and 150 deletions

View File

@ -6,6 +6,8 @@ services:
build: build:
context: ../.. context: ../..
dockerfile: .ci/docker-compose-file/openldap/Dockerfile dockerfile: .ci/docker-compose-file/openldap/Dockerfile
ulimits:
nofile: 1024
image: openldap image: openldap
#ports: #ports:
# - 389:389 # - 389:389

View File

@ -16,7 +16,7 @@ env:
jobs: jobs:
sanity-checks: sanity-checks:
runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} runs-on: ${{ fromJSON(github.repository_owner == 'emqx' && '["self-hosted","ephemeral"]' || '["ubuntu-22.04"]') }}
container: "ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04" container: "ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04"
outputs: outputs:
ct-matrix: ${{ steps.matrix.outputs.ct-matrix }} ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
@ -24,7 +24,7 @@ jobs:
ct-docker: ${{ steps.matrix.outputs.ct-docker }} ct-docker: ${{ steps.matrix.outputs.ct-docker }}
version-emqx: ${{ steps.matrix.outputs.version-emqx }} version-emqx: ${{ steps.matrix.outputs.version-emqx }}
version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }} version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }}
runner: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} runner_labels: ${{ github.repository_owner == 'emqx' && '["self-hosted", "ephemeral"]' || '["ubuntu-22.04"]' }}
builder: "ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04" builder: "ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04"
builder_vsn: "5.1-4" builder_vsn: "5.1-4"
otp_vsn: "25.3.2-2" otp_vsn: "25.3.2-2"
@ -115,7 +115,7 @@ jobs:
echo "version-emqx-enterprise=$(./pkg-vsn.sh emqx-enterprise)" | tee -a $GITHUB_OUTPUT echo "version-emqx-enterprise=$(./pkg-vsn.sh emqx-enterprise)" | tee -a $GITHUB_OUTPUT
compile: compile:
runs-on: ${{ needs.sanity-checks.outputs.runner }} runs-on: ${{ fromJSON(needs.sanity-checks.outputs.runner_labels) }}
container: ${{ needs.sanity-checks.outputs.builder }} container: ${{ needs.sanity-checks.outputs.builder }}
needs: needs:
- sanity-checks - sanity-checks
@ -153,7 +153,7 @@ jobs:
- compile - compile
uses: ./.github/workflows/run_emqx_app_tests.yaml uses: ./.github/workflows/run_emqx_app_tests.yaml
with: with:
runner: ${{ needs.sanity-checks.outputs.runner }} runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
builder: ${{ needs.sanity-checks.outputs.builder }} builder: ${{ needs.sanity-checks.outputs.builder }}
before_ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }} before_ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
after_ref: ${{ github.sha }} after_ref: ${{ github.sha }}
@ -164,7 +164,7 @@ jobs:
- compile - compile
uses: ./.github/workflows/run_test_cases.yaml uses: ./.github/workflows/run_test_cases.yaml
with: with:
runner: ${{ needs.sanity-checks.outputs.runner }} runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
builder: ${{ needs.sanity-checks.outputs.builder }} builder: ${{ needs.sanity-checks.outputs.builder }}
ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }} ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }}
ct-host: ${{ needs.sanity-checks.outputs.ct-host }} ct-host: ${{ needs.sanity-checks.outputs.ct-host }}
@ -176,7 +176,7 @@ jobs:
- compile - compile
uses: ./.github/workflows/static_checks.yaml uses: ./.github/workflows/static_checks.yaml
with: with:
runner: ${{ needs.sanity-checks.outputs.runner }} runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
builder: ${{ needs.sanity-checks.outputs.builder }} builder: ${{ needs.sanity-checks.outputs.builder }}
ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }} ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }}
@ -185,7 +185,7 @@ jobs:
- sanity-checks - sanity-checks
uses: ./.github/workflows/build_slim_packages.yaml uses: ./.github/workflows/build_slim_packages.yaml
with: with:
runner: ${{ needs.sanity-checks.outputs.runner }} runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
builder: ${{ needs.sanity-checks.outputs.builder }} builder: ${{ needs.sanity-checks.outputs.builder }}
builder_vsn: ${{ needs.sanity-checks.outputs.builder_vsn }} builder_vsn: ${{ needs.sanity-checks.outputs.builder_vsn }}
otp_vsn: ${{ needs.sanity-checks.outputs.otp_vsn }} otp_vsn: ${{ needs.sanity-checks.outputs.otp_vsn }}
@ -196,6 +196,7 @@ jobs:
- sanity-checks - sanity-checks
uses: ./.github/workflows/build_docker_for_test.yaml uses: ./.github/workflows/build_docker_for_test.yaml
with: with:
runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
otp_vsn: ${{ needs.sanity-checks.outputs.otp_vsn }} otp_vsn: ${{ needs.sanity-checks.outputs.otp_vsn }}
elixir_vsn: ${{ needs.sanity-checks.outputs.elixir_vsn }} elixir_vsn: ${{ needs.sanity-checks.outputs.elixir_vsn }}
version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
@ -207,7 +208,7 @@ jobs:
- build_slim_packages - build_slim_packages
uses: ./.github/workflows/spellcheck.yaml uses: ./.github/workflows/spellcheck.yaml
with: with:
runner: ${{ needs.sanity-checks.outputs.runner }} runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
run_conf_tests: run_conf_tests:
needs: needs:
@ -215,7 +216,7 @@ jobs:
- compile - compile
uses: ./.github/workflows/run_conf_tests.yaml uses: ./.github/workflows/run_conf_tests.yaml
with: with:
runner: ${{ needs.sanity-checks.outputs.runner }} runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
builder: ${{ needs.sanity-checks.outputs.builder }} builder: ${{ needs.sanity-checks.outputs.builder }}
check_deps_integrity: check_deps_integrity:
@ -223,7 +224,7 @@ jobs:
- sanity-checks - sanity-checks
uses: ./.github/workflows/check_deps_integrity.yaml uses: ./.github/workflows/check_deps_integrity.yaml
with: with:
runner: ${{ needs.sanity-checks.outputs.runner }} runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
builder: ${{ needs.sanity-checks.outputs.builder }} builder: ${{ needs.sanity-checks.outputs.builder }}
run_jmeter_tests: run_jmeter_tests:
@ -232,6 +233,7 @@ jobs:
- build_docker_for_test - build_docker_for_test
uses: ./.github/workflows/run_jmeter_tests.yaml uses: ./.github/workflows/run_jmeter_tests.yaml
with: with:
runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
run_docker_tests: run_docker_tests:
@ -240,7 +242,7 @@ jobs:
- build_docker_for_test - build_docker_for_test
uses: ./.github/workflows/run_docker_tests.yaml uses: ./.github/workflows/run_docker_tests.yaml
with: with:
runner: ${{ needs.sanity-checks.outputs.runner }} runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }} version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }}
@ -250,5 +252,6 @@ jobs:
- build_docker_for_test - build_docker_for_test
uses: ./.github/workflows/run_helm_tests.yaml uses: ./.github/workflows/run_helm_tests.yaml
with: with:
runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }} version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }}

View File

@ -11,9 +11,7 @@ on:
- 'e*' - 'e*'
branches: branches:
- 'master' - 'master'
- 'release-51' - 'release-5?'
- 'release-52'
- 'release-53'
- 'ci/**' - 'ci/**'
env: env:
@ -21,7 +19,7 @@ env:
jobs: jobs:
prepare: prepare:
runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} runs-on: ${{ fromJSON(github.repository_owner == 'emqx' && '["self-hosted", "ephemeral"]' || '["ubuntu-22.04"]') }}
container: 'ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04' container: 'ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04'
outputs: outputs:
profile: ${{ steps.parse-git-ref.outputs.profile }} profile: ${{ steps.parse-git-ref.outputs.profile }}
@ -31,7 +29,7 @@ jobs:
ct-matrix: ${{ steps.matrix.outputs.ct-matrix }} ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
ct-host: ${{ steps.matrix.outputs.ct-host }} ct-host: ${{ steps.matrix.outputs.ct-host }}
ct-docker: ${{ steps.matrix.outputs.ct-docker }} ct-docker: ${{ steps.matrix.outputs.ct-docker }}
runner: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} runner_labels: ${{ github.repository_owner == 'emqx' && '["self-hosted", "ephemeral"]' || '["ubuntu-22.04"]' }}
builder: 'ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04' builder: 'ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04'
builder_vsn: '5.1-4' builder_vsn: '5.1-4'
otp_vsn: '25.3.2-2' otp_vsn: '25.3.2-2'
@ -95,7 +93,6 @@ jobs:
otp_vsn: ${{ needs.prepare.outputs.otp_vsn }} otp_vsn: ${{ needs.prepare.outputs.otp_vsn }}
elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }} elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }}
builder_vsn: ${{ needs.prepare.outputs.builder_vsn }} builder_vsn: ${{ needs.prepare.outputs.builder_vsn }}
runner: ${{ needs.prepare.outputs.runner }}
secrets: inherit secrets: inherit
build_and_push_docker_images: build_and_push_docker_images:
@ -111,7 +108,7 @@ jobs:
otp_vsn: ${{ needs.prepare.outputs.otp_vsn }} otp_vsn: ${{ needs.prepare.outputs.otp_vsn }}
elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }} elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }}
builder_vsn: ${{ needs.prepare.outputs.builder_vsn }} builder_vsn: ${{ needs.prepare.outputs.builder_vsn }}
runner: ${{ needs.prepare.outputs.runner }} runner_labels: ${{ needs.prepare.outputs.runner_labels }}
secrets: inherit secrets: inherit
build_slim_packages: build_slim_packages:
@ -120,7 +117,7 @@ jobs:
- prepare - prepare
uses: ./.github/workflows/build_slim_packages.yaml uses: ./.github/workflows/build_slim_packages.yaml
with: with:
runner: ${{ needs.prepare.outputs.runner }} runner_labels: ${{ needs.prepare.outputs.runner_labels }}
builder: ${{ needs.prepare.outputs.builder }} builder: ${{ needs.prepare.outputs.builder }}
builder_vsn: ${{ needs.prepare.outputs.builder_vsn }} builder_vsn: ${{ needs.prepare.outputs.builder_vsn }}
otp_vsn: ${{ needs.prepare.outputs.otp_vsn }} otp_vsn: ${{ needs.prepare.outputs.otp_vsn }}
@ -128,7 +125,7 @@ jobs:
compile: compile:
if: needs.prepare.outputs.release != 'true' if: needs.prepare.outputs.release != 'true'
runs-on: ${{ needs.prepare.outputs.runner }} runs-on: ${{ fromJSON(needs.prepare.outputs.runner_labels) }}
container: ${{ needs.prepare.outputs.builder }} container: ${{ needs.prepare.outputs.builder }}
needs: needs:
- prepare - prepare
@ -165,7 +162,7 @@ jobs:
- compile - compile
uses: ./.github/workflows/run_emqx_app_tests.yaml uses: ./.github/workflows/run_emqx_app_tests.yaml
with: with:
runner: ${{ needs.prepare.outputs.runner }} runner_labels: ${{ needs.prepare.outputs.runner_labels }}
builder: ${{ needs.prepare.outputs.builder }} builder: ${{ needs.prepare.outputs.builder }}
before_ref: ${{ github.event.before }} before_ref: ${{ github.event.before }}
after_ref: ${{ github.sha }} after_ref: ${{ github.sha }}
@ -177,7 +174,7 @@ jobs:
- compile - compile
uses: ./.github/workflows/run_test_cases.yaml uses: ./.github/workflows/run_test_cases.yaml
with: with:
runner: ${{ needs.prepare.outputs.runner }} runner_labels: ${{ needs.prepare.outputs.runner_labels }}
builder: ${{ needs.prepare.outputs.builder }} builder: ${{ needs.prepare.outputs.builder }}
ct-matrix: ${{ needs.prepare.outputs.ct-matrix }} ct-matrix: ${{ needs.prepare.outputs.ct-matrix }}
ct-host: ${{ needs.prepare.outputs.ct-host }} ct-host: ${{ needs.prepare.outputs.ct-host }}
@ -190,7 +187,7 @@ jobs:
- compile - compile
uses: ./.github/workflows/run_conf_tests.yaml uses: ./.github/workflows/run_conf_tests.yaml
with: with:
runner: ${{ needs.prepare.outputs.runner }} runner_labels: ${{ needs.prepare.outputs.runner_labels }}
builder: ${{ needs.prepare.outputs.builder }} builder: ${{ needs.prepare.outputs.builder }}
static_checks: static_checks:
@ -200,6 +197,6 @@ jobs:
- compile - compile
uses: ./.github/workflows/static_checks.yaml uses: ./.github/workflows/static_checks.yaml
with: with:
runner: ${{ needs.prepare.outputs.runner }} runner_labels: ${{ needs.prepare.outputs.runner_labels }}
builder: ${{ needs.prepare.outputs.builder }} builder: ${{ needs.prepare.outputs.builder }}
ct-matrix: ${{ needs.prepare.outputs.ct-matrix }} ct-matrix: ${{ needs.prepare.outputs.ct-matrix }}

View File

@ -28,7 +28,7 @@ on:
builder_vsn: builder_vsn:
required: true required: true
type: string type: string
runner: runner_labels:
required: true required: true
type: string type: string
secrets: secrets:
@ -70,14 +70,14 @@ on:
required: false required: false
type: string type: string
default: '5.1-4' default: '5.1-4'
runner: runner_labels:
required: false required: false
type: string type: string
default: 'ubuntu-22.04' default: '["self-hosted","ephemeral", "linux"]'
jobs: jobs:
docker: docker:
runs-on: ${{ inputs.runner }} runs-on: ${{ fromJSON(inputs.runner_labels) }}
strategy: strategy:
fail-fast: false fail-fast: false

View File

@ -7,6 +7,9 @@ concurrency:
on: on:
workflow_call: workflow_call:
inputs: inputs:
runner_labels:
required: true
type: string
otp_vsn: otp_vsn:
required: true required: true
type: string type: string
@ -22,7 +25,7 @@ on:
jobs: jobs:
docker: docker:
runs-on: ubuntu-latest runs-on: ${{ fromJSON(inputs.runner_labels) }}
env: env:
EMQX_NAME: ${{ matrix.profile }} EMQX_NAME: ${{ matrix.profile }}
PKG_VSN: ${{ startsWith(matrix.profile, 'emqx-enterprise') && inputs.version-emqx-enterprise || inputs.version-emqx }} PKG_VSN: ${{ startsWith(matrix.profile, 'emqx-enterprise') && inputs.version-emqx-enterprise || inputs.version-emqx }}

View File

@ -19,9 +19,6 @@ on:
elixir_vsn: elixir_vsn:
required: true required: true
type: string type: string
runner:
required: true
type: string
builder_vsn: builder_vsn:
required: true required: true
type: string type: string
@ -62,10 +59,6 @@ on:
required: false required: false
type: string type: string
default: '1.14.5' default: '1.14.5'
runner:
required: false
type: string
default: 'ubuntu-22.04'
builder_vsn: builder_vsn:
required: false required: false
type: string type: string
@ -158,7 +151,7 @@ jobs:
path: _packages/${{ matrix.profile }}/ path: _packages/${{ matrix.profile }}/
linux: linux:
runs-on: ${{ matrix.build_machine }} runs-on: ['self-hosted', 'linux', "${{ matrix.arch }}"]
# always run in builder container because the host might have the wrong OTP version etc. # always run in builder container because the host might have the wrong OTP version etc.
# otherwise buildx.sh does not run docker if arch and os matches the target arch and os. # otherwise buildx.sh does not run docker if arch and os matches the target arch and os.
container: container:
@ -172,7 +165,7 @@ jobs:
otp: otp:
- ${{ inputs.otp_vsn }} - ${{ inputs.otp_vsn }}
arch: arch:
- amd64 - x64
- arm64 - arm64
os: os:
- ubuntu22.04 - ubuntu22.04
@ -186,26 +179,17 @@ jobs:
- el7 - el7
- amzn2 - amzn2
- amzn2023 - amzn2023
build_machine:
- aws-arm64
- aws-amd64
builder: builder:
- ${{ inputs.builder_vsn }} - ${{ inputs.builder_vsn }}
elixir: elixir:
- ${{ inputs.elixir_vsn }} - ${{ inputs.elixir_vsn }}
with_elixir: with_elixir:
- 'no' - 'no'
exclude:
- arch: arm64
build_machine: aws-amd64
- arch: amd64
build_machine: aws-arm64
include: include:
- profile: emqx - profile: emqx
otp: ${{ inputs.otp_vsn }} otp: ${{ inputs.otp_vsn }}
arch: amd64 arch: x64
os: ubuntu22.04 os: ubuntu22.04
build_machine: aws-amd64
builder: ${{ inputs.builder_vsn }} builder: ${{ inputs.builder_vsn }}
elixir: ${{ inputs.elixir_vsn }} elixir: ${{ inputs.elixir_vsn }}
with_elixir: 'yes' with_elixir: 'yes'

View File

@ -12,7 +12,7 @@ on:
jobs: jobs:
linux: linux:
if: github.repository_owner == 'emqx' if: github.repository_owner == 'emqx'
runs-on: aws-${{ matrix.arch }} runs-on: ['self-hosted', 'linux', "${{ matrix.arch }}", 'ephemeral']
container: container:
image: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" image: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
@ -26,7 +26,7 @@ jobs:
otp: otp:
- 25.3.2-2 - 25.3.2-2
arch: arch:
- amd64 - x64
os: os:
- debian10 - debian10
- ubuntu22.04 - ubuntu22.04
@ -41,7 +41,6 @@ jobs:
shell: bash shell: bash
steps: steps:
- uses: emqx/self-hosted-cleanup-action@v1.0.3
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:
ref: ${{ matrix.profile[1] }} ref: ${{ matrix.profile[1] }}
@ -105,7 +104,6 @@ jobs:
- macos-12-arm64 - macos-12-arm64
steps: steps:
- uses: emqx/self-hosted-cleanup-action@v1.0.3
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:
ref: ${{ matrix.branch }} ref: ${{ matrix.branch }}

View File

@ -7,7 +7,7 @@ concurrency:
on: on:
workflow_call: workflow_call:
inputs: inputs:
runner: runner_labels:
required: true required: true
type: string type: string
builder: builder:
@ -27,10 +27,10 @@ on:
inputs: inputs:
ref: ref:
required: false required: false
runner: runner_labels:
required: false required: false
type: string type: string
default: 'ubuntu-22.04' default: '["self-hosted","ephemeral"]'
builder: builder:
required: false required: false
type: string type: string
@ -50,7 +50,7 @@ on:
jobs: jobs:
linux: linux:
runs-on: ${{ inputs.runner }} runs-on: ${{ fromJSON(inputs.runner_labels) }}
env: env:
EMQX_NAME: ${{ matrix.profile[0] }} EMQX_NAME: ${{ matrix.profile[0] }}
@ -64,7 +64,6 @@ jobs:
container: "ghcr.io/emqx/emqx-builder/${{ inputs.builder_vsn }}:${{ inputs.elixir_vsn }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}" container: "ghcr.io/emqx/emqx-builder/${{ inputs.builder_vsn }}:${{ inputs.elixir_vsn }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"
steps: steps:
- uses: AutoModality/action-clean@v1
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:
fetch-depth: 0 fetch-depth: 0

View File

@ -3,7 +3,7 @@ name: Check integrity of rebar and mix dependencies
on: on:
workflow_call: workflow_call:
inputs: inputs:
runner: runner_labels:
required: true required: true
type: string type: string
builder: builder:
@ -12,7 +12,7 @@ on:
jobs: jobs:
check_deps_integrity: check_deps_integrity:
runs-on: ${{ inputs.runner }} runs-on: ${{ fromJSON(inputs.runner_labels) }}
container: ${{ inputs.builder }} container: ${{ inputs.builder }}
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3

View File

@ -10,8 +10,8 @@ on:
jobs: jobs:
rerun-failed-jobs: rerun-failed-jobs:
runs-on: ubuntu-22.04
if: github.repository_owner == 'emqx' if: github.repository_owner == 'emqx'
runs-on: ['self-hosted', 'linux', 'x64', 'ephemeral']
permissions: permissions:
checks: read checks: read
actions: write actions: write

View File

@ -7,7 +7,7 @@ concurrency:
on: on:
workflow_call: workflow_call:
inputs: inputs:
runner: runner_labels:
required: true required: true
type: string type: string
builder: builder:
@ -16,7 +16,7 @@ on:
jobs: jobs:
run_conf_tests: run_conf_tests:
runs-on: ${{ inputs.runner }} runs-on: ${{ fromJSON(inputs.runner_labels) }}
container: ${{ inputs.builder }} container: ${{ inputs.builder }}
env: env:
PROFILE: ${{ matrix.profile }} PROFILE: ${{ matrix.profile }}
@ -27,7 +27,6 @@ jobs:
- emqx - emqx
- emqx-enterprise - emqx-enterprise
steps: steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: ${{ matrix.profile }} name: ${{ matrix.profile }}

View File

@ -7,7 +7,7 @@ concurrency:
on: on:
workflow_call: workflow_call:
inputs: inputs:
runner: runner_labels:
required: true required: true
type: string type: string
version-emqx: version-emqx:
@ -19,7 +19,7 @@ on:
jobs: jobs:
basic-tests: basic-tests:
runs-on: ${{ inputs.runner }} runs-on: ${{ fromJSON(inputs.runner_labels) }}
defaults: defaults:
run: run:
shell: bash shell: bash
@ -63,7 +63,7 @@ jobs:
docker compose rm -fs docker compose rm -fs
paho-mqtt-testing: paho-mqtt-testing:
runs-on: ${{ inputs.runner }} runs-on: ${{ fromJSON(inputs.runner_labels) }}
defaults: defaults:
run: run:
shell: bash shell: bash
@ -83,7 +83,6 @@ jobs:
- mnesia - mnesia
- rlog - rlog
steps: steps:
- uses: AutoModality/action-clean@v1
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
@ -110,9 +109,8 @@ jobs:
echo "DUMP_CONTAINER_LOGS_END" echo "DUMP_CONTAINER_LOGS_END"
exit 1 exit 1
fi fi
# node_dump requires netstat, which is not available in the container
# simple smoke test for node_dump # simple smoke test for node_dump
# - name: test node_dump - name: test node_dump
# run: | run: |
# docker exec -u root node1.emqx.io apt update && apt install -y net-tools docker exec -t -u root node1.emqx.io bash -c 'apt-get -y update && apt-get -y install net-tools'
# docker exec node1.emqx.io node_dump docker exec node1.emqx.io node_dump

View File

@ -10,7 +10,7 @@ concurrency:
on: on:
workflow_call: workflow_call:
inputs: inputs:
runner: runner_labels:
required: true required: true
type: string type: string
builder: builder:
@ -28,7 +28,7 @@ env:
jobs: jobs:
run_emqx_app_tests: run_emqx_app_tests:
runs-on: ${{ inputs.runner }} runs-on: ${{ fromJSON(inputs.runner_labels) }}
container: ${{ inputs.builder }} container: ${{ inputs.builder }}
defaults: defaults:
@ -61,5 +61,5 @@ jobs:
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
if: failure() if: failure()
with: with:
name: logs-${{ inputs.runner }} name: logs-emqx-app-tests
path: apps/emqx/_build/test/logs path: apps/emqx/_build/test/logs

View File

@ -7,6 +7,9 @@ concurrency:
on: on:
workflow_call: workflow_call:
inputs: inputs:
runner_labels:
required: true
type: string
version-emqx: version-emqx:
required: true required: true
type: string type: string
@ -16,7 +19,7 @@ on:
jobs: jobs:
helm_test: helm_test:
runs-on: ubuntu-22.04 runs-on: ${{ fromJSON(inputs.runner_labels) }}
defaults: defaults:
run: run:
shell: bash shell: bash

View File

@ -3,13 +3,16 @@ name: JMeter integration tests
on: on:
workflow_call: workflow_call:
inputs: inputs:
runner_labels:
required: true
type: string
version-emqx: version-emqx:
required: true required: true
type: string type: string
jobs: jobs:
jmeter_artifact: jmeter_artifact:
runs-on: ubuntu-22.04 runs-on: ${{ fromJSON(inputs.runner_labels) }}
steps: steps:
- name: Cache Jmeter - name: Cache Jmeter
id: cache-jmeter id: cache-jmeter
@ -38,7 +41,7 @@ jobs:
path: /tmp/apache-jmeter.tgz path: /tmp/apache-jmeter.tgz
advanced_feat: advanced_feat:
runs-on: ubuntu-22.04 runs-on: ${{ fromJSON(inputs.runner_labels) }}
strategy: strategy:
fail-fast: false fail-fast: false
@ -89,7 +92,7 @@ jobs:
path: ./jmeter_logs path: ./jmeter_logs
pgsql_authn_authz: pgsql_authn_authz:
runs-on: ubuntu-22.04 runs-on: ${{ fromJSON(inputs.runner_labels) }}
strategy: strategy:
fail-fast: false fail-fast: false
@ -115,6 +118,7 @@ jobs:
env: env:
PGSQL_TAG: ${{ matrix.pgsql_tag }} PGSQL_TAG: ${{ matrix.pgsql_tag }}
run: | run: |
docker pull postgres:${PGSQL_TAG}
docker compose \ docker compose \
-f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
-f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \ -f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \
@ -155,7 +159,7 @@ jobs:
path: ./jmeter_logs path: ./jmeter_logs
mysql_authn_authz: mysql_authn_authz:
runs-on: ubuntu-22.04 runs-on: ${{ fromJSON(inputs.runner_labels) }}
strategy: strategy:
fail-fast: false fail-fast: false
@ -214,7 +218,7 @@ jobs:
path: ./jmeter_logs path: ./jmeter_logs
JWT_authn: JWT_authn:
runs-on: ubuntu-22.04 runs-on: ${{ fromJSON(inputs.runner_labels) }}
strategy: strategy:
fail-fast: false fail-fast: false
@ -265,7 +269,7 @@ jobs:
path: ./jmeter_logs path: ./jmeter_logs
built_in_database_authn_authz: built_in_database_authn_authz:
runs-on: ubuntu-22.04 runs-on: ${{ fromJSON(inputs.runner_labels) }}
strategy: strategy:
fail-fast: false fail-fast: false

View File

@ -16,7 +16,7 @@ on:
jobs: jobs:
relup_test_plan: relup_test_plan:
runs-on: ${{ inputs.runner }} runs-on: ["${{ inputs.runner }}", 'linux', 'x64', 'ephemeral']
container: ${{ inputs.builder }} container: ${{ inputs.builder }}
outputs: outputs:
CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }} CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }}
@ -25,7 +25,6 @@ jobs:
run: run:
shell: bash shell: bash
steps: steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: emqx-enterprise name: emqx-enterprise
@ -60,7 +59,7 @@ jobs:
needs: needs:
- relup_test_plan - relup_test_plan
if: needs.relup_test_plan.outputs.OLD_VERSIONS != '[]' if: needs.relup_test_plan.outputs.OLD_VERSIONS != '[]'
runs-on: ubuntu-22.04 runs-on: ["${{ inputs.runner }}", 'linux', 'x64', 'ephemeral']
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:

View File

@ -7,7 +7,7 @@ concurrency:
on: on:
workflow_call: workflow_call:
inputs: inputs:
runner: runner_labels:
required: true required: true
type: string type: string
builder: builder:
@ -28,7 +28,7 @@ env:
jobs: jobs:
eunit_and_proper: eunit_and_proper:
runs-on: ${{ inputs.runner }} runs-on: ${{ fromJSON(inputs.runner_labels) }}
name: "eunit_and_proper (${{ matrix.profile }})" name: "eunit_and_proper (${{ matrix.profile }})"
strategy: strategy:
fail-fast: false fail-fast: false
@ -41,7 +41,6 @@ jobs:
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
steps: steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: ${{ matrix.profile }} name: ${{ matrix.profile }}
@ -69,7 +68,7 @@ jobs:
path: _build/test/cover path: _build/test/cover
ct_docker: ct_docker:
runs-on: ${{ inputs.runner }} runs-on: ${{ fromJSON(inputs.runner_labels) }}
name: "ct_docker (${{ matrix.app }}-${{ matrix.suitegroup }})" name: "ct_docker (${{ matrix.app }}-${{ matrix.suitegroup }})"
strategy: strategy:
fail-fast: false fail-fast: false
@ -81,7 +80,6 @@ jobs:
shell: bash shell: bash
steps: steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: ${{ matrix.profile }} name: ${{ matrix.profile }}
@ -110,14 +108,17 @@ jobs:
with: with:
name: coverdata name: coverdata
path: _build/test/cover path: _build/test/cover
- name: compress logs
if: failure()
run: tar -czf logs.tar.gz _build/test/logs
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
if: failure() if: failure()
with: with:
name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }} name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
path: _build/test/logs path: logs.tar.gz
ct: ct:
runs-on: ${{ inputs.runner }} runs-on: ${{ fromJSON(inputs.runner_labels) }}
name: "ct (${{ matrix.app }}-${{ matrix.suitegroup }})" name: "ct (${{ matrix.app }}-${{ matrix.suitegroup }})"
strategy: strategy:
fail-fast: false fail-fast: false
@ -130,7 +131,6 @@ jobs:
shell: bash shell: bash
steps: steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: ${{ matrix.profile }} name: ${{ matrix.profile }}
@ -152,18 +152,21 @@ jobs:
name: coverdata name: coverdata
path: _build/test/cover path: _build/test/cover
if-no-files-found: warn # do not fail if no coverdata found if-no-files-found: warn # do not fail if no coverdata found
- name: compress logs
if: failure()
run: tar -czf logs.tar.gz _build/test/logs
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
if: failure() if: failure()
with: with:
name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }} name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
path: _build/test/logs path: logs.tar.gz
tests_passed: tests_passed:
needs: needs:
- eunit_and_proper - eunit_and_proper
- ct - ct
- ct_docker - ct_docker
runs-on: ${{ inputs.runner }} runs-on: ${{ fromJSON(inputs.runner_labels) }}
strategy: strategy:
fail-fast: false fail-fast: false
steps: steps:
@ -174,7 +177,7 @@ jobs:
- eunit_and_proper - eunit_and_proper
- ct - ct
- ct_docker - ct_docker
runs-on: ${{ inputs.runner }} runs-on: ${{ fromJSON(inputs.runner_labels) }}
container: ${{ inputs.builder }} container: ${{ inputs.builder }}
strategy: strategy:
fail-fast: false fail-fast: false
@ -182,7 +185,6 @@ jobs:
profile: profile:
- emqx-enterprise - emqx-enterprise
steps: steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: ${{ matrix.profile }} name: ${{ matrix.profile }}
@ -215,7 +217,7 @@ jobs:
# do this in a separate job # do this in a separate job
upload_coverdata: upload_coverdata:
needs: make_cover needs: make_cover
runs-on: ${{ inputs.runner }} runs-on: ${{ fromJSON(inputs.runner_labels) }}
steps: steps:
- name: Coveralls Finished - name: Coveralls Finished
env: env:

View File

@ -7,7 +7,7 @@ concurrency:
on: on:
workflow_call: workflow_call:
inputs: inputs:
runner: runner_labels:
required: true required: true
type: string type: string
@ -18,7 +18,7 @@ jobs:
profile: profile:
- emqx - emqx
- emqx-enterprise - emqx-enterprise
runs-on: ${{ inputs.runner }} runs-on: ${{ fromJSON(inputs.runner_labels) }}
steps: steps:
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:

View File

@ -10,8 +10,8 @@ on:
jobs: jobs:
stale: stale:
runs-on: ubuntu-22.04
if: github.repository_owner == 'emqx' if: github.repository_owner == 'emqx'
runs-on: ['self-hosted', 'linux', 'x64', 'ephemeral']
permissions: permissions:
issues: write issues: write
pull-requests: none pull-requests: none

View File

@ -7,7 +7,7 @@ concurrency:
on: on:
workflow_call: workflow_call:
inputs: inputs:
runner: runner_labels:
required: true required: true
type: string type: string
builder: builder:
@ -22,7 +22,7 @@ env:
jobs: jobs:
static_checks: static_checks:
runs-on: ${{ inputs.runner }} runs-on: ${{ fromJSON(inputs.runner_labels) }}
name: "static_checks (${{ matrix.profile }})" name: "static_checks (${{ matrix.profile }})"
strategy: strategy:
fail-fast: false fail-fast: false
@ -30,7 +30,6 @@ jobs:
include: ${{ fromJson(inputs.ct-matrix) }} include: ${{ fromJson(inputs.ct-matrix) }}
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
steps: steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: ${{ matrix.profile }} name: ${{ matrix.profile }}

View File

@ -240,7 +240,8 @@ user_seeds() ->
%% Disabled %% Disabled
New(<<"mqttuser0006">>, <<"mqttuser0006">>, {error, user_disabled}), New(<<"mqttuser0006">>, <<"mqttuser0006">>, {error, user_disabled}),
%% IsSuperuser %% IsSuperuser
New(<<"mqttuser0007">>, <<"mqttuser0007">>, {ok, #{is_superuser => true}}) New(<<"mqttuser0007">>, <<"mqttuser0007">>, {ok, #{is_superuser => true}}),
New(<<"mqttuser0008 (test)">>, <<"mqttuser0008 (test)">>, {ok, #{is_superuser => true}})
| Valid | Valid
]. ].

View File

@ -166,7 +166,8 @@ t_update(_Config) ->
CorrectConfig = raw_ldap_auth_config(), CorrectConfig = raw_ldap_auth_config(),
IncorrectConfig = IncorrectConfig =
CorrectConfig#{ CorrectConfig#{
<<"base_dn">> => <<"ou=testdevice,dc=emqx,dc=io">> <<"base_dn">> => <<"ou=testdevice,dc=emqx,dc=io">>,
<<"filter">> => <<"(objectClass=mqttUser)">>
}, },
{ok, _} = emqx:update_config( {ok, _} = emqx:update_config(
@ -207,7 +208,8 @@ raw_ldap_auth_config() ->
<<"mechanism">> => <<"password_based">>, <<"mechanism">> => <<"password_based">>,
<<"backend">> => <<"ldap_bind">>, <<"backend">> => <<"ldap_bind">>,
<<"server">> => ldap_server(), <<"server">> => ldap_server(),
<<"base_dn">> => <<"uid=${username},ou=testdevice,dc=emqx,dc=io">>, <<"base_dn">> => <<"ou=testdevice,dc=emqx,dc=io">>,
<<"filter">> => <<"(uid=${username})">>,
<<"username">> => <<"cn=root,dc=emqx,dc=io">>, <<"username">> => <<"cn=root,dc=emqx,dc=io">>,
<<"password">> => <<"public">>, <<"password">> => <<"public">>,
<<"pool_size">> => 8, <<"pool_size">> => 8,
@ -225,13 +227,22 @@ user_seeds() ->
result => Result result => Result
} }
end, end,
Normal = lists:map(
fun(Idx) ->
erlang:iolist_to_binary(io_lib:format("mqttuser000~b", [Idx]))
end,
lists:seq(1, 5)
),
Specials = [<<"mqttuser0008 (test)">>],
Valid = Valid =
lists:map( lists:map(
fun(Idx) -> fun(Username) ->
Username = erlang:iolist_to_binary(io_lib:format("mqttuser000~b", [Idx])),
New(Username, Username, {ok, #{is_superuser => false}}) New(Username, Username, {ok, #{is_superuser => false}})
end, end,
lists:seq(1, 5) Normal ++ Specials
), ),
[ [
%% Not exists %% Not exists

View File

@ -3,6 +3,8 @@
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
-module(emqx_bridge_kafka_impl_producer). -module(emqx_bridge_kafka_impl_producer).
-behaviour(emqx_resource).
-include_lib("emqx_resource/include/emqx_resource.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl").
-include_lib("snabbkaffe/include/trace.hrl"). -include_lib("snabbkaffe/include/trace.hrl").

View File

@ -1,6 +1,6 @@
{application, emqx_ctl, [ {application, emqx_ctl, [
{description, "Backend for emqx_ctl script"}, {description, "Backend for emqx_ctl script"},
{vsn, "0.1.4"}, {vsn, "0.1.5"},
{registered, []}, {registered, []},
{mod, {emqx_ctl_app, []}}, {mod, {emqx_ctl_app, []}},
{applications, [ {applications, [

View File

@ -331,13 +331,14 @@ safe_to_existing_atom(Str) ->
is_initialized() -> is_initialized() ->
ets:info(?CMD_TAB) =/= undefined. ets:info(?CMD_TAB) =/= undefined.
audit_log(Level, From, Log) -> audit_log(Level, From, Log = #{args := Args}) ->
case lookup_command(audit) of case lookup_command(audit) of
{error, _} -> {error, _} ->
ignore; ignore;
{ok, {Mod, Fun}} -> {ok, {Mod, Fun}} ->
try try
apply(Mod, Fun, [Level, From, Log]) Log1 = Log#{args => [unicode:characters_to_binary(A) || A <- Args]},
apply(Mod, Fun, [Level, From, Log1])
catch catch
_:Reason:Stacktrace -> _:Reason:Stacktrace ->
?LOG_ERROR(#{ ?LOG_ERROR(#{

View File

@ -2,7 +2,7 @@
{application, emqx_dashboard, [ {application, emqx_dashboard, [
{description, "EMQX Web Dashboard"}, {description, "EMQX Web Dashboard"},
% strict semver, bump manually! % strict semver, bump manually!
{vsn, "5.0.28"}, {vsn, "5.0.29"},
{modules, []}, {modules, []},
{registered, [emqx_dashboard_sup]}, {registered, [emqx_dashboard_sup]},
{applications, [ {applications, [

View File

@ -70,7 +70,7 @@ admins(_) ->
unload() -> unload() ->
emqx_ctl:unregister_command(admins). emqx_ctl:unregister_command(admins).
bin(S) -> iolist_to_binary(S). bin(S) -> unicode:characters_to_binary(S).
print_error(Reason) when is_binary(Reason) -> print_error(Reason) when is_binary(Reason) ->
emqx_ctl:print("Error: ~s~n", [Reason]). emqx_ctl:print("Error: ~s~n", [Reason]).

View File

@ -1,6 +1,6 @@
{application, emqx_eviction_agent, [ {application, emqx_eviction_agent, [
{description, "EMQX Eviction Agent"}, {description, "EMQX Eviction Agent"},
{vsn, "5.1.3"}, {vsn, "5.1.4"},
{registered, [ {registered, [
emqx_eviction_agent_sup, emqx_eviction_agent_sup,
emqx_eviction_agent, emqx_eviction_agent,

View File

@ -23,6 +23,8 @@
roots/0 roots/0
]). ]).
-define(TAGS, [<<"Node Eviction">>]).
%% API callbacks %% API callbacks
-export([ -export([
'/node_eviction/status'/2 '/node_eviction/status'/2
@ -44,7 +46,7 @@ schema("/node_eviction/status") ->
#{ #{
'operationId' => '/node_eviction/status', 'operationId' => '/node_eviction/status',
get => #{ get => #{
tags => [<<"node_eviction">>], tags => ?TAGS,
summary => <<"Get node eviction status">>, summary => <<"Get node eviction status">>,
description => ?DESC("node_eviction_status_get"), description => ?DESC("node_eviction_status_get"),
responses => #{ responses => #{

View File

@ -1,6 +1,6 @@
{application, emqx_ft, [ {application, emqx_ft, [
{description, "EMQX file transfer over MQTT"}, {description, "EMQX file transfer over MQTT"},
{vsn, "0.1.7"}, {vsn, "0.1.8"},
{registered, []}, {registered, []},
{mod, {emqx_ft_app, []}}, {mod, {emqx_ft_app, []}},
{applications, [ {applications, [

View File

@ -110,7 +110,7 @@ schema("/file_transfer") ->
#{ #{
'operationId' => '/file_transfer', 'operationId' => '/file_transfer',
get => #{ get => #{
tags => [<<"file_transfer">>], tags => ?TAGS,
summary => <<"Get current File Transfer configuration">>, summary => <<"Get current File Transfer configuration">>,
description => ?DESC("file_transfer_get_config"), description => ?DESC("file_transfer_get_config"),
responses => #{ responses => #{
@ -118,7 +118,7 @@ schema("/file_transfer") ->
} }
}, },
put => #{ put => #{
tags => [<<"file_transfer">>], tags => ?TAGS,
summary => <<"Update File Transfer configuration">>, summary => <<"Update File Transfer configuration">>,
description => ?DESC("file_transfer_update_config"), description => ?DESC("file_transfer_update_config"),
'requestBody' => ?SCHEMA_CONFIG, 'requestBody' => ?SCHEMA_CONFIG,

View File

@ -226,7 +226,9 @@ on_query(
#{base_tokens := BaseTks, filter_tokens := FilterTks} = State #{base_tokens := BaseTks, filter_tokens := FilterTks} = State
) -> ) ->
Base = emqx_placeholder:proc_tmpl(BaseTks, Data), Base = emqx_placeholder:proc_tmpl(BaseTks, Data),
FilterBin = emqx_placeholder:proc_tmpl(FilterTks, Data), FilterBin = emqx_placeholder:proc_tmpl(FilterTks, Data, #{
return => full_binary, var_trans => fun filter_escape/1
}),
case emqx_ldap_filter_parser:scan_and_parse(FilterBin) of case emqx_ldap_filter_parser:scan_and_parse(FilterBin) of
{ok, Filter} -> {ok, Filter} ->
do_ldap_query( do_ldap_query(
@ -278,7 +280,7 @@ do_ldap_query(
error, error,
LogMeta#{ LogMeta#{
msg => "ldap_query_found_more_than_one_match", msg => "ldap_query_found_more_than_one_match",
count => length(Entries) count => Count
} }
), ),
{error, {unrecoverable_error, Msg}} {error, {unrecoverable_error, Msg}}
@ -314,3 +316,20 @@ do_prepare_template([{filter, V} | T], State) ->
do_prepare_template(T, State#{filter_tokens => emqx_placeholder:preproc_tmpl(V)}); do_prepare_template(T, State#{filter_tokens => emqx_placeholder:preproc_tmpl(V)});
do_prepare_template([], State) -> do_prepare_template([], State) ->
State. State.
filter_escape(Binary) when is_binary(Binary) ->
filter_escape(erlang:binary_to_list(Binary));
filter_escape([$\\ | T]) ->
[$\\, $\\ | filter_escape(T)];
filter_escape([Char | T]) ->
case lists:member(Char, filter_control_chars()) of
true ->
[$\\, Char | filter_escape(T)];
_ ->
[Char | filter_escape(T)]
end;
filter_escape([]) ->
[].
filter_control_chars() ->
[$(, $), $&, $|, $=, $!, $~, $>, $<, $:, $*, $\t, $\n, $\r].

View File

@ -2,8 +2,9 @@ Definitions.
Control = [()&|!=~><:*] Control = [()&|!=~><:*]
White = [\s\t\n\r]+ White = [\s\t\n\r]+
NonString = [^()&|!=~><:*\s\t\n\r] StringChars = [^()&|!=~><:*\t\n\r]
String = {NonString}+ Escape = \\{Control}|\\{White}
String = ({Escape}|{StringChars})+
Rules. Rules.
@ -17,10 +18,10 @@ Rules.
>= : {token, {greaterOrEqual, TokenLine}}. >= : {token, {greaterOrEqual, TokenLine}}.
<= : {token, {lessOrEqual, TokenLine}}. <= : {token, {lessOrEqual, TokenLine}}.
\* : {token, {asterisk, TokenLine}}. \* : {token, {asterisk, TokenLine}}.
\:dn : {token, {dn, TokenLine}}.
\: : {token, {colon, TokenLine}}. \: : {token, {colon, TokenLine}}.
dn : {token, {dn, TokenLine}}.
{White} : skip_token. {White} : skip_token.
{String} : {token, {string, TokenLine, TokenChars}}. {String} : {token, {string, TokenLine, to_string(TokenChars)}}.
%% Leex will hang if a composite operation is missing a character %% Leex will hang if a composite operation is missing a character
{Control} : {error, lists:flatten(io_lib:format("Unexpected Tokens:~ts", [TokenChars]))}. {Control} : {error, lists:flatten(io_lib:format("Unexpected Tokens:~ts", [TokenChars]))}.
@ -29,3 +30,8 @@ Erlang code.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% eldap does not support neither the '\28value\29' nor '\(value\)'
%% so after the tokenization we should remove all escape character
to_string(TokenChars) ->
String = string:trim(TokenChars),
lists:flatten(string:replace(String, "\\", "", all)).

View File

@ -92,7 +92,7 @@ type ->
value: {type, '$1'}. value: {type, '$1'}.
dnattrs -> dnattrs ->
colon dn: {dnAttributes, true}. dn: {dnAttributes, true}.
matchingrule -> matchingrule ->
colon value: {matchingRule, '$2'}. colon value: {matchingRule, '$2'}.

View File

@ -157,6 +157,15 @@ uid: mqttuser0007
isSuperuser: TRUE isSuperuser: TRUE
userPassword: {SHA}axpQGbl00j3jvOG058y313ocnBk= userPassword: {SHA}axpQGbl00j3jvOG058y313ocnBk=
objectClass: top
dn:uid=mqttuser0008 (test),ou=testdevice,dc=emqx,dc=io
objectClass: mqttUser
objectClass: mqttDevice
objectClass: mqttSecurity
uid: mqttuser0008 (test)
isSuperuser: TRUE
userPassword: {SHA}FCzJLOp66OwsZ9DQzXSxdTd9c0U=
## Try to test with base DN 'ou=dashboard,dc=emqx,dc=io' ## Try to test with base DN 'ou=dashboard,dc=emqx,dc=io'
## with a filter ugroup=group1 ## with a filter ugroup=group1
## this should return 2 users in the query and fail the test ## this should return 2 users in the query and fail the test

View File

@ -223,6 +223,23 @@ t_error(_Config) ->
?assertMatch({error, _}, scan_and_parse("attr=value")), ?assertMatch({error, _}, scan_and_parse("attr=value")),
?assertMatch({error, _}, scan_and_parse("(a=b)(c=d)")). ?assertMatch({error, _}, scan_and_parse("(a=b)(c=d)")).
t_escape(_Config) ->
?assertEqual(
'and'([equalityMatch("a", "(value)")]),
parse("(&(a=\\(value\\)))")
),
?assertEqual(
'or'([equalityMatch("a", "name (1)")]),
parse("(|(a=name \\(1\\)))")
),
?assertEqual(
'or'([equalityMatch("a", "name (1) *")]),
parse("(|(a=name\\ \\(1\\) \\*))")
).
t_value_eql_dn(_Config) ->
?assertEqual('and'([equalityMatch("a", "dn")]), parse("(&(a=dn))")).
% %%------------------------------------------------------------------------------ % %%------------------------------------------------------------------------------
% %% Helpers % %% Helpers
% %%------------------------------------------------------------------------------ % %%------------------------------------------------------------------------------

View File

@ -1,6 +1,6 @@
{application, emqx_node_rebalance, [ {application, emqx_node_rebalance, [
{description, "EMQX Node Rebalance"}, {description, "EMQX Node Rebalance"},
{vsn, "5.0.5"}, {vsn, "5.0.6"},
{registered, [ {registered, [
emqx_node_rebalance_sup, emqx_node_rebalance_sup,
emqx_node_rebalance, emqx_node_rebalance,

View File

@ -50,6 +50,7 @@
-define(NODE_EVACUATING, 'NODE_EVACUATING'). -define(NODE_EVACUATING, 'NODE_EVACUATING').
-define(RPC_ERROR, 'RPC_ERROR'). -define(RPC_ERROR, 'RPC_ERROR').
-define(NOT_FOUND, 'NOT_FOUND'). -define(NOT_FOUND, 'NOT_FOUND').
-define(TAGS, [<<"Load Rebalance">>]).
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% API Spec %% API Spec
@ -78,7 +79,7 @@ schema("/load_rebalance/status") ->
#{ #{
'operationId' => '/load_rebalance/status', 'operationId' => '/load_rebalance/status',
get => #{ get => #{
tags => [<<"load_rebalance">>], tags => ?TAGS,
summary => <<"Get rebalance status">>, summary => <<"Get rebalance status">>,
description => ?DESC("load_rebalance_status"), description => ?DESC("load_rebalance_status"),
responses => #{ responses => #{
@ -90,7 +91,7 @@ schema("/load_rebalance/global_status") ->
#{ #{
'operationId' => '/load_rebalance/global_status', 'operationId' => '/load_rebalance/global_status',
get => #{ get => #{
tags => [<<"load_rebalance">>], tags => ?TAGS,
summary => <<"Get global rebalance status">>, summary => <<"Get global rebalance status">>,
description => ?DESC("load_rebalance_global_status"), description => ?DESC("load_rebalance_global_status"),
responses => #{ responses => #{
@ -102,7 +103,7 @@ schema("/load_rebalance/availability_check") ->
#{ #{
'operationId' => '/load_rebalance/availability_check', 'operationId' => '/load_rebalance/availability_check',
get => #{ get => #{
tags => [<<"load_rebalance">>], tags => ?TAGS,
summary => <<"Node rebalance availability check">>, summary => <<"Node rebalance availability check">>,
description => ?DESC("load_rebalance_availability_check"), description => ?DESC("load_rebalance_availability_check"),
responses => #{ responses => #{
@ -115,7 +116,7 @@ schema("/load_rebalance/:node/start") ->
#{ #{
'operationId' => '/load_rebalance/:node/start', 'operationId' => '/load_rebalance/:node/start',
post => #{ post => #{
tags => [<<"load_rebalance">>], tags => ?TAGS,
summary => <<"Start rebalancing with the node as coordinator">>, summary => <<"Start rebalancing with the node as coordinator">>,
description => ?DESC("load_rebalance_start"), description => ?DESC("load_rebalance_start"),
parameters => [param_node()], parameters => [param_node()],
@ -135,7 +136,7 @@ schema("/load_rebalance/:node/stop") ->
#{ #{
'operationId' => '/load_rebalance/:node/stop', 'operationId' => '/load_rebalance/:node/stop',
post => #{ post => #{
tags => [<<"load_rebalance">>], tags => ?TAGS,
summary => <<"Stop rebalancing coordinated by the node">>, summary => <<"Stop rebalancing coordinated by the node">>,
description => ?DESC("load_rebalance_stop"), description => ?DESC("load_rebalance_stop"),
parameters => [param_node()], parameters => [param_node()],
@ -150,7 +151,7 @@ schema("/load_rebalance/:node/evacuation/start") ->
#{ #{
'operationId' => '/load_rebalance/:node/evacuation/start', 'operationId' => '/load_rebalance/:node/evacuation/start',
post => #{ post => #{
tags => [<<"load_rebalance">>], tags => ?TAGS,
summary => <<"Start evacuation on a node">>, summary => <<"Start evacuation on a node">>,
description => ?DESC("load_rebalance_evacuation_start"), description => ?DESC("load_rebalance_evacuation_start"),
parameters => [param_node()], parameters => [param_node()],
@ -170,7 +171,7 @@ schema("/load_rebalance/:node/evacuation/stop") ->
#{ #{
'operationId' => '/load_rebalance/:node/evacuation/stop', 'operationId' => '/load_rebalance/:node/evacuation/stop',
post => #{ post => #{
tags => [<<"load_rebalance">>], tags => ?TAGS,
summary => <<"Stop evacuation on a node">>, summary => <<"Stop evacuation on a node">>,
description => ?DESC("load_rebalance_evacuation_stop"), description => ?DESC("load_rebalance_evacuation_stop"),
parameters => [param_node()], parameters => [param_node()],
@ -186,7 +187,7 @@ schema("/load_rebalance/:node/evacuation/stop") ->
%% #{ %% #{
%% 'operationId' => '/load_rebalance/:node/purge/start', %% 'operationId' => '/load_rebalance/:node/purge/start',
%% post => #{ %% post => #{
%% tags => [<<"load_rebalance">>], %% tags => ?TAGS,
%% summary => <<"Start purge on the whole cluster">>, %% summary => <<"Start purge on the whole cluster">>,
%% description => ?DESC("cluster_purge_start"), %% description => ?DESC("cluster_purge_start"),
%% parameters => [param_node()], %% parameters => [param_node()],
@ -206,7 +207,7 @@ schema("/load_rebalance/:node/evacuation/stop") ->
%% #{ %% #{
%% 'operationId' => '/load_rebalance/:node/purge/stop', %% 'operationId' => '/load_rebalance/:node/purge/stop',
%% post => #{ %% post => #{
%% tags => [<<"load_rebalance">>], %% tags => ?TAGS,
%% summary => <<"Stop purge on the whole cluster">>, %% summary => <<"Stop purge on the whole cluster">>,
%% description => ?DESC("cluster_purge_stop"), %% description => ?DESC("cluster_purge_stop"),
%% parameters => [param_node()], %% parameters => [param_node()],

View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*- %% -*- mode: erlang -*-
{application, emqx_resource, [ {application, emqx_resource, [
{description, "Manager for all external resources"}, {description, "Manager for all external resources"},
{vsn, "0.1.23"}, {vsn, "0.1.24"},
{registered, []}, {registered, []},
{mod, {emqx_resource_app, []}}, {mod, {emqx_resource_app, []}},
{applications, [ {applications, [

View File

@ -306,8 +306,7 @@ query(ResId, Request, Opts) ->
{simple_async, _} -> {simple_async, _} ->
%% TODO(5.1.1): pass Resource instead of ResId to simple APIs %% TODO(5.1.1): pass Resource instead of ResId to simple APIs
%% so the buffer worker does not need to lookup the cache again %% so the buffer worker does not need to lookup the cache again
Opts1 = Opts#{is_buffer_supported => true}, emqx_resource_buffer_worker:simple_async_query(ResId, Request, Opts);
emqx_resource_buffer_worker:simple_async_query(ResId, Request, Opts1);
{simple_sync, _} -> {simple_sync, _} ->
%% TODO(5.1.1): pass Resource instead of ResId to simple APIs %% TODO(5.1.1): pass Resource instead of ResId to simple APIs
%% so the buffer worker does not need to lookup the cache again %% so the buffer worker does not need to lookup the cache again

View File

@ -1048,7 +1048,9 @@ call_query(QM, Id, Index, Ref, Query, QueryOpts) ->
?RESOURCE_ERROR(not_found, "resource not found") ?RESOURCE_ERROR(not_found, "resource not found")
end. end.
do_call_query(QM, Id, Index, Ref, Query, #{is_buffer_supported := true} = QueryOpts, Resource) -> do_call_query(QM, Id, Index, Ref, Query, QueryOpts, #{query_mode := ResQM} = Resource) when
ResQM =:= simple_async; ResQM =:= simple_sync
->
%% The connector supports buffer, send even in disconnected state %% The connector supports buffer, send even in disconnected state
#{mod := Mod, state := ResSt, callback_mode := CBM} = Resource, #{mod := Mod, state := ResSt, callback_mode := CBM} = Resource,
CallMode = call_mode(QM, CBM), CallMode = call_mode(QM, CBM),
@ -1059,7 +1061,7 @@ do_call_query(QM, Id, Index, Ref, Query, QueryOpts, #{status := connected} = Res
#{mod := Mod, state := ResSt, callback_mode := CBM} = Resource, #{mod := Mod, state := ResSt, callback_mode := CBM} = Resource,
CallMode = call_mode(QM, CBM), CallMode = call_mode(QM, CBM),
apply_query_fun(CallMode, Mod, Id, Index, Ref, Query, ResSt, QueryOpts); apply_query_fun(CallMode, Mod, Id, Index, Ref, Query, ResSt, QueryOpts);
do_call_query(_QM, _Id, _Index, _Ref, _Query, _QueryOpts, _Data) -> do_call_query(_QM, _Id, _Index, _Ref, _Query, _QueryOpts, _Resource) ->
?RESOURCE_ERROR(not_connected, "resource not connected"). ?RESOURCE_ERROR(not_connected, "resource not connected").
-define(APPLY_RESOURCE(NAME, EXPR, REQ), -define(APPLY_RESOURCE(NAME, EXPR, REQ),

View File

@ -1,6 +1,6 @@
{application, emqx_schema_registry, [ {application, emqx_schema_registry, [
{description, "EMQX Schema Registry"}, {description, "EMQX Schema Registry"},
{vsn, "0.1.6"}, {vsn, "0.1.7"},
{registered, [emqx_schema_registry_sup]}, {registered, [emqx_schema_registry_sup]},
{mod, {emqx_schema_registry_app, []}}, {mod, {emqx_schema_registry_app, []}},
{included_applications, [ {included_applications, [

View File

@ -22,6 +22,8 @@
'/schema_registry/:name'/2 '/schema_registry/:name'/2
]). ]).
-define(TAGS, [<<"Schema Registry">>]).
%%------------------------------------------------------------------------------------------------- %%-------------------------------------------------------------------------------------------------
%% `minirest' and `minirest_trails' API %% `minirest' and `minirest_trails' API
%%------------------------------------------------------------------------------------------------- %%-------------------------------------------------------------------------------------------------
@ -41,7 +43,7 @@ schema("/schema_registry") ->
#{ #{
'operationId' => '/schema_registry', 'operationId' => '/schema_registry',
get => #{ get => #{
tags => [<<"schema_registry">>], tags => ?TAGS,
summary => <<"List registered schemas">>, summary => <<"List registered schemas">>,
description => ?DESC("desc_schema_registry_api_list"), description => ?DESC("desc_schema_registry_api_list"),
responses => responses =>
@ -57,7 +59,7 @@ schema("/schema_registry") ->
} }
}, },
post => #{ post => #{
tags => [<<"schema_registry">>], tags => ?TAGS,
summary => <<"Register a new schema">>, summary => <<"Register a new schema">>,
description => ?DESC("desc_schema_registry_api_post"), description => ?DESC("desc_schema_registry_api_post"),
'requestBody' => emqx_dashboard_swagger:schema_with_examples( 'requestBody' => emqx_dashboard_swagger:schema_with_examples(
@ -79,7 +81,7 @@ schema("/schema_registry/:name") ->
#{ #{
'operationId' => '/schema_registry/:name', 'operationId' => '/schema_registry/:name',
get => #{ get => #{
tags => [<<"schema_registry">>], tags => ?TAGS,
summary => <<"Get registered schema">>, summary => <<"Get registered schema">>,
description => ?DESC("desc_schema_registry_api_get"), description => ?DESC("desc_schema_registry_api_get"),
parameters => [param_path_schema_name()], parameters => [param_path_schema_name()],
@ -94,7 +96,7 @@ schema("/schema_registry/:name") ->
} }
}, },
put => #{ put => #{
tags => [<<"schema_registry">>], tags => ?TAGS,
summary => <<"Update a schema">>, summary => <<"Update a schema">>,
description => ?DESC("desc_schema_registry_api_put"), description => ?DESC("desc_schema_registry_api_put"),
parameters => [param_path_schema_name()], parameters => [param_path_schema_name()],
@ -113,7 +115,7 @@ schema("/schema_registry/:name") ->
} }
}, },
delete => #{ delete => #{
tags => [<<"schema_registry">>], tags => ?TAGS,
summary => <<"Delete registered schema">>, summary => <<"Delete registered schema">>,
description => ?DESC("desc_schema_registry_api_delete"), description => ?DESC("desc_schema_registry_api_delete"),
parameters => [param_path_schema_name()], parameters => [param_path_schema_name()],

View File

@ -0,0 +1 @@
Fixed an issue where a Kafka Producer bridge with `sync` query mode would not buffer messages when in the `connecting` state.

View File

@ -0,0 +1,3 @@
Improved the LDAP filter string parser:
1. Automatically escape special characters in filter strings.
2. Fixed a bug that the filter value can't be `dn`.

View File

@ -93,7 +93,7 @@ exit 0
%defattr(-,root,root) %defattr(-,root,root)
%{_service_dst} %{_service_dst}
%attr(-,%{_user},%{_group}) %{_lib_home}/* %attr(-,%{_user},%{_group}) %{_lib_home}/*
%attr(-,%{_user},%{_group}) %dir %{_var_home} %attr(750,%{_user},%{_group}) %dir %{_var_home}
%attr(-,%{_user},%{_group}) %config(noreplace) %{_var_home}/* %attr(-,%{_user},%{_group}) %config(noreplace) %{_var_home}/*
%attr(-,%{_user},%{_group}) %dir %{_log_dir} %attr(-,%{_user},%{_group}) %dir %{_log_dir}
%attr(-,%{_user},%{_group}) %config(noreplace) %{_conf_dir}/* %attr(-,%{_user},%{_group}) %config(noreplace) %{_conf_dir}/*

View File

@ -41,7 +41,7 @@ license(_) ->
emqx_ctl:usage( emqx_ctl:usage(
[ [
{"license info", "Show license info"}, {"license info", "Show license info"},
{"license update License", "Update license given as a string"} {"license update <License>", "Update license given as a string"}
] ]
). ).