Merge remote-tracking branch 'origin/master' into 0919-auth-refactor

This commit is contained in:
Zaiming (Stone) Shi 2023-10-09 14:18:10 +02:00
commit 9ab49a7ae3
47 changed files with 216 additions and 150 deletions

View File

@ -6,6 +6,8 @@ services:
build:
context: ../..
dockerfile: .ci/docker-compose-file/openldap/Dockerfile
ulimits:
nofile: 1024
image: openldap
#ports:
# - 389:389

View File

@ -16,7 +16,7 @@ env:
jobs:
sanity-checks:
runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
runs-on: ${{ fromJSON(github.repository_owner == 'emqx' && '["self-hosted","ephemeral"]' || '["ubuntu-22.04"]') }}
container: "ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04"
outputs:
ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
@ -24,7 +24,7 @@ jobs:
ct-docker: ${{ steps.matrix.outputs.ct-docker }}
version-emqx: ${{ steps.matrix.outputs.version-emqx }}
version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }}
runner: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
runner_labels: ${{ github.repository_owner == 'emqx' && '["self-hosted", "ephemeral"]' || '["ubuntu-22.04"]' }}
builder: "ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04"
builder_vsn: "5.1-4"
otp_vsn: "25.3.2-2"
@ -115,7 +115,7 @@ jobs:
echo "version-emqx-enterprise=$(./pkg-vsn.sh emqx-enterprise)" | tee -a $GITHUB_OUTPUT
compile:
runs-on: ${{ needs.sanity-checks.outputs.runner }}
runs-on: ${{ fromJSON(needs.sanity-checks.outputs.runner_labels) }}
container: ${{ needs.sanity-checks.outputs.builder }}
needs:
- sanity-checks
@ -153,7 +153,7 @@ jobs:
- compile
uses: ./.github/workflows/run_emqx_app_tests.yaml
with:
runner: ${{ needs.sanity-checks.outputs.runner }}
runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
builder: ${{ needs.sanity-checks.outputs.builder }}
before_ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
after_ref: ${{ github.sha }}
@ -164,7 +164,7 @@ jobs:
- compile
uses: ./.github/workflows/run_test_cases.yaml
with:
runner: ${{ needs.sanity-checks.outputs.runner }}
runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
builder: ${{ needs.sanity-checks.outputs.builder }}
ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }}
ct-host: ${{ needs.sanity-checks.outputs.ct-host }}
@ -176,7 +176,7 @@ jobs:
- compile
uses: ./.github/workflows/static_checks.yaml
with:
runner: ${{ needs.sanity-checks.outputs.runner }}
runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
builder: ${{ needs.sanity-checks.outputs.builder }}
ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }}
@ -185,7 +185,7 @@ jobs:
- sanity-checks
uses: ./.github/workflows/build_slim_packages.yaml
with:
runner: ${{ needs.sanity-checks.outputs.runner }}
runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
builder: ${{ needs.sanity-checks.outputs.builder }}
builder_vsn: ${{ needs.sanity-checks.outputs.builder_vsn }}
otp_vsn: ${{ needs.sanity-checks.outputs.otp_vsn }}
@ -196,6 +196,7 @@ jobs:
- sanity-checks
uses: ./.github/workflows/build_docker_for_test.yaml
with:
runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
otp_vsn: ${{ needs.sanity-checks.outputs.otp_vsn }}
elixir_vsn: ${{ needs.sanity-checks.outputs.elixir_vsn }}
version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
@ -207,7 +208,7 @@ jobs:
- build_slim_packages
uses: ./.github/workflows/spellcheck.yaml
with:
runner: ${{ needs.sanity-checks.outputs.runner }}
runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
run_conf_tests:
needs:
@ -215,7 +216,7 @@ jobs:
- compile
uses: ./.github/workflows/run_conf_tests.yaml
with:
runner: ${{ needs.sanity-checks.outputs.runner }}
runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
builder: ${{ needs.sanity-checks.outputs.builder }}
check_deps_integrity:
@ -223,7 +224,7 @@ jobs:
- sanity-checks
uses: ./.github/workflows/check_deps_integrity.yaml
with:
runner: ${{ needs.sanity-checks.outputs.runner }}
runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
builder: ${{ needs.sanity-checks.outputs.builder }}
run_jmeter_tests:
@ -232,6 +233,7 @@ jobs:
- build_docker_for_test
uses: ./.github/workflows/run_jmeter_tests.yaml
with:
runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
run_docker_tests:
@ -240,7 +242,7 @@ jobs:
- build_docker_for_test
uses: ./.github/workflows/run_docker_tests.yaml
with:
runner: ${{ needs.sanity-checks.outputs.runner }}
runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }}
@ -250,5 +252,6 @@ jobs:
- build_docker_for_test
uses: ./.github/workflows/run_helm_tests.yaml
with:
runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }}

View File

@ -11,9 +11,7 @@ on:
- 'e*'
branches:
- 'master'
- 'release-51'
- 'release-52'
- 'release-53'
- 'release-5?'
- 'ci/**'
env:
@ -21,7 +19,7 @@ env:
jobs:
prepare:
runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
runs-on: ${{ fromJSON(github.repository_owner == 'emqx' && '["self-hosted", "ephemeral"]' || '["ubuntu-22.04"]') }}
container: 'ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04'
outputs:
profile: ${{ steps.parse-git-ref.outputs.profile }}
@ -31,7 +29,7 @@ jobs:
ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
ct-host: ${{ steps.matrix.outputs.ct-host }}
ct-docker: ${{ steps.matrix.outputs.ct-docker }}
runner: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
runner_labels: ${{ github.repository_owner == 'emqx' && '["self-hosted", "ephemeral"]' || '["ubuntu-22.04"]' }}
builder: 'ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04'
builder_vsn: '5.1-4'
otp_vsn: '25.3.2-2'
@ -95,7 +93,6 @@ jobs:
otp_vsn: ${{ needs.prepare.outputs.otp_vsn }}
elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }}
builder_vsn: ${{ needs.prepare.outputs.builder_vsn }}
runner: ${{ needs.prepare.outputs.runner }}
secrets: inherit
build_and_push_docker_images:
@ -111,7 +108,7 @@ jobs:
otp_vsn: ${{ needs.prepare.outputs.otp_vsn }}
elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }}
builder_vsn: ${{ needs.prepare.outputs.builder_vsn }}
runner: ${{ needs.prepare.outputs.runner }}
runner_labels: ${{ needs.prepare.outputs.runner_labels }}
secrets: inherit
build_slim_packages:
@ -120,7 +117,7 @@ jobs:
- prepare
uses: ./.github/workflows/build_slim_packages.yaml
with:
runner: ${{ needs.prepare.outputs.runner }}
runner_labels: ${{ needs.prepare.outputs.runner_labels }}
builder: ${{ needs.prepare.outputs.builder }}
builder_vsn: ${{ needs.prepare.outputs.builder_vsn }}
otp_vsn: ${{ needs.prepare.outputs.otp_vsn }}
@ -128,7 +125,7 @@ jobs:
compile:
if: needs.prepare.outputs.release != 'true'
runs-on: ${{ needs.prepare.outputs.runner }}
runs-on: ${{ fromJSON(needs.prepare.outputs.runner_labels) }}
container: ${{ needs.prepare.outputs.builder }}
needs:
- prepare
@ -165,7 +162,7 @@ jobs:
- compile
uses: ./.github/workflows/run_emqx_app_tests.yaml
with:
runner: ${{ needs.prepare.outputs.runner }}
runner_labels: ${{ needs.prepare.outputs.runner_labels }}
builder: ${{ needs.prepare.outputs.builder }}
before_ref: ${{ github.event.before }}
after_ref: ${{ github.sha }}
@ -177,7 +174,7 @@ jobs:
- compile
uses: ./.github/workflows/run_test_cases.yaml
with:
runner: ${{ needs.prepare.outputs.runner }}
runner_labels: ${{ needs.prepare.outputs.runner_labels }}
builder: ${{ needs.prepare.outputs.builder }}
ct-matrix: ${{ needs.prepare.outputs.ct-matrix }}
ct-host: ${{ needs.prepare.outputs.ct-host }}
@ -190,7 +187,7 @@ jobs:
- compile
uses: ./.github/workflows/run_conf_tests.yaml
with:
runner: ${{ needs.prepare.outputs.runner }}
runner_labels: ${{ needs.prepare.outputs.runner_labels }}
builder: ${{ needs.prepare.outputs.builder }}
static_checks:
@ -200,6 +197,6 @@ jobs:
- compile
uses: ./.github/workflows/static_checks.yaml
with:
runner: ${{ needs.prepare.outputs.runner }}
runner_labels: ${{ needs.prepare.outputs.runner_labels }}
builder: ${{ needs.prepare.outputs.builder }}
ct-matrix: ${{ needs.prepare.outputs.ct-matrix }}

View File

@ -28,7 +28,7 @@ on:
builder_vsn:
required: true
type: string
runner:
runner_labels:
required: true
type: string
secrets:
@ -70,14 +70,14 @@ on:
required: false
type: string
default: '5.1-4'
runner:
runner_labels:
required: false
type: string
default: 'ubuntu-22.04'
default: '["self-hosted","ephemeral", "linux"]'
jobs:
docker:
runs-on: ${{ inputs.runner }}
runs-on: ${{ fromJSON(inputs.runner_labels) }}
strategy:
fail-fast: false

View File

@ -7,6 +7,9 @@ concurrency:
on:
workflow_call:
inputs:
runner_labels:
required: true
type: string
otp_vsn:
required: true
type: string
@ -22,7 +25,7 @@ on:
jobs:
docker:
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(inputs.runner_labels) }}
env:
EMQX_NAME: ${{ matrix.profile }}
PKG_VSN: ${{ startsWith(matrix.profile, 'emqx-enterprise') && inputs.version-emqx-enterprise || inputs.version-emqx }}

View File

@ -19,9 +19,6 @@ on:
elixir_vsn:
required: true
type: string
runner:
required: true
type: string
builder_vsn:
required: true
type: string
@ -62,10 +59,6 @@ on:
required: false
type: string
default: '1.14.5'
runner:
required: false
type: string
default: 'ubuntu-22.04'
builder_vsn:
required: false
type: string
@ -158,7 +151,7 @@ jobs:
path: _packages/${{ matrix.profile }}/
linux:
runs-on: ${{ matrix.build_machine }}
runs-on: ['self-hosted', 'linux', "${{ matrix.arch }}"]
# always run in builder container because the host might have the wrong OTP version etc.
# otherwise buildx.sh does not run docker if arch and os matches the target arch and os.
container:
@ -172,7 +165,7 @@ jobs:
otp:
- ${{ inputs.otp_vsn }}
arch:
- amd64
- x64
- arm64
os:
- ubuntu22.04
@ -186,26 +179,17 @@ jobs:
- el7
- amzn2
- amzn2023
build_machine:
- aws-arm64
- aws-amd64
builder:
- ${{ inputs.builder_vsn }}
elixir:
- ${{ inputs.elixir_vsn }}
with_elixir:
- 'no'
exclude:
- arch: arm64
build_machine: aws-amd64
- arch: amd64
build_machine: aws-arm64
include:
- profile: emqx
otp: ${{ inputs.otp_vsn }}
arch: amd64
arch: x64
os: ubuntu22.04
build_machine: aws-amd64
builder: ${{ inputs.builder_vsn }}
elixir: ${{ inputs.elixir_vsn }}
with_elixir: 'yes'

View File

@ -12,7 +12,7 @@ on:
jobs:
linux:
if: github.repository_owner == 'emqx'
runs-on: aws-${{ matrix.arch }}
runs-on: ['self-hosted', 'linux', "${{ matrix.arch }}", 'ephemeral']
container:
image: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
@ -26,7 +26,7 @@ jobs:
otp:
- 25.3.2-2
arch:
- amd64
- x64
os:
- debian10
- ubuntu22.04
@ -41,7 +41,6 @@ jobs:
shell: bash
steps:
- uses: emqx/self-hosted-cleanup-action@v1.0.3
- uses: actions/checkout@v3
with:
ref: ${{ matrix.profile[1] }}
@ -105,7 +104,6 @@ jobs:
- macos-12-arm64
steps:
- uses: emqx/self-hosted-cleanup-action@v1.0.3
- uses: actions/checkout@v3
with:
ref: ${{ matrix.branch }}

View File

@ -7,7 +7,7 @@ concurrency:
on:
workflow_call:
inputs:
runner:
runner_labels:
required: true
type: string
builder:
@ -27,10 +27,10 @@ on:
inputs:
ref:
required: false
runner:
runner_labels:
required: false
type: string
default: 'ubuntu-22.04'
default: '["self-hosted","ephemeral"]'
builder:
required: false
type: string
@ -50,7 +50,7 @@ on:
jobs:
linux:
runs-on: ${{ inputs.runner }}
runs-on: ${{ fromJSON(inputs.runner_labels) }}
env:
EMQX_NAME: ${{ matrix.profile[0] }}
@ -64,7 +64,6 @@ jobs:
container: "ghcr.io/emqx/emqx-builder/${{ inputs.builder_vsn }}:${{ inputs.elixir_vsn }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/checkout@v3
with:
fetch-depth: 0

View File

@ -3,7 +3,7 @@ name: Check integrity of rebar and mix dependencies
on:
workflow_call:
inputs:
runner:
runner_labels:
required: true
type: string
builder:
@ -12,7 +12,7 @@ on:
jobs:
check_deps_integrity:
runs-on: ${{ inputs.runner }}
runs-on: ${{ fromJSON(inputs.runner_labels) }}
container: ${{ inputs.builder }}
steps:
- uses: actions/checkout@v3

View File

@ -10,8 +10,8 @@ on:
jobs:
rerun-failed-jobs:
runs-on: ubuntu-22.04
if: github.repository_owner == 'emqx'
runs-on: ['self-hosted', 'linux', 'x64', 'ephemeral']
permissions:
checks: read
actions: write

View File

@ -7,7 +7,7 @@ concurrency:
on:
workflow_call:
inputs:
runner:
runner_labels:
required: true
type: string
builder:
@ -16,7 +16,7 @@ on:
jobs:
run_conf_tests:
runs-on: ${{ inputs.runner }}
runs-on: ${{ fromJSON(inputs.runner_labels) }}
container: ${{ inputs.builder }}
env:
PROFILE: ${{ matrix.profile }}
@ -27,7 +27,6 @@ jobs:
- emqx
- emqx-enterprise
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: ${{ matrix.profile }}

View File

@ -7,7 +7,7 @@ concurrency:
on:
workflow_call:
inputs:
runner:
runner_labels:
required: true
type: string
version-emqx:
@ -19,7 +19,7 @@ on:
jobs:
basic-tests:
runs-on: ${{ inputs.runner }}
runs-on: ${{ fromJSON(inputs.runner_labels) }}
defaults:
run:
shell: bash
@ -63,7 +63,7 @@ jobs:
docker compose rm -fs
paho-mqtt-testing:
runs-on: ${{ inputs.runner }}
runs-on: ${{ fromJSON(inputs.runner_labels) }}
defaults:
run:
shell: bash
@ -83,7 +83,6 @@ jobs:
- mnesia
- rlog
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/checkout@v3
- uses: actions/download-artifact@v3
with:
@ -110,9 +109,8 @@ jobs:
echo "DUMP_CONTAINER_LOGS_END"
exit 1
fi
# node_dump requires netstat, which is not available in the container
# simple smoke test for node_dump
# - name: test node_dump
# run: |
# docker exec -u root node1.emqx.io apt update && apt install -y net-tools
# docker exec node1.emqx.io node_dump
- name: test node_dump
run: |
docker exec -t -u root node1.emqx.io bash -c 'apt-get -y update && apt-get -y install net-tools'
docker exec node1.emqx.io node_dump

View File

@ -10,7 +10,7 @@ concurrency:
on:
workflow_call:
inputs:
runner:
runner_labels:
required: true
type: string
builder:
@ -28,7 +28,7 @@ env:
jobs:
run_emqx_app_tests:
runs-on: ${{ inputs.runner }}
runs-on: ${{ fromJSON(inputs.runner_labels) }}
container: ${{ inputs.builder }}
defaults:
@ -61,5 +61,5 @@ jobs:
- uses: actions/upload-artifact@v3
if: failure()
with:
name: logs-${{ inputs.runner }}
name: logs-emqx-app-tests
path: apps/emqx/_build/test/logs

View File

@ -7,6 +7,9 @@ concurrency:
on:
workflow_call:
inputs:
runner_labels:
required: true
type: string
version-emqx:
required: true
type: string
@ -16,7 +19,7 @@ on:
jobs:
helm_test:
runs-on: ubuntu-22.04
runs-on: ${{ fromJSON(inputs.runner_labels) }}
defaults:
run:
shell: bash

View File

@ -3,13 +3,16 @@ name: JMeter integration tests
on:
workflow_call:
inputs:
runner_labels:
required: true
type: string
version-emqx:
required: true
type: string
jobs:
jmeter_artifact:
runs-on: ubuntu-22.04
runs-on: ${{ fromJSON(inputs.runner_labels) }}
steps:
- name: Cache Jmeter
id: cache-jmeter
@ -38,7 +41,7 @@ jobs:
path: /tmp/apache-jmeter.tgz
advanced_feat:
runs-on: ubuntu-22.04
runs-on: ${{ fromJSON(inputs.runner_labels) }}
strategy:
fail-fast: false
@ -89,7 +92,7 @@ jobs:
path: ./jmeter_logs
pgsql_authn_authz:
runs-on: ubuntu-22.04
runs-on: ${{ fromJSON(inputs.runner_labels) }}
strategy:
fail-fast: false
@ -115,6 +118,7 @@ jobs:
env:
PGSQL_TAG: ${{ matrix.pgsql_tag }}
run: |
docker pull postgres:${PGSQL_TAG}
docker compose \
-f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
-f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \
@ -155,7 +159,7 @@ jobs:
path: ./jmeter_logs
mysql_authn_authz:
runs-on: ubuntu-22.04
runs-on: ${{ fromJSON(inputs.runner_labels) }}
strategy:
fail-fast: false
@ -214,7 +218,7 @@ jobs:
path: ./jmeter_logs
JWT_authn:
runs-on: ubuntu-22.04
runs-on: ${{ fromJSON(inputs.runner_labels) }}
strategy:
fail-fast: false
@ -265,7 +269,7 @@ jobs:
path: ./jmeter_logs
built_in_database_authn_authz:
runs-on: ubuntu-22.04
runs-on: ${{ fromJSON(inputs.runner_labels) }}
strategy:
fail-fast: false

View File

@ -16,7 +16,7 @@ on:
jobs:
relup_test_plan:
runs-on: ${{ inputs.runner }}
runs-on: ["${{ inputs.runner }}", 'linux', 'x64', 'ephemeral']
container: ${{ inputs.builder }}
outputs:
CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }}
@ -25,7 +25,6 @@ jobs:
run:
shell: bash
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: emqx-enterprise
@ -60,7 +59,7 @@ jobs:
needs:
- relup_test_plan
if: needs.relup_test_plan.outputs.OLD_VERSIONS != '[]'
runs-on: ubuntu-22.04
runs-on: ["${{ inputs.runner }}", 'linux', 'x64', 'ephemeral']
strategy:
fail-fast: false
matrix:

View File

@ -7,7 +7,7 @@ concurrency:
on:
workflow_call:
inputs:
runner:
runner_labels:
required: true
type: string
builder:
@ -28,7 +28,7 @@ env:
jobs:
eunit_and_proper:
runs-on: ${{ inputs.runner }}
runs-on: ${{ fromJSON(inputs.runner_labels) }}
name: "eunit_and_proper (${{ matrix.profile }})"
strategy:
fail-fast: false
@ -41,7 +41,6 @@ jobs:
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: ${{ matrix.profile }}
@ -69,7 +68,7 @@ jobs:
path: _build/test/cover
ct_docker:
runs-on: ${{ inputs.runner }}
runs-on: ${{ fromJSON(inputs.runner_labels) }}
name: "ct_docker (${{ matrix.app }}-${{ matrix.suitegroup }})"
strategy:
fail-fast: false
@ -81,7 +80,6 @@ jobs:
shell: bash
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: ${{ matrix.profile }}
@ -110,14 +108,17 @@ jobs:
with:
name: coverdata
path: _build/test/cover
- name: compress logs
if: failure()
run: tar -czf logs.tar.gz _build/test/logs
- uses: actions/upload-artifact@v3
if: failure()
with:
name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
path: _build/test/logs
path: logs.tar.gz
ct:
runs-on: ${{ inputs.runner }}
runs-on: ${{ fromJSON(inputs.runner_labels) }}
name: "ct (${{ matrix.app }}-${{ matrix.suitegroup }})"
strategy:
fail-fast: false
@ -130,7 +131,6 @@ jobs:
shell: bash
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: ${{ matrix.profile }}
@ -152,18 +152,21 @@ jobs:
name: coverdata
path: _build/test/cover
if-no-files-found: warn # do not fail if no coverdata found
- name: compress logs
if: failure()
run: tar -czf logs.tar.gz _build/test/logs
- uses: actions/upload-artifact@v3
if: failure()
with:
name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
path: _build/test/logs
path: logs.tar.gz
tests_passed:
needs:
- eunit_and_proper
- ct
- ct_docker
runs-on: ${{ inputs.runner }}
runs-on: ${{ fromJSON(inputs.runner_labels) }}
strategy:
fail-fast: false
steps:
@ -174,7 +177,7 @@ jobs:
- eunit_and_proper
- ct
- ct_docker
runs-on: ${{ inputs.runner }}
runs-on: ${{ fromJSON(inputs.runner_labels) }}
container: ${{ inputs.builder }}
strategy:
fail-fast: false
@ -182,7 +185,6 @@ jobs:
profile:
- emqx-enterprise
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: ${{ matrix.profile }}
@ -215,7 +217,7 @@ jobs:
# do this in a separate job
upload_coverdata:
needs: make_cover
runs-on: ${{ inputs.runner }}
runs-on: ${{ fromJSON(inputs.runner_labels) }}
steps:
- name: Coveralls Finished
env:

View File

@ -7,7 +7,7 @@ concurrency:
on:
workflow_call:
inputs:
runner:
runner_labels:
required: true
type: string
@ -18,7 +18,7 @@ jobs:
profile:
- emqx
- emqx-enterprise
runs-on: ${{ inputs.runner }}
runs-on: ${{ fromJSON(inputs.runner_labels) }}
steps:
- uses: actions/download-artifact@v3
with:

View File

@ -10,8 +10,8 @@ on:
jobs:
stale:
runs-on: ubuntu-22.04
if: github.repository_owner == 'emqx'
runs-on: ['self-hosted', 'linux', 'x64', 'ephemeral']
permissions:
issues: write
pull-requests: none

View File

@ -7,7 +7,7 @@ concurrency:
on:
workflow_call:
inputs:
runner:
runner_labels:
required: true
type: string
builder:
@ -22,7 +22,7 @@ env:
jobs:
static_checks:
runs-on: ${{ inputs.runner }}
runs-on: ${{ fromJSON(inputs.runner_labels) }}
name: "static_checks (${{ matrix.profile }})"
strategy:
fail-fast: false
@ -30,7 +30,6 @@ jobs:
include: ${{ fromJson(inputs.ct-matrix) }}
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: ${{ matrix.profile }}

View File

@ -240,7 +240,8 @@ user_seeds() ->
%% Disabled
New(<<"mqttuser0006">>, <<"mqttuser0006">>, {error, user_disabled}),
%% IsSuperuser
New(<<"mqttuser0007">>, <<"mqttuser0007">>, {ok, #{is_superuser => true}})
New(<<"mqttuser0007">>, <<"mqttuser0007">>, {ok, #{is_superuser => true}}),
New(<<"mqttuser0008 (test)">>, <<"mqttuser0008 (test)">>, {ok, #{is_superuser => true}})
| Valid
].

View File

@ -166,7 +166,8 @@ t_update(_Config) ->
CorrectConfig = raw_ldap_auth_config(),
IncorrectConfig =
CorrectConfig#{
<<"base_dn">> => <<"ou=testdevice,dc=emqx,dc=io">>
<<"base_dn">> => <<"ou=testdevice,dc=emqx,dc=io">>,
<<"filter">> => <<"(objectClass=mqttUser)">>
},
{ok, _} = emqx:update_config(
@ -207,7 +208,8 @@ raw_ldap_auth_config() ->
<<"mechanism">> => <<"password_based">>,
<<"backend">> => <<"ldap_bind">>,
<<"server">> => ldap_server(),
<<"base_dn">> => <<"uid=${username},ou=testdevice,dc=emqx,dc=io">>,
<<"base_dn">> => <<"ou=testdevice,dc=emqx,dc=io">>,
<<"filter">> => <<"(uid=${username})">>,
<<"username">> => <<"cn=root,dc=emqx,dc=io">>,
<<"password">> => <<"public">>,
<<"pool_size">> => 8,
@ -225,13 +227,22 @@ user_seeds() ->
result => Result
}
end,
Normal = lists:map(
fun(Idx) ->
erlang:iolist_to_binary(io_lib:format("mqttuser000~b", [Idx]))
end,
lists:seq(1, 5)
),
Specials = [<<"mqttuser0008 (test)">>],
Valid =
lists:map(
fun(Idx) ->
Username = erlang:iolist_to_binary(io_lib:format("mqttuser000~b", [Idx])),
fun(Username) ->
New(Username, Username, {ok, #{is_superuser => false}})
end,
lists:seq(1, 5)
Normal ++ Specials
),
[
%% Not exists

View File

@ -3,6 +3,8 @@
%%--------------------------------------------------------------------
-module(emqx_bridge_kafka_impl_producer).
-behaviour(emqx_resource).
-include_lib("emqx_resource/include/emqx_resource.hrl").
-include_lib("snabbkaffe/include/trace.hrl").

View File

@ -1,6 +1,6 @@
{application, emqx_ctl, [
{description, "Backend for emqx_ctl script"},
{vsn, "0.1.4"},
{vsn, "0.1.5"},
{registered, []},
{mod, {emqx_ctl_app, []}},
{applications, [

View File

@ -331,13 +331,14 @@ safe_to_existing_atom(Str) ->
is_initialized() ->
ets:info(?CMD_TAB) =/= undefined.
audit_log(Level, From, Log) ->
audit_log(Level, From, Log = #{args := Args}) ->
case lookup_command(audit) of
{error, _} ->
ignore;
{ok, {Mod, Fun}} ->
try
apply(Mod, Fun, [Level, From, Log])
Log1 = Log#{args => [unicode:characters_to_binary(A) || A <- Args]},
apply(Mod, Fun, [Level, From, Log1])
catch
_:Reason:Stacktrace ->
?LOG_ERROR(#{

View File

@ -2,7 +2,7 @@
{application, emqx_dashboard, [
{description, "EMQX Web Dashboard"},
% strict semver, bump manually!
{vsn, "5.0.28"},
{vsn, "5.0.29"},
{modules, []},
{registered, [emqx_dashboard_sup]},
{applications, [

View File

@ -70,7 +70,7 @@ admins(_) ->
unload() ->
emqx_ctl:unregister_command(admins).
bin(S) -> iolist_to_binary(S).
bin(S) -> unicode:characters_to_binary(S).
print_error(Reason) when is_binary(Reason) ->
emqx_ctl:print("Error: ~s~n", [Reason]).

View File

@ -1,6 +1,6 @@
{application, emqx_eviction_agent, [
{description, "EMQX Eviction Agent"},
{vsn, "5.1.3"},
{vsn, "5.1.4"},
{registered, [
emqx_eviction_agent_sup,
emqx_eviction_agent,

View File

@ -23,6 +23,8 @@
roots/0
]).
-define(TAGS, [<<"Node Eviction">>]).
%% API callbacks
-export([
'/node_eviction/status'/2
@ -44,7 +46,7 @@ schema("/node_eviction/status") ->
#{
'operationId' => '/node_eviction/status',
get => #{
tags => [<<"node_eviction">>],
tags => ?TAGS,
summary => <<"Get node eviction status">>,
description => ?DESC("node_eviction_status_get"),
responses => #{

View File

@ -1,6 +1,6 @@
{application, emqx_ft, [
{description, "EMQX file transfer over MQTT"},
{vsn, "0.1.7"},
{vsn, "0.1.8"},
{registered, []},
{mod, {emqx_ft_app, []}},
{applications, [

View File

@ -110,7 +110,7 @@ schema("/file_transfer") ->
#{
'operationId' => '/file_transfer',
get => #{
tags => [<<"file_transfer">>],
tags => ?TAGS,
summary => <<"Get current File Transfer configuration">>,
description => ?DESC("file_transfer_get_config"),
responses => #{
@ -118,7 +118,7 @@ schema("/file_transfer") ->
}
},
put => #{
tags => [<<"file_transfer">>],
tags => ?TAGS,
summary => <<"Update File Transfer configuration">>,
description => ?DESC("file_transfer_update_config"),
'requestBody' => ?SCHEMA_CONFIG,

View File

@ -226,7 +226,9 @@ on_query(
#{base_tokens := BaseTks, filter_tokens := FilterTks} = State
) ->
Base = emqx_placeholder:proc_tmpl(BaseTks, Data),
FilterBin = emqx_placeholder:proc_tmpl(FilterTks, Data),
FilterBin = emqx_placeholder:proc_tmpl(FilterTks, Data, #{
return => full_binary, var_trans => fun filter_escape/1
}),
case emqx_ldap_filter_parser:scan_and_parse(FilterBin) of
{ok, Filter} ->
do_ldap_query(
@ -278,7 +280,7 @@ do_ldap_query(
error,
LogMeta#{
msg => "ldap_query_found_more_than_one_match",
count => length(Entries)
count => Count
}
),
{error, {unrecoverable_error, Msg}}
@ -314,3 +316,20 @@ do_prepare_template([{filter, V} | T], State) ->
do_prepare_template(T, State#{filter_tokens => emqx_placeholder:preproc_tmpl(V)});
do_prepare_template([], State) ->
State.
filter_escape(Binary) when is_binary(Binary) ->
filter_escape(erlang:binary_to_list(Binary));
filter_escape([$\\ | T]) ->
[$\\, $\\ | filter_escape(T)];
filter_escape([Char | T]) ->
case lists:member(Char, filter_control_chars()) of
true ->
[$\\, Char | filter_escape(T)];
_ ->
[Char | filter_escape(T)]
end;
filter_escape([]) ->
[].
filter_control_chars() ->
[$(, $), $&, $|, $=, $!, $~, $>, $<, $:, $*, $\t, $\n, $\r].

View File

@ -2,8 +2,9 @@ Definitions.
Control = [()&|!=~><:*]
White = [\s\t\n\r]+
NonString = [^()&|!=~><:*\s\t\n\r]
String = {NonString}+
StringChars = [^()&|!=~><:*\t\n\r]
Escape = \\{Control}|\\{White}
String = ({Escape}|{StringChars})+
Rules.
@ -17,10 +18,10 @@ Rules.
>= : {token, {greaterOrEqual, TokenLine}}.
<= : {token, {lessOrEqual, TokenLine}}.
\* : {token, {asterisk, TokenLine}}.
\:dn : {token, {dn, TokenLine}}.
\: : {token, {colon, TokenLine}}.
dn : {token, {dn, TokenLine}}.
{White} : skip_token.
{String} : {token, {string, TokenLine, TokenChars}}.
{String} : {token, {string, TokenLine, to_string(TokenChars)}}.
%% Leex will hang if a composite operation is missing a character
{Control} : {error, lists:flatten(io_lib:format("Unexpected Tokens:~ts", [TokenChars]))}.
@ -29,3 +30,8 @@ Erlang code.
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
%% eldap does not support neither the '\28value\29' nor '\(value\)'
%% so after the tokenization we should remove all escape character
to_string(TokenChars) ->
String = string:trim(TokenChars),
lists:flatten(string:replace(String, "\\", "", all)).

View File

@ -92,7 +92,7 @@ type ->
value: {type, '$1'}.
dnattrs ->
colon dn: {dnAttributes, true}.
dn: {dnAttributes, true}.
matchingrule ->
colon value: {matchingRule, '$2'}.

View File

@ -157,6 +157,15 @@ uid: mqttuser0007
isSuperuser: TRUE
userPassword: {SHA}axpQGbl00j3jvOG058y313ocnBk=
objectClass: top
dn:uid=mqttuser0008 (test),ou=testdevice,dc=emqx,dc=io
objectClass: mqttUser
objectClass: mqttDevice
objectClass: mqttSecurity
uid: mqttuser0008 (test)
isSuperuser: TRUE
userPassword: {SHA}FCzJLOp66OwsZ9DQzXSxdTd9c0U=
## Try to test with base DN 'ou=dashboard,dc=emqx,dc=io'
## with a filter ugroup=group1
## this should return 2 users in the query and fail the test

View File

@ -223,6 +223,23 @@ t_error(_Config) ->
?assertMatch({error, _}, scan_and_parse("attr=value")),
?assertMatch({error, _}, scan_and_parse("(a=b)(c=d)")).
t_escape(_Config) ->
?assertEqual(
'and'([equalityMatch("a", "(value)")]),
parse("(&(a=\\(value\\)))")
),
?assertEqual(
'or'([equalityMatch("a", "name (1)")]),
parse("(|(a=name \\(1\\)))")
),
?assertEqual(
'or'([equalityMatch("a", "name (1) *")]),
parse("(|(a=name\\ \\(1\\) \\*))")
).
t_value_eql_dn(_Config) ->
?assertEqual('and'([equalityMatch("a", "dn")]), parse("(&(a=dn))")).
% %%------------------------------------------------------------------------------
% %% Helpers
% %%------------------------------------------------------------------------------

View File

@ -1,6 +1,6 @@
{application, emqx_node_rebalance, [
{description, "EMQX Node Rebalance"},
{vsn, "5.0.5"},
{vsn, "5.0.6"},
{registered, [
emqx_node_rebalance_sup,
emqx_node_rebalance,

View File

@ -50,6 +50,7 @@
-define(NODE_EVACUATING, 'NODE_EVACUATING').
-define(RPC_ERROR, 'RPC_ERROR').
-define(NOT_FOUND, 'NOT_FOUND').
-define(TAGS, [<<"Load Rebalance">>]).
%%--------------------------------------------------------------------
%% API Spec
@ -78,7 +79,7 @@ schema("/load_rebalance/status") ->
#{
'operationId' => '/load_rebalance/status',
get => #{
tags => [<<"load_rebalance">>],
tags => ?TAGS,
summary => <<"Get rebalance status">>,
description => ?DESC("load_rebalance_status"),
responses => #{
@ -90,7 +91,7 @@ schema("/load_rebalance/global_status") ->
#{
'operationId' => '/load_rebalance/global_status',
get => #{
tags => [<<"load_rebalance">>],
tags => ?TAGS,
summary => <<"Get global rebalance status">>,
description => ?DESC("load_rebalance_global_status"),
responses => #{
@ -102,7 +103,7 @@ schema("/load_rebalance/availability_check") ->
#{
'operationId' => '/load_rebalance/availability_check',
get => #{
tags => [<<"load_rebalance">>],
tags => ?TAGS,
summary => <<"Node rebalance availability check">>,
description => ?DESC("load_rebalance_availability_check"),
responses => #{
@ -115,7 +116,7 @@ schema("/load_rebalance/:node/start") ->
#{
'operationId' => '/load_rebalance/:node/start',
post => #{
tags => [<<"load_rebalance">>],
tags => ?TAGS,
summary => <<"Start rebalancing with the node as coordinator">>,
description => ?DESC("load_rebalance_start"),
parameters => [param_node()],
@ -135,7 +136,7 @@ schema("/load_rebalance/:node/stop") ->
#{
'operationId' => '/load_rebalance/:node/stop',
post => #{
tags => [<<"load_rebalance">>],
tags => ?TAGS,
summary => <<"Stop rebalancing coordinated by the node">>,
description => ?DESC("load_rebalance_stop"),
parameters => [param_node()],
@ -150,7 +151,7 @@ schema("/load_rebalance/:node/evacuation/start") ->
#{
'operationId' => '/load_rebalance/:node/evacuation/start',
post => #{
tags => [<<"load_rebalance">>],
tags => ?TAGS,
summary => <<"Start evacuation on a node">>,
description => ?DESC("load_rebalance_evacuation_start"),
parameters => [param_node()],
@ -170,7 +171,7 @@ schema("/load_rebalance/:node/evacuation/stop") ->
#{
'operationId' => '/load_rebalance/:node/evacuation/stop',
post => #{
tags => [<<"load_rebalance">>],
tags => ?TAGS,
summary => <<"Stop evacuation on a node">>,
description => ?DESC("load_rebalance_evacuation_stop"),
parameters => [param_node()],
@ -186,7 +187,7 @@ schema("/load_rebalance/:node/evacuation/stop") ->
%% #{
%% 'operationId' => '/load_rebalance/:node/purge/start',
%% post => #{
%% tags => [<<"load_rebalance">>],
%% tags => ?TAGS,
%% summary => <<"Start purge on the whole cluster">>,
%% description => ?DESC("cluster_purge_start"),
%% parameters => [param_node()],
@ -206,7 +207,7 @@ schema("/load_rebalance/:node/evacuation/stop") ->
%% #{
%% 'operationId' => '/load_rebalance/:node/purge/stop',
%% post => #{
%% tags => [<<"load_rebalance">>],
%% tags => ?TAGS,
%% summary => <<"Stop purge on the whole cluster">>,
%% description => ?DESC("cluster_purge_stop"),
%% parameters => [param_node()],

View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_resource, [
{description, "Manager for all external resources"},
{vsn, "0.1.23"},
{vsn, "0.1.24"},
{registered, []},
{mod, {emqx_resource_app, []}},
{applications, [

View File

@ -306,8 +306,7 @@ query(ResId, Request, Opts) ->
{simple_async, _} ->
%% TODO(5.1.1): pass Resource instead of ResId to simple APIs
%% so the buffer worker does not need to lookup the cache again
Opts1 = Opts#{is_buffer_supported => true},
emqx_resource_buffer_worker:simple_async_query(ResId, Request, Opts1);
emqx_resource_buffer_worker:simple_async_query(ResId, Request, Opts);
{simple_sync, _} ->
%% TODO(5.1.1): pass Resource instead of ResId to simple APIs
%% so the buffer worker does not need to lookup the cache again

View File

@ -1048,7 +1048,9 @@ call_query(QM, Id, Index, Ref, Query, QueryOpts) ->
?RESOURCE_ERROR(not_found, "resource not found")
end.
do_call_query(QM, Id, Index, Ref, Query, #{is_buffer_supported := true} = QueryOpts, Resource) ->
do_call_query(QM, Id, Index, Ref, Query, QueryOpts, #{query_mode := ResQM} = Resource) when
ResQM =:= simple_async; ResQM =:= simple_sync
->
%% The connector supports buffer, send even in disconnected state
#{mod := Mod, state := ResSt, callback_mode := CBM} = Resource,
CallMode = call_mode(QM, CBM),
@ -1059,7 +1061,7 @@ do_call_query(QM, Id, Index, Ref, Query, QueryOpts, #{status := connected} = Res
#{mod := Mod, state := ResSt, callback_mode := CBM} = Resource,
CallMode = call_mode(QM, CBM),
apply_query_fun(CallMode, Mod, Id, Index, Ref, Query, ResSt, QueryOpts);
do_call_query(_QM, _Id, _Index, _Ref, _Query, _QueryOpts, _Data) ->
do_call_query(_QM, _Id, _Index, _Ref, _Query, _QueryOpts, _Resource) ->
?RESOURCE_ERROR(not_connected, "resource not connected").
-define(APPLY_RESOURCE(NAME, EXPR, REQ),

View File

@ -1,6 +1,6 @@
{application, emqx_schema_registry, [
{description, "EMQX Schema Registry"},
{vsn, "0.1.6"},
{vsn, "0.1.7"},
{registered, [emqx_schema_registry_sup]},
{mod, {emqx_schema_registry_app, []}},
{included_applications, [

View File

@ -22,6 +22,8 @@
'/schema_registry/:name'/2
]).
-define(TAGS, [<<"Schema Registry">>]).
%%-------------------------------------------------------------------------------------------------
%% `minirest' and `minirest_trails' API
%%-------------------------------------------------------------------------------------------------
@ -41,7 +43,7 @@ schema("/schema_registry") ->
#{
'operationId' => '/schema_registry',
get => #{
tags => [<<"schema_registry">>],
tags => ?TAGS,
summary => <<"List registered schemas">>,
description => ?DESC("desc_schema_registry_api_list"),
responses =>
@ -57,7 +59,7 @@ schema("/schema_registry") ->
}
},
post => #{
tags => [<<"schema_registry">>],
tags => ?TAGS,
summary => <<"Register a new schema">>,
description => ?DESC("desc_schema_registry_api_post"),
'requestBody' => emqx_dashboard_swagger:schema_with_examples(
@ -79,7 +81,7 @@ schema("/schema_registry/:name") ->
#{
'operationId' => '/schema_registry/:name',
get => #{
tags => [<<"schema_registry">>],
tags => ?TAGS,
summary => <<"Get registered schema">>,
description => ?DESC("desc_schema_registry_api_get"),
parameters => [param_path_schema_name()],
@ -94,7 +96,7 @@ schema("/schema_registry/:name") ->
}
},
put => #{
tags => [<<"schema_registry">>],
tags => ?TAGS,
summary => <<"Update a schema">>,
description => ?DESC("desc_schema_registry_api_put"),
parameters => [param_path_schema_name()],
@ -113,7 +115,7 @@ schema("/schema_registry/:name") ->
}
},
delete => #{
tags => [<<"schema_registry">>],
tags => ?TAGS,
summary => <<"Delete registered schema">>,
description => ?DESC("desc_schema_registry_api_delete"),
parameters => [param_path_schema_name()],

View File

@ -0,0 +1 @@
Fixed an issue where a Kafka Producer bridge with `sync` query mode would not buffer messages when in the `connecting` state.

View File

@ -0,0 +1,3 @@
Improved the LDAP filter string parser:
1. Automatically escape special characters in filter strings.
2. Fixed a bug that the filter value can't be `dn`.

View File

@ -93,7 +93,7 @@ exit 0
%defattr(-,root,root)
%{_service_dst}
%attr(-,%{_user},%{_group}) %{_lib_home}/*
%attr(-,%{_user},%{_group}) %dir %{_var_home}
%attr(750,%{_user},%{_group}) %dir %{_var_home}
%attr(-,%{_user},%{_group}) %config(noreplace) %{_var_home}/*
%attr(-,%{_user},%{_group}) %dir %{_log_dir}
%attr(-,%{_user},%{_group}) %config(noreplace) %{_conf_dir}/*

View File

@ -41,7 +41,7 @@ license(_) ->
emqx_ctl:usage(
[
{"license info", "Show license info"},
{"license update License", "Update license given as a string"}
{"license update <License>", "Update license given as a string"}
]
).