yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into 0301-merge-release-50-to-master

This commit is contained in:
Zaiming (Stone) Shi 2023-03-01 14:04:29 +01:00
parent b04f7ecccf 1b305e66ab
commit a5f31d0bf4
39 changed files with 366 additions and 324 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.github/workflows/build_and_push_docker_images.yaml vendored
View File

@ -23,9 +23,9 @@ on:
jobs:
prepare:
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
# prepare source with any OTP version, no need for a matrix
container: "ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04"
container: "ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-24.3.4.2-2-ubuntu22.04"
outputs:
PROFILE: ${{ steps.get_profile.outputs.PROFILE }}
@ -109,7 +109,7 @@ jobs:
path: source.zip
docker:
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
needs: prepare
strategy:
@ -125,7 +125,7 @@ jobs:
# NOTE: 'otp' and 'elixir' are to configure emqx-builder image
# only support latest otp and elixir, not a matrix
builder:
- 5.0-28 # update to latest
- 5.0-29 # update to latest
otp:
- 24.3.4.2-2 # switch to 25 once ready to release 5.1
elixir:

25
.github/workflows/build_packages.yaml vendored
View File

@ -22,8 +22,9 @@ on:
jobs:
prepare:
runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04
runs-on: ubuntu-22.04
if: (github.repository_owner == 'emqx' && github.event_name == 'schedule') || github.event_name != 'schedule'
container: ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-24.3.4.2-2-ubuntu22.04
outputs:
BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }}
IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }}
@ -153,6 +154,7 @@ jobs:
- 24.3.4.2-2
os:
- macos-11
- macos-12
- macos-12-arm64
runs-on: ${{ matrix.os }}
steps:
@ -211,19 +213,20 @@ jobs:
- ubuntu18.04
- debian11
- debian10
- el9
- el8
- el7
- amzn2
build_machine:
- aws-arm64
- ubuntu-20.04
- ubuntu-22.04
builder:
- 5.0-28
- 5.0-29
elixir:
- 1.13.4
exclude:
- arch: arm64
build_machine: ubuntu-20.04
build_machine: ubuntu-22.04
- arch: amd64
build_machine: aws-arm64
include:
@ -232,7 +235,7 @@ jobs:
arch: amd64
os: ubuntu22.04
build_machine: ubuntu-22.04
builder: 5.0-28
builder: 5.0-29
elixir: 1.13.4
release_with: elixir
- profile: emqx
@ -240,7 +243,7 @@ jobs:
arch: amd64
os: amzn2
build_machine: ubuntu-22.04
builder: 5.0-28
builder: 5.0-29
elixir: 1.13.4
release_with: elixir
@ -306,7 +309,7 @@ jobs:
{"text": "Scheduled run of ${{ github.workflow }}@${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}
publish_artifacts:
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
needs: [prepare, mac, linux]
if: needs.prepare.outputs.IS_EXACT_TAG && github.event_name != 'schedule'
strategy:
@ -375,9 +378,11 @@ jobs:
push "ubuntu/focal" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu20.04-arm64.deb"
push "ubuntu/jammy" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu22.04-amd64.deb"
push "ubuntu/jammy" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu22.04-arm64.deb"
push "el/6" "packages/$PROFILE/$PROFILE-$VERSION-amzn2-amd64.rpm"
push "el/6" "packages/$PROFILE/$PROFILE-$VERSION-amzn2-arm64.rpm"
push "el/7" "packages/$PROFILE/$PROFILE-$VERSION-el7-amd64.rpm"
push "el/7" "packages/$PROFILE/$PROFILE-$VERSION-el7-arm64.rpm"
push "el/8" "packages/$PROFILE/$PROFILE-$VERSION-el8-amd64.rpm"
push "el/8" "packages/$PROFILE/$PROFILE-$VERSION-el8-arm64.rpm"
push "el/6" "packages/$PROFILE/$PROFILE-$VERSION-amzn2-amd64.rpm"
push "el/6" "packages/$PROFILE/$PROFILE-$VERSION-amzn2-arm64.rpm"
push "el/9" "packages/$PROFILE/$PROFILE-$VERSION-el9-amd64.rpm"
push "el/9" "packages/$PROFILE/$PROFILE-$VERSION-el9-arm64.rpm"

17
.github/workflows/build_slim_packages.yaml vendored
View File

@ -29,15 +29,14 @@ jobs:
fail-fast: false
matrix:
profile:
- ["emqx", "24.3.4.2-2", "el7"]
- ["emqx", "24.3.4.2-2", "ubuntu20.04"]
- ["emqx", "25.1.2-2", "ubuntu22.04"]
- ["emqx-enterprise", "24.3.4.2-2", "ubuntu20.04"]
- ["emqx-enterprise", "25.1.2-2", "ubuntu22.04"]
- ["emqx", "24.3.4.2-2", "el7", "erlang"]
- ["emqx", "25.1.2-2", "ubuntu22.04", "elixir"]
- ["emqx-enterprise", "24.3.4.2-2", "amzn2", "erlang"]
- ["emqx-enterprise", "25.1.2-2", "ubuntu20.04", "erlang"]
builder:
- 5.0-28
- 5.0-29
elixir:
- 1.13.4
- '1.13.4'
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"
@ -54,18 +53,22 @@ jobs:
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: build and test tgz package
if: matrix.profile[3] == 'erlang'
run: |
make ${EMQX_NAME}-tgz
./scripts/pkg-tests.sh ${EMQX_NAME}-tgz
- name: build and test deb/rpm packages
if: matrix.profile[3] == 'erlang'
run: |
make ${EMQX_NAME}-pkg
./scripts/pkg-tests.sh ${EMQX_NAME}-pkg
- name: build and test tgz package (Elixir)
if: matrix.profile[3] == 'elixir'
run: |
make ${EMQX_NAME}-elixir-tgz
./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-tgz
- name: build and test deb/rpm packages (Elixir)
if: matrix.profile[3] == 'elixir'
run: |
make ${EMQX_NAME}-elixir-pkg
./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-pkg

7
.github/workflows/check_deps_integrity.yaml vendored
View File

@ -1,11 +1,12 @@
name: Check Rebar Dependencies
on: [pull_request, push]
on:
pull_request:
jobs:
check_deps_integrity:
runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-25.1.2-2-ubuntu20.04
runs-on: ubuntu-latest
container: ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-25.1.2-2-ubuntu22.04
steps:
- uses: actions/checkout@v3

4
.github/workflows/code_style_check.yaml vendored
View File

@ -4,8 +4,8 @@ on: [pull_request]
jobs:
code_style_check:
runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-25.1.2-2-ubuntu20.04"
runs-on: ubuntu-22.04
container: "ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-25.1.2-2-ubuntu22.04"
steps:
- uses: actions/checkout@v3
with:

5
.github/workflows/elixir_apps_check.yaml vendored
View File

@ -2,13 +2,14 @@
name: Check Elixir Release Applications
on: [pull_request, push]
on:
pull_request:
jobs:
elixir_apps_check:
runs-on: ubuntu-latest
# just use the latest builder
container: "ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-25.1.2-2-ubuntu20.04"
container: "ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-25.1.2-2-ubuntu22.04"
strategy:
fail-fast: false

7
.github/workflows/elixir_deps_check.yaml vendored
View File

@ -2,12 +2,13 @@
name: Elixir Dependency Version Check
on: [pull_request, push]
on:
pull_request:
jobs:
elixir_deps_check:
runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-25.1.2-2-ubuntu20.04
runs-on: ubuntu-latest
container: ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-25.1.2-2-ubuntu22.04
steps:
- name: Checkout

2
.github/workflows/elixir_release.yml vendored
View File

@ -17,7 +17,7 @@ jobs:
profile:
- emqx
- emqx-enterprise
container: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-25.1.2-2-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-25.1.2-2-ubuntu22.04
steps:
- name: Checkout
uses: actions/checkout@v3

8
.github/workflows/run_emqx_app_tests.yaml vendored
View File

@ -12,7 +12,7 @@ jobs:
strategy:
matrix:
builder:
- 5.0-28
- 5.0-29
otp:
- 24.3.4.2-2
- 25.1.2-2
@ -22,16 +22,16 @@ jobs:
elixir:
- 1.13.4
os:
- ubuntu20.04
- ubuntu22.04
arch:
- amd64
runs-on:
- aws-amd64
- ubuntu-20.04
- ubuntu-22.04
use-self-hosted:
- ${{ github.repository_owner == 'emqx' }}
exclude:
- runs-on: ubuntu-20.04
- runs-on: ubuntu-22.04
use-self-hosted: true
- runs-on: aws-amd64
use-self-hosted: false

15
.github/workflows/run_fvt_tests.yaml vendored
View File

@ -7,16 +7,17 @@ concurrency:
on:
push:
branches:
- '**'
- master
- 'ci/**'
tags:
- v*
pull_request:
jobs:
prepare:
runs-on: ubuntu-20.04
runs-on: ubuntu-latest
# prepare source with any OTP version, no need for a matrix
container: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-debian11
container: ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-24.3.4.2-2-debian11
steps:
- uses: actions/checkout@v3
@ -33,7 +34,7 @@ jobs:
path: source.zip
docker_test:
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
needs: prepare
strategy:
@ -49,7 +50,7 @@ jobs:
os:
- ["debian11", "debian:11-slim"]
builder:
- 5.0-28
- 5.0-29
otp:
- 24.3.4.2-2
elixir:
@ -107,7 +108,7 @@ jobs:
docker exec node1.emqx.io node_dump
helm_test:
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
needs: prepare
strategy:
@ -122,7 +123,7 @@ jobs:
os:
- ["debian11", "debian:11-slim"]
builder:
- 5.0-28
- 5.0-29
otp:
- 24.3.4.2-2
elixir:

4
.github/workflows/run_relup_tests.yaml vendored
View File

@ -14,8 +14,8 @@ concurrency:
jobs:
relup_test_plan:
runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04"
runs-on: ubuntu-22.04
container: "ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-24.3.4.2-2-ubuntu22.04"
outputs:
CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }}
OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }}

423
.github/workflows/run_test_cases.yaml vendored
View File

@ -7,225 +7,226 @@ concurrency:
on:
push:
branches:
- '**'
- master
- 'ci/**'
tags:
- v*
- e*
pull_request:
jobs:
build-matrix:
runs-on: ubuntu-latest
outputs:
prepare: ${{ steps.matrix.outputs.prepare }}
host: ${{ steps.matrix.outputs.host }}
docker: ${{ steps.matrix.outputs.docker }}
runs-on: ${{ steps.runner.outputs.runs-on }}
steps:
- uses: actions/checkout@v3
- name: Build matrix
id: matrix
run: |
APPS="$(./scripts/find-apps.sh --ci)"
MATRIX="$(echo "${APPS}" | jq -c '
[
(.[] | select(.profile == "emqx") | . + {
builder: "5.0-28",
otp: "25.1.2-2",
elixir: "1.13.4"
}),
(.[] | select(.profile == "emqx-enterprise") | . + {
builder: "5.0-28",
otp: ["24.3.4.2-2", "25.1.2-2"][],
elixir: "1.13.4"
})
]
')"
echo "${MATRIX}" | jq
MATRIX_PREPARE="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')"
MATRIX_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')"
MATRIX_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')"
echo "prepare=${MATRIX_PREPARE}" | tee -a $GITHUB_OUTPUT
echo "host=${MATRIX_HOST}" | tee -a $GITHUB_OUTPUT
echo "docker=${MATRIX_DOCKER}" | tee -a $GITHUB_OUTPUT
- name: Choose runner host
id: runner
run: |
RUNS_ON="ubuntu-20.04"
${{ github.repository_owner == 'emqx' }} && RUNS_ON="aws-amd64"
echo "runs-on=${RUNS_ON}" | tee -a $GITHUB_OUTPUT
build-matrix:
runs-on: ubuntu-latest
outputs:
prepare: ${{ steps.matrix.outputs.prepare }}
host: ${{ steps.matrix.outputs.host }}
docker: ${{ steps.matrix.outputs.docker }}
runs-on: ${{ steps.runner.outputs.runs-on }}
steps:
- uses: actions/checkout@v3
- name: Build matrix
id: matrix
run: |
APPS="$(./scripts/find-apps.sh --ci)"
MATRIX="$(echo "${APPS}" | jq -c '
[
(.[] | select(.profile == "emqx") | . + {
builder: "5.0-29",
otp: "25.1.2-2",
elixir: "1.13.4"
}),
(.[] | select(.profile == "emqx-enterprise") | . + {
builder: "5.0-29",
otp: ["24.3.4.2-2", "25.1.2-2"][],
elixir: "1.13.4"
})
]
')"
echo "${MATRIX}" | jq
MATRIX_PREPARE="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')"
MATRIX_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')"
MATRIX_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')"
echo "prepare=${MATRIX_PREPARE}" | tee -a $GITHUB_OUTPUT
echo "host=${MATRIX_HOST}" | tee -a $GITHUB_OUTPUT
echo "docker=${MATRIX_DOCKER}" | tee -a $GITHUB_OUTPUT
- name: Choose runner host
id: runner
run: |
RUNS_ON="ubuntu-22.04"
${{ github.repository_owner == 'emqx' }} && RUNS_ON="aws-amd64"
echo "runs-on=${RUNS_ON}" | tee -a $GITHUB_OUTPUT
prepare:
runs-on: ${{ needs.build-matrix.outputs.runs-on }}
needs: [build-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.build-matrix.outputs.prepare) }}
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04"
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/checkout@v3
with:
path: source
- name: get_all_deps
working-directory: source
env:
PROFILE: ${{ matrix.profile }}
#DIAGNOSTIC: 1
run: |
make ensure-rebar3
# fetch all deps and compile
make ${{ matrix.profile }}
make static_checks
make test-compile
cd ..
zip -ryq source.zip source/* source/.[^.]*
- uses: actions/upload-artifact@v3
with:
name: source-${{ matrix.profile }}-${{ matrix.otp }}
path: source.zip
prepare:
runs-on: ${{ needs.build-matrix.outputs.runs-on }}
needs: [build-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.build-matrix.outputs.prepare) }}
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04"
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/checkout@v3
with:
path: source
- name: get_all_deps
working-directory: source
env:
PROFILE: ${{ matrix.profile }}
#DIAGNOSTIC: 1
run: |
make ensure-rebar3
# fetch all deps and compile
make ${{ matrix.profile }}
make static_checks
make test-compile
cd ..
zip -ryq source.zip source/* source/.[^.]*
- uses: actions/upload-artifact@v3
with:
name: source-${{ matrix.profile }}-${{ matrix.otp }}
path: source.zip
eunit_and_proper:
needs:
- build-matrix
- prepare
runs-on: ${{ needs.build-matrix.outputs.runs-on }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.build-matrix.outputs.prepare) }}
eunit_and_proper:
needs:
- build-matrix
- prepare
runs-on: ${{ needs.build-matrix.outputs.runs-on }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.build-matrix.outputs.prepare) }}
defaults:
run:
shell: bash
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04"
defaults:
run:
shell: bash
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04"
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: source-${{ matrix.profile }}-${{ matrix.otp }}
path: .
- name: unzip source code
run: unzip -o -q source.zip
# produces eunit.coverdata
- name: eunit
env:
PROFILE: ${{ matrix.profile }}
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
working-directory: source
run: make eunit
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: source-${{ matrix.profile }}-${{ matrix.otp }}
path: .
- name: unzip source code
run: unzip -o -q source.zip
# produces eunit.coverdata
- name: eunit
env:
PROFILE: ${{ matrix.profile }}
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
working-directory: source
run: make eunit
# produces proper.coverdata
- name: proper
env:
PROFILE: ${{ matrix.profile }}
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
working-directory: source
run: make proper
# produces proper.coverdata
- name: proper
env:
PROFILE: ${{ matrix.profile }}
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
working-directory: source
run: make proper
- uses: actions/upload-artifact@v3
with:
name: coverdata
path: source/_build/test/cover
- uses: actions/upload-artifact@v3
with:
name: coverdata
path: source/_build/test/cover
ct_docker:
needs:
- build-matrix
- prepare
runs-on: ${{ needs.build-matrix.outputs.runs-on }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.build-matrix.outputs.docker) }}
ct_docker:
needs:
- build-matrix
- prepare
runs-on: ${{ needs.build-matrix.outputs.runs-on }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.build-matrix.outputs.docker) }}
defaults:
run:
shell: bash
defaults:
run:
shell: bash
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: source-${{ matrix.profile }}-${{ matrix.otp }}
path: .
- name: unzip source code
run: unzip -q source.zip
- name: run tests
working-directory: source
env:
DOCKER_CT_RUNNER_IMAGE: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04"
MONGO_TAG: "5"
MYSQL_TAG: "8"
PGSQL_TAG: "13"
REDIS_TAG: "7.0"
INFLUXDB_TAG: "2.5.0"
TDENGINE_TAG: "3.0.2.4"
PROFILE: ${{ matrix.profile }}
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }}
- uses: actions/upload-artifact@v3
with:
name: coverdata
path: source/_build/test/cover
- uses: actions/upload-artifact@v3
if: failure()
with:
name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}
path: source/_build/test/logs
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: source-${{ matrix.profile }}-${{ matrix.otp }}
path: .
- name: unzip source code
run: unzip -q source.zip
- name: run tests
working-directory: source
env:
DOCKER_CT_RUNNER_IMAGE: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04"
MONGO_TAG: "5"
MYSQL_TAG: "8"
PGSQL_TAG: "13"
REDIS_TAG: "7.0"
INFLUXDB_TAG: "2.5.0"
TDENGINE_TAG: "3.0.2.4"
PROFILE: ${{ matrix.profile }}
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }}
- uses: actions/upload-artifact@v3
with:
name: coverdata
path: source/_build/test/cover
- uses: actions/upload-artifact@v3
if: failure()
with:
name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}
path: source/_build/test/logs
ct:
needs:
- build-matrix
- prepare
runs-on: ${{ needs.build-matrix.outputs.runs-on }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.build-matrix.outputs.host) }}
ct:
needs:
- build-matrix
- prepare
runs-on: ${{ needs.build-matrix.outputs.runs-on }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.build-matrix.outputs.host) }}
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04"
defaults:
run:
shell: bash
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04"
defaults:
run:
shell: bash
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: source-${{ matrix.profile }}-${{ matrix.otp }}
path: .
- name: unzip source code
run: unzip -q source.zip
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: source-${{ matrix.profile }}-${{ matrix.otp }}
path: .
- name: unzip source code
run: unzip -q source.zip
# produces $PROFILE-<app-name>.coverdata
- name: run common test
working-directory: source
env:
PROFILE: ${{ matrix.profile }}
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
run: |
make "${{ matrix.app }}-ct"
- uses: actions/upload-artifact@v3
with:
name: coverdata
path: source/_build/test/cover
if-no-files-found: warn # do not fail if no coverdata found
- uses: actions/upload-artifact@v3
if: failure()
with:
name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}
path: source/_build/test/logs
# produces $PROFILE-<app-name>.coverdata
- name: run common test
working-directory: source
env:
PROFILE: ${{ matrix.profile }}
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
run: |
make "${{ matrix.app }}-ct"
- uses: actions/upload-artifact@v3
with:
name: coverdata
path: source/_build/test/cover
if-no-files-found: warn # do not fail if no coverdata found
- uses: actions/upload-artifact@v3
if: failure()
with:
name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}
path: source/_build/test/logs
make_cover:
needs:
- eunit_and_proper
- ct
- ct_docker
runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04"
steps:
make_cover:
needs:
- eunit_and_proper
- ct
- ct_docker
runs-on: ubuntu-22.04
container: "ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-24.3.4.2-2-ubuntu22.04"
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
@ -258,15 +259,15 @@ jobs:
if: failure()
run: cat rebar3.crashdump
# do this in a separate job
upload_coverdata:
needs: make_cover
runs-on: ubuntu-20.04
steps:
- name: Coveralls Finished
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
curl -v -k https://coveralls.io/webhook \
--header "Content-Type: application/json" \
--data "{\"repo_name\":\"$GITHUB_REPOSITORY\",\"repo_token\":\"$GITHUB_TOKEN\",\"payload\":{\"build_num\":$GITHUB_RUN_ID,\"status\":\"done\"}}" || true
# do this in a separate job
upload_coverdata:
needs: make_cover
runs-on: ubuntu-20.04
steps:
- name: Coveralls Finished
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
curl -v -k https://coveralls.io/webhook \
--header "Content-Type: application/json" \
--data "{\"repo_name\":\"$GITHUB_REPOSITORY\",\"repo_token\":\"$GITHUB_TOKEN\",\"payload\":{\"build_num\":$GITHUB_RUN_ID,\"status\":\"done\"}}" || true

5
.github/workflows/shellcheck.yaml vendored
View File

@ -1,10 +1,11 @@
name: Shellcheck
on: [pull_request, push]
on:
pull_request:
jobs:
shellcheck:
runs-on: ubuntu-20.04
runs-on: ubuntu-latest
steps:
- name: Checkout source code
uses: actions/checkout@v3

1
.github/workflows/stale.yaml vendored
View File

@ -11,6 +11,7 @@ on:
jobs:
stale:
runs-on: ubuntu-latest
if: github.repository_owner == 'emqx'
permissions:
issues: write
pull-requests: none

17
apps/emqx_connector/src/emqx_connector_http.erl
View File

@ -328,15 +328,17 @@ on_query(
{ok, StatusCode, Headers} ->
?SLOG(error, #{
msg => "http connector do request, received error response",
request => redact(NRequest),
note => "the body will be redacted due to security reasons",
request => redact_request(NRequest),
connector => InstId,
status_code => StatusCode
}),
{error, #{status_code => StatusCode, headers => Headers}};
{ok, StatusCode, Headers, Body} ->
?SLOG(error, #{
msg => "http connector do request, received error response",
request => redact(NRequest),
msg => "http connector do request, received error response.",
note => "the body will be redacted due to security reasons",
request => redact_request(NRequest),
connector => InstId,
status_code => StatusCode
}),
@ -601,6 +603,15 @@ is_sensitive_key(_) ->
redact(Data) ->
emqx_misc:redact(Data, fun is_sensitive_key/1).
%% because the body may contain some sensitive data
%% and at the same time the redact function will not scan the binary data
%% and we also can't know the body format and where the sensitive data will be
%% so the easy way to keep data security is redacted the whole body
redact_request({Path, Headers}) ->
{Path, redact(Headers)};
redact_request({Path, Headers, _Body}) ->
{Path, redact(Headers), <<"******">>}.
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").

1
changes/ce/feat-10022.en.md Normal file
View File

@ -0,0 +1 @@
Start releasing Rocky Linux 9 (compatible with Enterprise Linux 9) and MacOS 12 packages

1
changes/ce/feat-10022.zh.md Normal file
View File

@ -0,0 +1 @@
开始发布Rocky Linux 9（与Enterprise Linux 9兼容）和MacOS 12软件包。

2
changes/ce/feat-9213.zh.md
View File

@ -1 +1 @@
在舵手图中添加吊舱干扰预算。
在 helm chart 中增加 pod disruption budget。

1
changes/ce/feat-9986.en.md Normal file
View File

@ -0,0 +1 @@
For helm charts, add MQTT ingress bridge; and removed stale `mgmt` references.

1
changes/ce/feat-9986.zh.md Normal file
View File

@ -0,0 +1 @@
在 helm chart 中新增了 MQTT 桥接 ingress 的配置参数；并删除了旧版本遗留的 `mgmt` 配置。

1
changes/ce/fix-10013.en.md Normal file
View File

@ -0,0 +1 @@
Fix return type structure for error case in API schema for `/gateways/:name/clients`.

1
changes/ce/fix-10013.zh.md Normal file
View File

@ -0,0 +1 @@
修复 API `/gateways/:name/clients` 返回值的类型结构错误。

1
changes/ce/fix-10014.en.md Normal file
View File

@ -0,0 +1 @@
In dashboard API for `/monitor(_current)/nodes/:node` return `404` instead of `400` if node does not exist.

1
changes/ce/fix-10014.zh.md Normal file
View File

@ -0,0 +1 @@
如果 API 查询的节点不存在，将会返回 404 而不再是 400。

2
changes/ce/fix-10027.en.md Normal file
View File

@ -0,0 +1,2 @@
Allow setting node name from `EMQX_NODE__NAME` when running in docker.
Prior to this fix, only `EMQX_NODE_NAME` is allowed.

2
changes/ce/fix-10027.zh.md Normal file
View File

@ -0,0 +1,2 @@
在 docker 中启动时，允许使用 `EMQX_NODE__NAME` 环境变量来配置节点名。
在此修复前，只能使 `EMQX_NODE_NAME`。

1
changes/ce/perf-9998.en.md Normal file
View File

@ -0,0 +1 @@
Redact the HTTP request body in the authentication error logs for security reasons.

1
changes/ce/perf-9998.zh.md Normal file
View File

@ -0,0 +1 @@
出于安全原因，在身份验证错误日志中模糊 HTTP 请求正文。

2
changes/ee/feat-10011.zh.md
View File

@ -1 +1 @@
在舵手图中添加吊舱干扰预算。
在 helm chart 中增加 pod disruption budget。

4
changes/ee/feat-9881.en.md Normal file
View File

@ -0,0 +1,4 @@
In this pull request, we have enhanced the error logs related to InfluxDB connectivity health checks.
Previously, if InfluxDB failed to pass the health checks using the specified parameters, the only message provided was "timed out waiting for it to become healthy".
With the updated implementation, the error message will be displayed in both the logs and the dashboard, enabling easier identification and resolution of the issue.

3
changes/ee/feat-9881.zh.md Normal file
View File

@ -0,0 +1,3 @@
增强了与 InfluxDB 连接健康检查相关的错误日志。
在此更改之前，如果使用配置的参数 InfluxDB 未能通过健康检查，用户仅能获得一个“超时”的信息。
现在，详细的错误消息将显示在日志和控制台，从而让用户更容易地识别和解决问题。

46
deploy/docker/README.md
View File

@ -50,48 +50,41 @@ The EMQX broker runs as Linux user `emqx` in the docker container.
All EMQX Configuration in [`etc/emqx.conf`](https://github.com/emqx/emqx/blob/master/apps/emqx/etc/emqx.conf) can be configured via environment variables.
By default, the environment variables with `EMQX_` prefix are mapped to key-value pairs in configuration files.
You can change the prefix by overriding `HOCON_ENV_OVERRIDE_PREFIX`.
The environment variables with `EMQX_` prefix are mapped to key-value pairs in configuration files.
Example:
```bash
EMQX_LISTENERS__SSL__DEFAULT__ACCEPTORS <--> listeners.ssl.default.acceptors
EMQX_ZONES__DEFAULT__MQTT__MAX_PACKET_SIZE <--> zones.default.mqtt.max_packet_size
EMQX_DASHBOARD__DEFAULT_PASSWORD <--> dashboard.default_password
EMQX_NODE__COOKIE <--> node.cookie
EMQX_LISTENERS__SSL__default__ENABLE <--> listeners.ssl.default.enable
```
+ Prefix `EMQX_` is removed
+ All upper case letters is replaced with lower case letters
+ `__` is replaced with `.`
If `HOCON_ENV_OVERRIDE_PREFIX=DEV_` is set:
```bash
DEV_LISTENER__SSL__EXTERNAL__ACCEPTORS <--> listener.ssl.external.acceptors
DEV_MQTT__MAX_PACKET_SIZE <--> mqtt.max_packet_size
DEV_LISTENERS__TCP__DEFAULT__BIND <--> listeners.tcp.default.bind
```
For example, set MQTT TCP port to 1883
```console
$ docker run -d --name emqx -e DEV_LISTENERS__TCP__DEFAULT__BIND=1883 -p 18083:18083 -p 1883:1883 emqx/emqx:latest
$ docker run -d --name emqx -e EMQX_DASHBOARD__DEFAULT_PASSWORD=mysecret -p 18083:18083 -p 1883:1883 emqx/emqx:latest
```
Please read more about EMQX configuration in the [official documentation](https://www.emqx.io/docs/en/v5.0/admin/cfg.html).
Please read more about EMQX configuration in the [official documentation](https://www.emqx.io/docs/en/v5.0/configuration/configuration.html)
#### EMQX node name configuration
| Options | Default | Mapped | Description |
| ---------------------------| ------------------ | ------------------------- | ------------------------------------- |
| `EMQX_NAME` | container name | none | EMQX node short name |
| `EMQX_HOST` | container IP | none | EMQX node host, IP or FQDN |
A node name consists of two parts, `EMQX_NAME` part and `EMQX_HOST` part connected by a the symbol `@`. For example: `emqx@127.0.0.1`.
These environment variables are used during container startup phase only in [docker-entrypoint.sh](./docker-entrypoint.sh).
Environment variables `EMQX_NODE_NAME` or `EMQX_NODE__NAME` can be used to set a EMQX node name.
If neither of them is set, EMQX will resolve its node name from the running environment or other environment varialbes used for node discovery.
If `EMQX_NAME` and `EMQX_HOST` are set, and `EMQX_NODE_NAME` is not set, `EMQX_NODE_NAME=$EMQX_NAME@$EMQX_HOST`.
Otherwise `EMQX_NODE_NAME` is taken verbatim.
When running in docker, by default, `EMQX_NAME` and `EMQX_HOST` are resolved as below:
| Options | Default | Description |
| -------------| --------------- | -----------------------------|
| `EMQX_NAME` | container name | EMQX node short name |
| `EMQX_HOST` | container IP | EMQX node host, IP or FQDN |
### Cluster
@ -108,8 +101,7 @@ Let's create a static node list cluster from docker-compose.
emqx1:
image: emqx/emqx:latest
environment:
- "EMQX_NAME=emqx"
- "EMQX_HOST=node1.emqx.io"
- "EMQX_NODE_NAME=emqx@node1.emqx.io"
- "EMQX_CLUSTER__DISCOVERY_STRATEGY=static"
- "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io, emqx@node2.emqx.io]"
networks:
@ -120,8 +112,7 @@ Let's create a static node list cluster from docker-compose.
emqx2:
image: emqx/emqx:latest
environment:
- "EMQX_NAME=emqx"
- "EMQX_HOST=node2.emqx.io"
- "EMQX_NODE_NAME=emqx@node2.emqx.io"
- "EMQX_CLUSTER__DISCOVERY_STRATEGY=static"
- "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io, emqx@node2.emqx.io]"
networks:
@ -174,8 +165,7 @@ services:
image: emqx/emqx:latest
restart: always
environment:
EMQX_NAME: foo_emqx
EMQX_HOST: 127.0.0.1
EMQX_NODE_NAME: foo_emqx@127.0.0.1
volumes:
- vol-emqx-data:/opt/emqx/data
- vol-emqx-etc:/opt/emqx/etc

45
deploy/docker/docker-entrypoint.sh
View File

@ -18,28 +18,31 @@ LOCAL_IP=$(hostname -i | grep -oE '((25[0-5]|(2[0-4]|1[0-9]|[1-9]|)[0-9])\.){3}(
export EMQX_NAME="${EMQX_NAME:-emqx}"
if [[ -z "$EMQX_HOST" ]]; then
if [[ "$EMQX_CLUSTER__DISCOVERY_STRATEGY" == "dns" ]] && \
[[ "$EMQX_CLUSTER__DNS__RECORD_TYPE" == "srv" ]] && \
grep -q "$(hostname).$EMQX_CLUSTER__DNS__NAME" /etc/hosts; then
EMQX_HOST="$(hostname).$EMQX_CLUSTER__DNS__NAME"
elif [[ "$EMQX_CLUSTER__DISCOVERY_STRATEGY" == "k8s" ]] && \
[[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && \
[[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"}
EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX"
elif [[ "$EMQX_CLUSTER__DISCOVERY_STRATEGY" == "k8s" ]] && \
[[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && \
[[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'}
EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX")
else
EMQX_HOST="$LOCAL_IP"
## EMQX_NODE_NAME or EMQX_NODE__NAME to indicate the full node name to be used by EMQX
## If both are set EMQX_NODE_NAME takes higher precedence than EMQX_NODE__NAME
if [[ -z "${EMQX_NODE_NAME:-}" ]] && [[ -z "${EMQX_NODE__NAME:-}" ]]; then
# No node name is provide from environment variables
# try to resolve from other settings
if [[ -z "$EMQX_HOST" ]]; then
if [[ "$EMQX_CLUSTER__DISCOVERY_STRATEGY" == "dns" ]] && \
[[ "$EMQX_CLUSTER__DNS__RECORD_TYPE" == "srv" ]] && \
grep -q "$(hostname).$EMQX_CLUSTER__DNS__NAME" /etc/hosts; then
EMQX_HOST="$(hostname).$EMQX_CLUSTER__DNS__NAME"
elif [[ "$EMQX_CLUSTER__DISCOVERY_STRATEGY" == "k8s" ]] && \
[[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && \
[[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"}
EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX"
elif [[ "$EMQX_CLUSTER__DISCOVERY_STRATEGY" == "k8s" ]] && \
[[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && \
[[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'}
EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX")
else
EMQX_HOST="$LOCAL_IP"
fi
export EMQX_HOST
fi
export EMQX_HOST
fi
if [[ -z "$EMQX_NODE_NAME" ]]; then
export EMQX_NODE_NAME="$EMQX_NAME@$EMQX_HOST"
fi

2
lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl
View File

@ -875,7 +875,7 @@ t_create_disconnected(Config) ->
end),
fun(Trace) ->
?assertMatch(
[#{error := influxdb_client_not_alive}],
[#{error := influxdb_client_not_alive, reason := econnrefused}],
?of_kind(influxdb_connector_start_failed, Trace)
),
ok

2
lib-ee/emqx_ee_connector/rebar.config
View File

@ -1,7 +1,7 @@
{erl_opts, [debug_info]}.
{deps, [
{hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}},
{influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.8"}}},
{influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}},
{tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.5"}}},
{emqx, {path, "../../apps/emqx"}}
]}.

14
lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
View File

@ -234,7 +234,7 @@ do_start_client(
) ->
case influxdb:start_client(ClientConfig) of
{ok, Client} ->
case influxdb:is_alive(Client) of
case influxdb:is_alive(Client, true) of
true ->
State = #{
client => Client,
@ -249,13 +249,15 @@ do_start_client(
state => redact_auth(State)
}),
{ok, State};
false ->
?tp(influxdb_connector_start_failed, #{error => influxdb_client_not_alive}),
{false, Reason} ->
?tp(influxdb_connector_start_failed, #{
error => influxdb_client_not_alive, reason => Reason
}),
?SLOG(warning, #{
msg => "starting influxdb connector failed",
msg => "failed_to_start_influxdb_connector",
connector => InstId,
client => redact_auth(Client),
reason => "client is not alive"
reason => Reason
}),
%% no leak
_ = influxdb:stop_client(Client),
@ -273,7 +275,7 @@ do_start_client(
{error, Reason} ->
?tp(influxdb_connector_start_failed, #{error => Reason}),
?SLOG(warning, #{
msg => "starting influxdb connector failed",
msg => "failed_to_start_influxdb_connector",
connector => InstId,
reason => Reason
}),

4
mix.exs
View File

@ -53,7 +53,7 @@ defmodule EMQXUmbrella.MixProject do
{:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true},
{:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true},
{:esockd, github: "emqx/esockd", tag: "5.9.4", override: true},
{:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.7.2-emqx-7", override: true},
{:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.7.2-emqx-9", override: true},
{:ekka, github: "emqx/ekka", tag: "0.14.2", override: true},
{:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true},
{:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true},
@ -130,7 +130,7 @@ defmodule EMQXUmbrella.MixProject do
defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do
[
{:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"},
{:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.7", override: true},
{:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.9", override: true},
{:wolff, github: "kafka4beam/wolff", tag: "1.7.5"},
{:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.2", override: true},
{:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.0-rc1"},

2
rebar.config
View File

@ -55,7 +55,7 @@
, {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}}
, {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-7"}}}
, {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-9"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.2"}}}
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}
, {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}}

2
scripts/ct/run.sh
View File

@ -17,7 +17,7 @@ help() {
echo "--only-up: Only start the testbed but do not run CT"
echo "--keep-up: Keep the testbed running after CT"
echo "--ci: Set this flag in GitHub action to enforce no tests are skipped"
echo "--" If any, all args after '--' are passed to rebar3 ct
echo "--: If any, all args after '--' are passed to rebar3 ct"
echo " otherwise it runs the entire app's CT"
}