yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #9510 from keynslug/chore/otp-25-compat 

chore: enable OTP-25 compatibility
This commit is contained in:
Andrew Mayorov 2022-12-19 15:12:19 +04:00 committed by GitHub
parent bfe2b67fbe e267b3c89e
commit cbe54c001c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
41 changed files with 234 additions and 266 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.ci/docker-compose-file/Makefile.local
View File

@ -2,7 +2,7 @@
define usage define usage
make -f .ci/docker-compose-file/Makefile.local up make -f .ci/docker-compose-file/Makefile.local up
make -f .ci/docker-compose-file/Makefile.local ct CONTAINER=erlang24 SUITE=apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl make -f .ci/docker-compose-file/Makefile.local ct CONTAINER=erlang SUITE=apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl
make -f .ci/docker-compose-file/Makefile.local down make -f .ci/docker-compose-file/Makefile.local down
endef endef
export usage export usage

2
.ci/docker-compose-file/docker-compose-kafka.yaml
View File

@ -19,7 +19,7 @@ services:
command: /bin/generate-certs.sh command: /bin/generate-certs.sh
kdc: kdc:
hostname: kdc.emqx.net hostname: kdc.emqx.net
image: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04 image: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04
container_name: kdc.emqx.net container_name: kdc.emqx.net
networks: networks:
emqx_bridge: emqx_bridge:

6
.ci/docker-compose-file/docker-compose.yaml
View File

@ -1,9 +1,9 @@
version: '3.9' version: '3.9'
services: services:
erlang24: erlang:
container_name: erlang24 container_name: erlang
image: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04 image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04}
env_file: env_file:
- conf.env - conf.env
environment: environment:

2
.github/actions/package-macos/action.yaml vendored
View File

@ -3,7 +3,7 @@ inputs:
profile: # emqx, emqx-enterprise profile: # emqx, emqx-enterprise
required: true required: true
type: string type: string
otp: # 24.2.1-1, 23.3.4.9-3 otp: # 25.1.2-2, 24.3.4.2-1
required: true required: true
type: string type: string
os: os:

18
.github/workflows/build_and_push_docker_images.yaml vendored
View File

@ -23,7 +23,7 @@ jobs:
prepare: prepare:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
# prepare source with any OTP version, no need for a matrix # prepare source with any OTP version, no need for a matrix
container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04"
outputs: outputs:
BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }}
@ -119,8 +119,10 @@ jobs:
- [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"]
# NOTE: 'otp' and 'elixir' are to configure emqx-builder image # NOTE: 'otp' and 'elixir' are to configure emqx-builder image
# only support latest otp and elixir, not a matrix # only support latest otp and elixir, not a matrix
builder:
- 5.0-26 # update to latest
otp: otp:
- 24.3.4.2-1 # update to latest - 24.3.4.2-1 # switch to 25 once ready to release 5.1
elixir: elixir:
- 1.13.4 # update to latest - 1.13.4 # update to latest
exclude: # TODO: publish enterprise to ecr too? exclude: # TODO: publish enterprise to ecr too?
@ -174,7 +176,7 @@ jobs:
tags: ${{ steps.meta.outputs.tags }} tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }} labels: ${{ steps.meta.outputs.labels }}
build-args: | build-args: |
BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} BUILD_FROM=ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
RUN_FROM=${{ matrix.os[1] }} RUN_FROM=${{ matrix.os[1] }}
EMQX_NAME=${{ steps.meta.outputs.emqx_name }} EMQX_NAME=${{ steps.meta.outputs.emqx_name }}
file: source/${{ matrix.os[2] }} file: source/${{ matrix.os[2] }}
@ -208,8 +210,10 @@ jobs:
- 'docker.io' - 'docker.io'
os: os:
- [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"]
builder:
- 5.0-26 # update to latest
otp: otp:
- 24.3.4.2-1 # update to latest - 25.1.2-2 # update to latest
elixir: elixir:
- 1.13.4 # update to latest - 1.13.4 # update to latest
@ -252,7 +256,7 @@ jobs:
tags: ${{ steps.meta.outputs.tags }} tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }} labels: ${{ steps.meta.outputs.labels }}
build-args: | build-args: |
BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} BUILD_FROM=ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
RUN_FROM=${{ matrix.os[1] }} RUN_FROM=${{ matrix.os[1] }}
EMQX_NAME=${{ steps.meta.outputs.emqx_name }} EMQX_NAME=${{ steps.meta.outputs.emqx_name }}
file: source/${{ matrix.os[2] }} file: source/${{ matrix.os[2] }}
@ -277,7 +281,7 @@ jobs:
- [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"]
# NOTE: only support latest otp version, not a matrix # NOTE: only support latest otp version, not a matrix
otp: otp:
- 24.3.4.2-1 # update to latest - 24.3.4.2-1 # switch to 25 once ready to release 5.1
registry: registry:
- 'docker.io' - 'docker.io'
- 'public.ecr.aws' - 'public.ecr.aws'
@ -342,7 +346,7 @@ jobs:
- ${{ needs.prepare.outputs.BUILD_PROFILE }} - ${{ needs.prepare.outputs.BUILD_PROFILE }}
# NOTE: for docker, only support latest otp version, not a matrix # NOTE: for docker, only support latest otp version, not a matrix
otp: otp:
- 24.3.4.2-1 # update to latest - 25.1.2-2 # update to latest
elixir: elixir:
- 1.13.4 # update to latest - 1.13.4 # update to latest
registry: registry:

13
.github/workflows/build_packages.yaml vendored
View File

@ -23,7 +23,7 @@ on:
jobs: jobs:
prepare: prepare:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04 container: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04
outputs: outputs:
BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }}
IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }}
@ -48,7 +48,7 @@ jobs:
echo "WARN: This is NOT an exact git tag, will not publish release" echo "WARN: This is NOT an exact git tag, will not publish release"
is_exact_tag='false' is_exact_tag='false'
fi fi
echo "::set-output name=IS_EXACT_TAG::${is_exact_tag}" echo "IS_EXACT_TAG=${is_exact_tag}" >> $GITHUB_OUTPUT
case $tag in case $tag in
refs/tags/v*) refs/tags/v*)
PROFILE='emqx' PROFILE='emqx'
@ -103,7 +103,7 @@ jobs:
- uses: ilammy/msvc-dev-cmd@v1.12.0 - uses: ilammy/msvc-dev-cmd@v1.12.0
- uses: erlef/setup-beam@v1 - uses: erlef/setup-beam@v1
with: with:
otp-version: 24.2.1 otp-version: 24.3.4
- name: build - name: build
env: env:
PYTHON: python PYTHON: python
@ -173,13 +173,15 @@ jobs:
needs: prepare needs: prepare
runs-on: ${{ matrix.build_machine }} runs-on: ${{ matrix.build_machine }}
container: container:
image: "ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" image: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
profile: profile:
- ${{ needs.prepare.outputs.BUILD_PROFILE }} - ${{ needs.prepare.outputs.BUILD_PROFILE }}
builder:
- 5.0-26
otp: otp:
- 24.3.4.2-1 - 24.3.4.2-1
elixir: elixir:
@ -244,6 +246,7 @@ jobs:
- name: build emqx packages - name: build emqx packages
working-directory: source working-directory: source
env: env:
BUILDER: ${{ matrix.builder }}
OTP: ${{ matrix.otp }} OTP: ${{ matrix.otp }}
ELIXIR: ${{ matrix.elixir }} ELIXIR: ${{ matrix.elixir }}
PROFILE: ${{ matrix.profile }} PROFILE: ${{ matrix.profile }}
@ -272,7 +275,7 @@ jobs:
--pkgtype "${PKGTYPE}" \ --pkgtype "${PKGTYPE}" \
--arch "${ARCH}" \ --arch "${ARCH}" \
--elixir "${IsElixir}" \ --elixir "${IsElixir}" \
--builder "ghcr.io/emqx/emqx-builder/5.0-24:${ELIXIR}-${OTP}-${SYSTEM}" --builder "ghcr.io/emqx/emqx-builder/${BUILDER}:${ELIXIR}-${OTP}-${SYSTEM}"
done done
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
with: with:

17
.github/workflows/build_slim_packages.yaml vendored
View File

@ -29,14 +29,15 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
profile: profile:
- ["emqx", "el7"] - ["emqx", "24.3.4.2-1", "el7"]
- ["emqx-enterprise", "ubuntu20.04"] - ["emqx", "25.1.2-2", "ubuntu20.04"]
otp: - ["emqx-enterprise", "24.3.4.2-1", "ubuntu20.04"]
- 24.3.4.2-1 builder:
- 5.0-26
elixir: elixir:
- 1.13.4 - 1.13.4
container: "ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.profile[1] }}" container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"
steps: steps:
- uses: AutoModality/action-clean@v1 - uses: AutoModality/action-clean@v1
@ -72,7 +73,7 @@ jobs:
./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-pkg ./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-pkg
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
with: with:
name: "${{ matrix.profile[0] }}-${{ matrix.otp }}-${{ matrix.profile[1] }}" name: "${{ matrix.profile[0] }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"
path: _packages/${{ matrix.profile[0] }}/* path: _packages/${{ matrix.profile[0] }}/*
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
with: with:
@ -89,7 +90,7 @@ jobs:
profile: profile:
- emqx - emqx
otp: otp:
- 24.2.1 - 25.1.2
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: ilammy/msvc-dev-cmd@v1.12.0 - uses: ilammy/msvc-dev-cmd@v1.12.0
@ -119,7 +120,7 @@ jobs:
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
with: with:
name: windows name: windows
path: _packages/${{ matrix.profile}}/* path: _packages/${{ matrix.profile }}/*
mac: mac:
strategy: strategy:

2
.github/workflows/check_deps_integrity.yaml vendored
View File

@ -5,7 +5,7 @@ on: [pull_request, push]
jobs: jobs:
check_deps_integrity: check_deps_integrity:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04 container: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-25.1.2-2-ubuntu20.04
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3

2
.github/workflows/code_style_check.yaml vendored
View File

@ -5,7 +5,7 @@ on: [pull_request]
jobs: jobs:
code_style_check: code_style_check:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-25.1.2-2-ubuntu20.04"
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:

2
.github/workflows/elixir_apps_check.yaml vendored
View File

@ -8,7 +8,7 @@ jobs:
elixir_apps_check: elixir_apps_check:
runs-on: ubuntu-latest runs-on: ubuntu-latest
# just use the latest builder # just use the latest builder
container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-25.1.2-2-ubuntu20.04"
strategy: strategy:
fail-fast: false fail-fast: false

2
.github/workflows/elixir_deps_check.yaml vendored
View File

@ -7,7 +7,7 @@ on: [pull_request, push]
jobs: jobs:
elixir_deps_check: elixir_deps_check:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04 container: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-25.1.2-2-ubuntu20.04
steps: steps:
- name: Checkout - name: Checkout

3
.github/workflows/elixir_release.yml vendored
View File

@ -17,8 +17,7 @@ jobs:
profile: profile:
- emqx - emqx
- emqx-enterprise - emqx-enterprise
container: ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04 container: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-25.1.2-2-ubuntu20.04
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v3 uses: actions/checkout@v3

5
.github/workflows/run_emqx_app_tests.yaml vendored
View File

@ -11,8 +11,11 @@ jobs:
run_emqx_app_tests: run_emqx_app_tests:
strategy: strategy:
matrix: matrix:
builder:
- 5.0-26
otp: otp:
- 24.3.4.2-1 - 24.3.4.2-1
- 25.1.2-2
# no need to use more than 1 version of Elixir, since tests # no need to use more than 1 version of Elixir, since tests
# run using only Erlang code. This is needed just to specify # run using only Erlang code. This is needed just to specify
# the base image. # the base image.
@ -34,7 +37,7 @@ jobs:
use-self-hosted: false use-self-hosted: false
runs-on: ${{ matrix.runs-on }} runs-on: ${{ matrix.runs-on }}
container: "ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}" container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}"
defaults: defaults:
run: run:

14
.github/workflows/run_fvt_tests.yaml vendored
View File

@ -16,7 +16,7 @@ jobs:
prepare: prepare:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
# prepare source with any OTP version, no need for a matrix # prepare source with any OTP version, no need for a matrix
container: ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-alpine3.15.1 container: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-alpine3.15.1
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
@ -48,6 +48,8 @@ jobs:
- rlog - rlog
os: os:
- ["alpine3.15.1", "alpine:3.15.1"] - ["alpine3.15.1", "alpine:3.15.1"]
builder:
- 5.0-26
otp: otp:
- 24.3.4.2-1 - 24.3.4.2-1
elixir: elixir:
@ -57,7 +59,7 @@ jobs:
steps: steps:
- uses: erlef/setup-beam@v1 - uses: erlef/setup-beam@v1
with: with:
otp-version: "24.2" otp-version: 24.3.4
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: source name: source
@ -68,7 +70,7 @@ jobs:
- name: make docker image - name: make docker image
working-directory: source working-directory: source
env: env:
EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} EMQX_BUILDER: ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
EMQX_RUNNER: ${{ matrix.os[1] }} EMQX_RUNNER: ${{ matrix.os[1] }}
run: | run: |
make ${{ matrix.profile }}-docker make ${{ matrix.profile }}-docker
@ -119,6 +121,8 @@ jobs:
- emqx-enterprise - emqx-enterprise
os: os:
- ["debian11", "debian:11-slim"] - ["debian11", "debian:11-slim"]
builder:
- 5.0-26
otp: otp:
- 24.3.4.2-1 - 24.3.4.2-1
elixir: elixir:
@ -130,7 +134,7 @@ jobs:
steps: steps:
- uses: erlef/setup-beam@v1 - uses: erlef/setup-beam@v1
with: with:
otp-version: "24.2" otp-version: 24.3.4
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: source name: source
@ -141,7 +145,7 @@ jobs:
- name: make docker image - name: make docker image
working-directory: source working-directory: source
env: env:
EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} EMQX_BUILDER: ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
EMQX_RUNNER: ${{ matrix.os[1] }} EMQX_RUNNER: ${{ matrix.os[1] }}
run: | run: |
make ${{ matrix.profile }}-docker make ${{ matrix.profile }}-docker

14
.github/workflows/run_jmeter_tests.yaml vendored
View File

@ -16,7 +16,7 @@ jobs:
steps: steps:
- uses: erlef/setup-beam@v1 - uses: erlef/setup-beam@v1
with: with:
otp-version: "24.2" otp-version: 24.3
- name: download jmeter - name: download jmeter
timeout-minutes: 3 timeout-minutes: 3
env: env:
@ -36,7 +36,7 @@ jobs:
PROFILE='emqx' PROFILE='emqx'
make "${PROFILE}-docker" make "${PROFILE}-docker"
VSN="$(./pkg-vsn.sh $PROFILE)" VSN="$(./pkg-vsn.sh $PROFILE)"
echo "::set-output name=version::${VSN}" echo "version=${VSN}" >> $GITHUB_OUTPUT
docker save -o emqx.tar emqx/emqx:${VSN} docker save -o emqx.tar emqx/emqx:${VSN}
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
with: with:
@ -59,7 +59,7 @@ jobs:
steps: steps:
- uses: erlef/setup-beam@v1 - uses: erlef/setup-beam@v1
with: with:
otp-version: "24.2" otp-version: 24.3
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
@ -155,7 +155,7 @@ jobs:
steps: steps:
- uses: erlef/setup-beam@v1 - uses: erlef/setup-beam@v1
with: with:
otp-version: "24.2" otp-version: 24.3
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
@ -261,7 +261,7 @@ jobs:
steps: steps:
- uses: erlef/setup-beam@v1 - uses: erlef/setup-beam@v1
with: with:
otp-version: "24.2" otp-version: 24.3
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
@ -363,7 +363,7 @@ jobs:
steps: steps:
- uses: erlef/setup-beam@v1 - uses: erlef/setup-beam@v1
with: with:
otp-version: "24.2" otp-version: 24.3
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
@ -462,7 +462,7 @@ jobs:
steps: steps:
- uses: erlef/setup-beam@v1 - uses: erlef/setup-beam@v1
with: with:
otp-version: "24.2" otp-version: 24.3
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:

8
.github/workflows/run_relup_tests.yaml vendored
View File

@ -16,7 +16,7 @@ on:
jobs: jobs:
relup_test_plan: relup_test_plan:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04"
outputs: outputs:
CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }} CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }}
OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }} OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }}
@ -37,8 +37,8 @@ jobs:
ee_vsn="$(./pkg-vsn.sh enterprise)" ee_vsn="$(./pkg-vsn.sh enterprise)"
old_ee_vsns="$(./scripts/relup-build/base-vsns.sh enterprise | xargs)" old_ee_vsns="$(./scripts/relup-build/base-vsns.sh enterprise | xargs)"
old_vsns=$(echo -n "${old_ee_vsns}" | sed 's/ $//g' | jq -R -s -c 'split(" ")') old_vsns=$(echo -n "${old_ee_vsns}" | sed 's/ $//g' | jq -R -s -c 'split(" ")')
echo "::set-output name=CUR_EE_VSN::$ee_vsn" echo "CUR_EE_VSN=$ee_vsn" >> $GITHUB_OUTPUT
echo "::set-output name=OLD_VERSIONS::$old_vsns" echo "OLD_VERSIONS=$old_vsns" >> $GITHUB_OUTPUT
- name: build emqx - name: build emqx
run: | run: |
set -x set -x
@ -74,7 +74,7 @@ jobs:
# setup Erlang to run lux # setup Erlang to run lux
- uses: erlef/setup-beam@v1 - uses: erlef/setup-beam@v1
with: with:
otp-version: "24.2" otp-version: 24.3
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:
repository: hawk/lux repository: hawk/lux

183
.github/workflows/run_test_cases.yaml vendored
View File

@ -14,49 +14,55 @@ on:
pull_request: pull_request:
jobs: jobs:
build-matrix:
runs-on: ubuntu-latest
outputs:
prepare: ${{ steps.matrix.outputs.prepare }}
host: ${{ steps.matrix.outputs.host }}
docker: ${{ steps.matrix.outputs.docker }}
runs-on: ${{ steps.runner.outputs.runs-on }}
steps:
- uses: actions/checkout@v3
- name: Build matrix
id: matrix
run: |
APPS="$(./scripts/find-apps.sh --ci)"
MATRIX="$(echo "${APPS}" | jq -c '
[
(.[] | select(.profile == "emqx") | . + {
builder: "5.0-26",
otp: "25.1.2-2",
elixir: "1.13.4"
}),
(.[] | select(.profile == "emqx-enterprise") | . + {
builder: "5.0-26",
otp: ["24.3.4.2-1", "25.1.2-2"][],
elixir: "1.13.4"
})
]
')"
echo "${MATRIX}" | jq
MATRIX_PREPARE="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')"
MATRIX_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')"
MATRIX_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')"
echo "prepare=${MATRIX_PREPARE}" | tee -a $GITHUB_OUTPUT
echo "host=${MATRIX_HOST}" | tee -a $GITHUB_OUTPUT
echo "docker=${MATRIX_DOCKER}" | tee -a $GITHUB_OUTPUT
- name: Choose runner host
id: runner
run: |
RUNS_ON="ubuntu-20.04"
${{ github.repository_owner == 'emqx' }} && RUNS_ON="aws-amd64"
echo "runs-on=${RUNS_ON}" | tee -a $GITHUB_OUTPUT
prepare: prepare:
runs-on: aws-amd64 runs-on: aws-amd64
# prepare source with any OTP version, no need for a matrix needs: [build-matrix]
container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" strategy:
outputs: fail-fast: false
fast_ct_apps: ${{ steps.find_ct_apps.outputs.fast_ct_apps }} matrix:
docker_ct_apps: ${{ steps.find_ct_apps.outputs.docker_ct_apps }} include: ${{ fromJson(needs.build-matrix.outputs.prepare) }}
steps: container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04"
- uses: AutoModality/action-clean@v1
- uses: actions/checkout@v3
with:
path: source
- name: Find CT Apps
working-directory: source
id: find_ct_apps
run: |
fast_ct_apps="$(./scripts/find-apps.sh --ci fast)"
docker_ct_apps="$(./scripts/find-apps.sh --ci docker)"
echo "fast: $fast_ct_apps"
echo "docker: $docker_ct_apps"
echo "::set-output name=fast_ct_apps::$fast_ct_apps"
echo "::set-output name=docker_ct_apps::$docker_ct_apps"
- name: get_all_deps
working-directory: source
env:
PROFILE: emqx
#DIAGNOSTIC: 1
run: |
make ensure-rebar3
# fetch all deps and compile
make emqx
make test-compile
cd ..
zip -ryq source.zip source/* source/.[^.]*
- uses: actions/upload-artifact@v3
with:
name: source-emqx
path: source.zip
prepare_ee:
runs-on: aws-amd64
# prepare source with any OTP version, no need for a matrix
container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04"
steps: steps:
- uses: AutoModality/action-clean@v1 - uses: AutoModality/action-clean@v1
- uses: actions/checkout@v3 - uses: actions/checkout@v3
@ -65,61 +71,48 @@ jobs:
- name: get_all_deps - name: get_all_deps
working-directory: source working-directory: source
env: env:
PROFILE: emqx-enterprise PROFILE: ${{ matrix.profile }}
#DIAGNOSTIC: 1 #DIAGNOSTIC: 1
run: | run: |
make ensure-rebar3 make ensure-rebar3
# fetch all deps and compile # fetch all deps and compile
make emqx-enterprise make ${{ matrix.profile }}
make test-compile make test-compile
cd .. cd ..
zip -ryq source.zip source/* source/.[^.]* zip -ryq source.zip source/* source/.[^.]*
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
with: with:
name: source-emqx-enterprise name: source-${{ matrix.profile }}-${{ matrix.otp }}
path: source.zip path: source.zip
eunit_and_proper: eunit_and_proper:
needs: needs:
- build-matrix
- prepare - prepare
- prepare_ee runs-on: ${{ needs.build-matrix.outputs.runs-on }}
runs-on: ${{ matrix.runs-on }}
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
profile: include: ${{ fromJson(needs.build-matrix.outputs.prepare) }}
- emqx
- emqx-enterprise
runs-on:
- aws-amd64
- ubuntu-20.04
use-self-hosted:
- ${{ github.repository_owner == 'emqx' }}
exclude:
- runs-on: ubuntu-20.04
use-self-hosted: true
- runs-on: aws-amd64
use-self-hosted: false
defaults: defaults:
run: run:
shell: bash shell: bash
container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04"
steps: steps:
- uses: AutoModality/action-clean@v1 - uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: source-${{ matrix.profile }} name: source-${{ matrix.profile }}-${{ matrix.otp }}
path: . path: .
- name: unzip source code - name: unzip source code
env:
PROFILE: ${{ matrix.profile }}
run: unzip -o -q source.zip run: unzip -o -q source.zip
# produces eunit.coverdata # produces eunit.coverdata
- name: eunit - name: eunit
env: env:
PROFILE: ${{ matrix.profile }} PROFILE: ${{ matrix.profile }}
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
working-directory: source working-directory: source
run: make eunit run: make eunit
@ -127,6 +120,7 @@ jobs:
- name: proper - name: proper
env: env:
PROFILE: ${{ matrix.profile }} PROFILE: ${{ matrix.profile }}
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
working-directory: source working-directory: source
run: make proper run: make proper
@ -137,23 +131,14 @@ jobs:
ct_docker: ct_docker:
needs: needs:
- build-matrix
- prepare - prepare
- prepare_ee runs-on: ${{ needs.build-matrix.outputs.runs-on }}
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
app: ${{ fromJson(needs.prepare.outputs.docker_ct_apps) }} include: ${{ fromJson(needs.build-matrix.outputs.docker) }}
runs-on:
- aws-amd64
- ubuntu-20.04
use-self-hosted:
- ${{ github.repository_owner == 'emqx' }}
exclude:
- runs-on: ubuntu-20.04
use-self-hosted: true
- runs-on: aws-amd64
use-self-hosted: false
runs-on: ${{ matrix.runs-on }}
defaults: defaults:
run: run:
shell: bash shell: bash
@ -162,24 +147,22 @@ jobs:
- uses: AutoModality/action-clean@v1 - uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: source-${{ matrix.app[1] }} name: source-${{ matrix.profile }}-${{ matrix.otp }}
path: . path: .
- name: unzip source code - name: unzip source code
run: unzip -q source.zip run: unzip -q source.zip
- name: run tests - name: run tests
working-directory: source working-directory: source
env: env:
DOCKER_CT_RUNNER_IMAGE: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04"
MONGO_TAG: 5 MONGO_TAG: 5
MYSQL_TAG: 8 MYSQL_TAG: 8
PGSQL_TAG: 13 PGSQL_TAG: 13
REDIS_TAG: 6 REDIS_TAG: 6
INFLUXDB_TAG: 2.5.0 INFLUXDB_TAG: 2.5.0
WHICH_APP: ${{ matrix.app[0] }} PROFILE: ${{ matrix.profile }}
PROFILE: ${{ matrix.app[1] }} CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
run: | run: ./scripts/ct/run.sh --app ${{ matrix.app }}
echo $PROFILE
rm _build/default/lib/rocksdb/_build/cmake/CMakeCache.txt
./scripts/ct/run.sh --app $WHICH_APP
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
with: with:
name: coverdata name: coverdata
@ -187,30 +170,20 @@ jobs:
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
if: failure() if: failure()
with: with:
name: logs-${{ matrix.app[1] }}-${{ matrix.app[2] }} name: logs-${{ matrix.profile }}-${{ matrix.prefix }}
path: source/_build/test/logs path: source/_build/test/logs
ct: ct:
needs: needs:
- build-matrix
- prepare - prepare
- prepare_ee runs-on: ${{ needs.build-matrix.outputs.runs-on }}
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
app: ${{ fromJson(needs.prepare.outputs.fast_ct_apps) }} include: ${{ fromJson(needs.build-matrix.outputs.host) }}
runs-on:
- aws-amd64
- ubuntu-20.04
use-self-hosted:
- ${{ github.repository_owner == 'emqx' }}
exclude:
- runs-on: ubuntu-20.04
use-self-hosted: true
- runs-on: aws-amd64
use-self-hosted: false
runs-on: ${{ matrix.runs-on }} container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04"
container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04"
defaults: defaults:
run: run:
shell: bash shell: bash
@ -219,7 +192,7 @@ jobs:
- uses: AutoModality/action-clean@v1 - uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: source-${{ matrix.app[1] }} name: source-${{ matrix.profile }}-${{ matrix.otp }}
path: . path: .
- name: unzip source code - name: unzip source code
run: unzip -q source.zip run: unzip -q source.zip
@ -228,10 +201,10 @@ jobs:
- name: run common test - name: run common test
working-directory: source working-directory: source
env: env:
WHICH_APP: ${{ matrix.app[0] }} PROFILE: ${{ matrix.profile }}
PROFILE: ${{ matrix.app[1] }} CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
run: | run: |
make "${WHICH_APP}-ct" make "${{ matrix.app }}-ct"
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
with: with:
name: coverdata name: coverdata
@ -240,7 +213,7 @@ jobs:
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
if: failure() if: failure()
with: with:
name: logs-${{ matrix.app[1] }}-${{ matrix.app[2] }} name: logs-${{ matrix.profile }}-${{ matrix.prefix }}
path: source/_build/test/logs path: source/_build/test/logs
make_cover: make_cover:
@ -249,12 +222,12 @@ jobs:
- ct - ct
- ct_docker - ct_docker
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04"
steps: steps:
- uses: AutoModality/action-clean@v1 - uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: source-emqx-enterprise name: source-emqx-enterprise-24.3.4.2-1
path: . path: .
- name: unzip source code - name: unzip source code
run: unzip -q source.zip run: unzip -q source.zip

12
Makefile
View File

@ -2,7 +2,7 @@ REBAR = $(CURDIR)/rebar3
BUILD = $(CURDIR)/build BUILD = $(CURDIR)/build
SCRIPTS = $(CURDIR)/scripts SCRIPTS = $(CURDIR)/scripts
export EMQX_RELUP ?= true export EMQX_RELUP ?= true
export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-debian11 export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11
export EMQX_DEFAULT_RUNNER = debian:11-slim export EMQX_DEFAULT_RUNNER = debian:11-slim
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
@ -24,6 +24,7 @@ PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default
CT_NODE_NAME ?= 'test@127.0.0.1' CT_NODE_NAME ?= 'test@127.0.0.1'
CT_READABLE ?= true CT_READABLE ?= true
CT_COVER_EXPORT_PREFIX ?= $(PROFILE)
export REBAR_GIT_CLONE_OPTIONS += --depth=1 export REBAR_GIT_CLONE_OPTIONS += --depth=1
@ -62,7 +63,7 @@ mix-deps-get: $(ELIXIR_COMMON_DEPS)
.PHONY: eunit .PHONY: eunit
eunit: $(REBAR) merge-config eunit: $(REBAR) merge-config
@ENABLE_COVER_COMPILE=1 $(REBAR) eunit -v -c --cover_export_name $(PROFILE)-eunit @ENABLE_COVER_COMPILE=1 $(REBAR) eunit -v -c --cover_export_name $(CT_COVER_EXPORT_PREFIX)-eunit
.PHONY: proper .PHONY: proper
proper: $(REBAR) proper: $(REBAR)
@ -74,7 +75,7 @@ test-compile: $(REBAR) merge-config
.PHONY: ct .PHONY: ct
ct: $(REBAR) merge-config ct: $(REBAR) merge-config
@ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(PROFILE)-ct @ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(CT_COVER_EXPORT_PREFIX)-ct
.PHONY: static_checks .PHONY: static_checks
static_checks: static_checks:
@ -86,7 +87,10 @@ APPS=$(shell $(SCRIPTS)/find-apps.sh)
define gen-app-ct-target define gen-app-ct-target
$1-ct: $(REBAR) $1-ct: $(REBAR)
@$(SCRIPTS)/pre-compile.sh $(PROFILE) @$(SCRIPTS)/pre-compile.sh $(PROFILE)
@ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(PROFILE)-$(subst /,-,$1) --suite $(shell $(SCRIPTS)/find-suites.sh $1) @ENABLE_COVER_COMPILE=1 $(REBAR) ct -c -v \
--name $(CT_NODE_NAME) \
--cover_export_name $(CT_COVER_EXPORT_PREFIX)-$(subst /,-,$1) \
--suite $(shell $(SCRIPTS)/find-suites.sh $1)
endef endef
$(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))

2
apps/emqx/rebar.config
View File

@ -50,7 +50,7 @@
]}. ]}.
{dialyzer, [ {dialyzer, [
{warnings, [unmatched_returns, error_handling, race_conditions]}, {warnings, [unmatched_returns, error_handling]},
{plt_location, "."}, {plt_location, "."},
{plt_prefix, "emqx_dialyzer"}, {plt_prefix, "emqx_dialyzer"},
{plt_apps, all_apps}, {plt_apps, all_apps},

1
apps/emqx/src/emqx_connection.erl
View File

@ -641,6 +641,7 @@ close_socket_ok(State) ->
ok. ok.
%% tell truth about the original exception %% tell truth about the original exception
-spec maybe_raise_exception(any()) -> no_return().
maybe_raise_exception(#{ maybe_raise_exception(#{
exception := Exception, exception := Exception,
context := Context, context := Context,

16
apps/emqx/src/emqx_tls_lib.erl
View File

@ -55,18 +55,6 @@
%% non-empty list of strings %% non-empty list of strings
-define(IS_STRING_LIST(L), (is_list(L) andalso L =/= [] andalso ?IS_STRING(hd(L)))). -define(IS_STRING_LIST(L), (is_list(L) andalso L =/= [] andalso ?IS_STRING(hd(L)))).
%% The ciphers that ssl:cipher_suites(exclusive, 'tlsv1.3', openssl)
%% should return when running on otp 23.
%% But we still have to hard-code them because tlsv1.3 on otp 22 is
%% not trustworthy.
-define(TLSV13_EXCLUSIVE_CIPHERS, [
"TLS_AES_256_GCM_SHA384",
"TLS_AES_128_GCM_SHA256",
"TLS_CHACHA20_POLY1305_SHA256",
"TLS_AES_128_CCM_SHA256",
"TLS_AES_128_CCM_8_SHA256"
]).
-define(SELECTED_CIPHERS, [ -define(SELECTED_CIPHERS, [
"ECDHE-ECDSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-RSA-AES256-GCM-SHA384",
@ -162,7 +150,7 @@ all_ciphers(['tlsv1.3']) ->
%% because 'all' returns legacy cipher suites too, %% because 'all' returns legacy cipher suites too,
%% which does not make sense since tlsv1.3 can not use %% which does not make sense since tlsv1.3 can not use
%% legacy cipher suites. %% legacy cipher suites.
?TLSV13_EXCLUSIVE_CIPHERS; ssl:cipher_suites(exclusive, 'tlsv1.3', openssl);
all_ciphers(Versions) -> all_ciphers(Versions) ->
%% assert non-empty %% assert non-empty
List = lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions]), List = lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions]),
@ -197,7 +185,7 @@ selected_ciphers(Vsns) ->
do_selected_ciphers('tlsv1.3') -> do_selected_ciphers('tlsv1.3') ->
case lists:member('tlsv1.3', proplists:get_value(available, ssl:versions())) of case lists:member('tlsv1.3', proplists:get_value(available, ssl:versions())) of
true -> ?TLSV13_EXCLUSIVE_CIPHERS; true -> ssl:cipher_suites(exclusive, 'tlsv1.3', openssl);
false -> [] false -> []
end ++ do_selected_ciphers('tlsv1.2'); end ++ do_selected_ciphers('tlsv1.2');
do_selected_ciphers(_) -> do_selected_ciphers(_) ->

8
apps/emqx/test/emqx_common_test_helpers.erl
View File

@ -16,8 +16,6 @@
-module(emqx_common_test_helpers). -module(emqx_common_test_helpers).
-include_lib("common_test/include/ct.hrl").
-type special_config_handler() :: fun(). -type special_config_handler() :: fun().
-type apps() :: list(atom()). -type apps() :: list(atom()).
@ -78,12 +76,6 @@
-define(CERTS_PATH(CertName), filename:join(["etc", "certs", CertName])). -define(CERTS_PATH(CertName), filename:join(["etc", "certs", CertName])).
-define(MQTT_SSL_TWOWAY, [
{cacertfile, ?CERTS_PATH("cacert.pem")},
{verify, verify_peer},
{fail_if_no_peer_cert, true}
]).
-define(MQTT_SSL_CLIENT_CERTS, [ -define(MQTT_SSL_CLIENT_CERTS, [
{keyfile, ?CERTS_PATH("client-key.pem")}, {keyfile, ?CERTS_PATH("client-key.pem")},
{cacertfile, ?CERTS_PATH("cacert.pem")}, {cacertfile, ?CERTS_PATH("cacert.pem")},

1
apps/emqx/test/emqx_router_helper_SUITE.erl
View File

@ -19,7 +19,6 @@
-compile(export_all). -compile(export_all).
-compile(nowarn_export_all). -compile(nowarn_export_all).
-include("emqx.hrl").
-include_lib("eunit/include/eunit.hrl"). -include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl").

2
apps/emqx/test/emqx_shared_sub_SUITE.erl
View File

@ -21,8 +21,6 @@
-include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx.hrl").
-include_lib("eunit/include/eunit.hrl"). -include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-define(SUITE, ?MODULE). -define(SUITE, ?MODULE).

3
apps/emqx_conf/test/emqx_conf_app_SUITE.erl
View File

@ -19,10 +19,7 @@
-compile(export_all). -compile(export_all).
-compile(nowarn_export_all). -compile(nowarn_export_all).
-include("emqx_conf.hrl").
-include_lib("eunit/include/eunit.hrl"). -include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
all() -> all() ->
emqx_common_test_helpers:all(?MODULE). emqx_common_test_helpers:all(?MODULE).

27
apps/emqx_connector/src/emqx_connector_jwt_worker.erl
View File

@ -154,8 +154,8 @@ handle_info({timeout, TRef, ?refresh_jwt}, State0 = #{refresh_timer := TRef}) ->
handle_info(_Msg, State) -> handle_info(_Msg, State) ->
{noreply, State}. {noreply, State}.
format_status(State) -> format_status(Status = #{state := State}) ->
censor_secrets(State). Status#{state => censor_secrets(State)}.
format_status(_Opt, [_PDict, State0]) -> format_status(_Opt, [_PDict, State0]) ->
State = censor_secrets(State0), State = censor_secrets(State0),
@ -222,16 +222,13 @@ ensure_timer(State) ->
State. State.
-spec censor_secrets(state()) -> map(). -spec censor_secrets(state()) -> map().
censor_secrets(State) -> censor_secrets(State = #{jwt := JWT, jwk := JWK}) ->
maps:map( State#{
fun jwt := censor_secret(JWT),
(Key, _Value) when jwk := censor_secret(JWK)
Key =:= jwt; }.
Key =:= jwk
-> censor_secret(undefined) ->
"******"; undefined;
(_Key, Value) -> censor_secret(_Secret) ->
Value "******".
end,
State
).

17
apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl
View File

@ -209,20 +209,27 @@ t_log_file(_Config) ->
?assertMatch([#{<<"mtime">> := _, <<"size">> := _, <<"node">> := _}], json(Detail)), ?assertMatch([#{<<"mtime">> := _, <<"size">> := _, <<"node">> := _}], json(Detail)),
{ok, Binary} = request_api(get, api_path("trace/test_client_id/download"), Header), {ok, Binary} = request_api(get, api_path("trace/test_client_id/download"), Header),
{ok, [ {ok, [
_Comment, Comment,
#zip_file{ #zip_file{
name = ZipName, name = ZipName,
info = #file_info{size = Size, type = regular, access = read_write} info = #file_info{size = Size, type = regular, access = read_write}
} }
]} = ]} = zip:table(Binary),
ZipTab =
zip:table(Binary),
?assert(Size > 0), ?assert(Size > 0),
ZipNamePrefix = lists:flatten(io_lib:format("~s-trace_~s", [node(), Name])), ZipNamePrefix = lists:flatten(io_lib:format("~s-trace_~s", [node(), Name])),
?assertNotEqual(nomatch, re:run(ZipName, [ZipNamePrefix])), ?assertNotEqual(nomatch, re:run(ZipName, [ZipNamePrefix])),
Path = api_path("trace/test_client_id/download?node=" ++ atom_to_list(node())), Path = api_path("trace/test_client_id/download?node=" ++ atom_to_list(node())),
{ok, Binary2} = request_api(get, Path, Header), {ok, Binary2} = request_api(get, Path, Header),
?assertEqual(ZipTab, zip:table(Binary2)), ?assertMatch(
{ok, [
Comment,
#zip_file{
name = ZipName,
info = #file_info{size = Size, type = regular, access = read_write}
}
]},
zip:table(Binary2)
),
{error, {_, 400, _}, _} = {error, {_, 400, _}, _} =
request_api( request_api(
get, get,

2
apps/emqx_plugins/test/emqx_plugins_SUITE.erl
View File

@ -22,7 +22,7 @@
-include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx.hrl").
-include_lib("eunit/include/eunit.hrl"). -include_lib("eunit/include/eunit.hrl").
-define(EMQX_PLUGIN_TEMPLATE_VSN, "5.0.0-rc.3"). -define(EMQX_PLUGIN_TEMPLATE_VSN, "5.0.0").
-define(EMQX_ELIXIR_PLUGIN_TEMPLATE_VSN, "0.1.0"). -define(EMQX_ELIXIR_PLUGIN_TEMPLATE_VSN, "0.1.0").
-define(PACKAGE_SUFFIX, ".tar.gz"). -define(PACKAGE_SUFFIX, ".tar.gz").

4
apps/emqx_resource/rebar.config
View File

@ -10,10 +10,6 @@
{extra_src_dirs, ["examples"]}. {extra_src_dirs, ["examples"]}.
%% try to override the dialyzer 'race_conditions' defined in the top-level dir,
%% but it doesn't work
{dialyzer, [{warnings, [unmatched_returns, error_handling]}]}.
{deps, [ {deps, [
{jsx, {git, "https://github.com/talentdeficit/jsx", {tag, "v3.1.0"}}}, {jsx, {git, "https://github.com/talentdeficit/jsx", {tag, "v3.1.0"}}},
{emqx, {path, "../emqx"}} {emqx, {path, "../emqx"}}

2
deploy/docker/Dockerfile
View File

@ -1,4 +1,4 @@
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-debian11 ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11
ARG RUN_FROM=debian:11-slim ARG RUN_FROM=debian:11-slim
FROM ${BUILD_FROM} AS builder FROM ${BUILD_FROM} AS builder

2
deploy/docker/Dockerfile.alpine
View File

@ -1,4 +1,4 @@
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-15:1.13.3-24.2.1-1-alpine3.15.1 ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-alpine3.15.1
ARG RUN_FROM=alpine:3.15.1 ARG RUN_FROM=alpine:3.15.1
FROM ${BUILD_FROM} AS builder FROM ${BUILD_FROM} AS builder

3
mix.exs
View File

@ -45,13 +45,14 @@ defmodule EMQXUmbrella.MixProject do
# other exact versions, and not ranges. # other exact versions, and not ranges.
[ [
{:lc, github: "emqx/lc", tag: "0.3.2", override: true}, {:lc, github: "emqx/lc", tag: "0.3.2", override: true},
{:redbug, "2.0.7"}, {:redbug, "2.0.8"},
{:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true}, {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true},
{:ehttpc, github: "emqx/ehttpc", tag: "0.4.2", override: true}, {:ehttpc, github: "emqx/ehttpc", tag: "0.4.2", override: true},
{:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true},
{:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true},
{:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true},
{:esockd, github: "emqx/esockd", tag: "5.9.4", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.4", override: true},
{:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.7.2-emqx-7", override: true},
{:ekka, github: "emqx/ekka", tag: "0.13.7", override: true}, {:ekka, github: "emqx/ekka", tag: "0.13.7", override: true},
{:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true},
{:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true},

3
rebar.config
View File

@ -45,7 +45,7 @@
{deps, {deps,
[ {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}} [ {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}}
, {redbug, "2.0.7"} , {redbug, "2.0.8"}
, {gpb, "4.19.5"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps , {gpb, "4.19.5"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps
, {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}} , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}}
, {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.9"}}} , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.9"}}}
@ -54,6 +54,7 @@
, {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}}
, {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-7"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}}
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}
, {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}}

21
rebar.config.erl
View File

@ -15,8 +15,8 @@ do(Dir, CONFIG) ->
end. end.
assert_otp() -> assert_otp() ->
Oldest = 23, Oldest = 24,
Latest = 24, Latest = 25,
OtpRelease = list_to_integer(erlang:system_info(otp_release)), OtpRelease = list_to_integer(erlang:system_info(otp_release)),
case OtpRelease < Oldest orelse OtpRelease > Latest of case OtpRelease < Oldest orelse OtpRelease > Latest of
true -> true ->
@ -119,7 +119,7 @@ is_win32() ->
win32 =:= element(1, os:type()). win32 =:= element(1, os:type()).
project_app_dirs() -> project_app_dirs() ->
project_app_dirs(get_edition_from_profille_env()). project_app_dirs(get_edition_from_profile_env()).
project_app_dirs(Edition) -> project_app_dirs(Edition) ->
["apps/*"] ++ ["apps/*"] ++
@ -154,14 +154,14 @@ test_deps() ->
{erl_csv, "0.2.0"} {erl_csv, "0.2.0"}
]. ].
common_compile_opts(Vsn) -> common_compile_opts() ->
common_compile_opts(get_edition_from_profille_env(), Vsn). common_compile_opts(get_edition_from_profile_env(), undefined).
common_compile_opts(Edition, Vsn) -> common_compile_opts(Edition, Vsn) ->
% always include debug_info % always include debug_info
[ [
debug_info, debug_info,
{compile_info, [{emqx_vsn, Vsn}]}, {compile_info, [{emqx_vsn, Vsn} || Vsn /= undefined]},
{d, 'EMQX_RELEASE_EDITION', Edition} {d, 'EMQX_RELEASE_EDITION', Edition}
] ++ ] ++
[{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1"] ++ [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1"] ++
@ -180,7 +180,7 @@ warn_profile_env() ->
end. end.
%% this function is only used for test/check profiles %% this function is only used for test/check profiles
get_edition_from_profille_env() -> get_edition_from_profile_env() ->
case os:getenv("PROFILE") of case os:getenv("PROFILE") of
"emqx-enterprise" ++ _ -> "emqx-enterprise" ++ _ ->
ee; ee;
@ -243,15 +243,14 @@ profiles_ee() ->
%% EE has more files than CE, always test/check with EE options. %% EE has more files than CE, always test/check with EE options.
profiles_dev() -> profiles_dev() ->
Vsn = get_vsn('emqx-enterprise'),
[ [
{check, [ {check, [
{erl_opts, common_compile_opts(Vsn)}, {erl_opts, common_compile_opts()},
{project_app_dirs, project_app_dirs()} {project_app_dirs, project_app_dirs()}
]}, ]},
{test, [ {test, [
{deps, test_deps()}, {deps, test_deps()},
{erl_opts, common_compile_opts(Vsn) ++ erl_opts_i()}, {erl_opts, common_compile_opts() ++ erl_opts_i()},
{extra_src_dirs, [{"test", [{recursive, true}]}]}, {extra_src_dirs, [{"test", [{recursive, true}]}]},
{project_app_dirs, project_app_dirs()} {project_app_dirs, project_app_dirs()}
]} ]}
@ -270,7 +269,7 @@ relx(Vsn, RelType, PkgType, Edition) ->
{vm_args, false}, {vm_args, false},
{release, {emqx, Vsn}, relx_apps(RelType, Edition)}, {release, {emqx, Vsn}, relx_apps(RelType, Edition)},
{overlay, relx_overlay(RelType, Edition)}, {overlay, relx_overlay(RelType, Edition)},
{overlay_vars, {overlay_vars_values,
build_info() ++ build_info() ++
[ [
{emqx_description, emqx_description(RelType, Edition)} {emqx_description, emqx_description(RelType, Edition)}

6
scripts/buildx.sh
View File

@ -9,7 +9,7 @@
## example: ## example:
## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \ ## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \
## --builder ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-debian10 ## --builder ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11
set -euo pipefail set -euo pipefail
@ -22,9 +22,9 @@ help() {
echo "--elixir: Specify if the release should be built with Elixir, " echo "--elixir: Specify if the release should be built with Elixir, "
echo " defaults to 'no'." echo " defaults to 'no'."
echo "--arch amd64|arm64: Target arch to build the EMQX package for" echo "--arch amd64|arm64: Target arch to build the EMQX package for"
echo "--src_dir <SRC_DIR>: EMQX source ode in this dir, default to PWD" echo "--src_dir <SRC_DIR>: EMQX source code in this dir, default to PWD"
echo "--builder <BUILDER>: Builder image to pull" echo "--builder <BUILDER>: Builder image to pull"
echo " E.g. ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-debian10" echo " E.g. ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11"
} }
while [ "$#" -gt 0 ]; do while [ "$#" -gt 0 ]; do

4
scripts/ct/run.sh
View File

@ -66,7 +66,7 @@ if [ "${WHICH_APP}" = 'novalue' ]; then
exit 1 exit 1
fi fi
ERLANG_CONTAINER='erlang24' ERLANG_CONTAINER='erlang'
DOCKER_CT_ENVS_FILE="${WHICH_APP}/docker-ct" DOCKER_CT_ENVS_FILE="${WHICH_APP}/docker-ct"
case "${WHICH_APP}" in case "${WHICH_APP}" in
@ -89,7 +89,7 @@ FILES=( )
for dep in ${CT_DEPS}; do for dep in ${CT_DEPS}; do
case "${dep}" in case "${dep}" in
erlang24) erlang)
FILES+=( '.ci/docker-compose-file/docker-compose.yaml' ) FILES+=( '.ci/docker-compose-file/docker-compose.yaml' )
;; ;;
toxiproxy) toxiproxy)

2
scripts/elvis-check.sh
View File

@ -5,7 +5,7 @@
set -euo pipefail set -euo pipefail
elvis_version='1.0.0-emqx-2' elvis_version='1.1.0-emqx-2'
base="${1:-}" base="${1:-}"
repo="${2:-emqx/emqx}" repo="${2:-emqx/emqx}"

5
scripts/ensure-rebar3.sh
View File

@ -2,7 +2,7 @@
set -euo pipefail set -euo pipefail
## rebar3 tag 3.18.0-emqx-1 is compiled using otp24.1.5. ## rebar3 tag 3.19.0-emqx-1 is compiled using latest official OTP-24 image.
## we have to use an otp24-compiled rebar3 because the defination of record #application{} ## we have to use an otp24-compiled rebar3 because the defination of record #application{}
## in systools.hrl is changed in otp24. ## in systools.hrl is changed in otp24.
OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}"
@ -13,6 +13,9 @@ case ${OTP_VSN} in
24*) 24*)
VERSION="3.18.0-emqx-1" VERSION="3.18.0-emqx-1"
;; ;;
25*)
VERSION="3.19.0-emqx-1"
;;
*) *)
echo "Unsupporetd Erlang/OTP version $OTP_VSN" echo "Unsupporetd Erlang/OTP version $OTP_VSN"
exit 1 exit 1

52
scripts/find-apps.sh
View File

@ -8,10 +8,10 @@ cd -P -- "$(dirname -- "$0")/.."
help() { help() {
echo echo
echo "-h|--help: To display this usage info" echo "-h|--help: To display this usage info"
echo "--ci fast|docker: Print apps in json format for github ci mtrix" echo "--ci: Print apps in json format for github ci matrix"
} }
CI='novalue' MODE='list'
while [ "$#" -gt 0 ]; do while [ "$#" -gt 0 ]; do
case $1 in case $1 in
-h|--help) -h|--help)
@ -19,8 +19,8 @@ while [ "$#" -gt 0 ]; do
exit 0 exit 0
;; ;;
--ci) --ci)
CI="$2" MODE='ci'
shift 2 shift 1
;; ;;
*) *)
echo "unknown option $1" echo "unknown option $1"
@ -45,7 +45,7 @@ CE="$(find_app 'apps')"
EE="$(find_app 'lib-ee')" EE="$(find_app 'lib-ee')"
APPS_ALL="$(echo -e "${CE}\n${EE}")" APPS_ALL="$(echo -e "${CE}\n${EE}")"
if [ "$CI" = 'novalue' ]; then if [ "$MODE" = 'list' ]; then
echo "${APPS_ALL}" echo "${APPS_ALL}"
exit 0 exit 0
fi fi
@ -54,16 +54,22 @@ fi
###### now deal with the github action's matrix. ###### now deal with the github action's matrix.
################################################## ##################################################
dimensions() { format_app_description() {
## prefix is for github actions (they don't like slash in variables)
local prefix=${1//\//_}
echo -n -e "$(
cat <<END
{"app": "${1}", "profile": "${2}", "runner": "${3}", "prefix": "${prefix}"}
END
)"
}
describe_app() {
app="$1" app="$1"
local runner="host"
local profile
if [ -f "${app}/docker-ct" ]; then if [ -f "${app}/docker-ct" ]; then
if [[ "$CI" != 'docker' ]]; then runner="docker"
return
fi
else
if [[ "$CI" != 'fast' ]]; then
return
fi
fi fi
case "${app}" in case "${app}" in
apps/*) apps/*)
@ -77,28 +83,20 @@ dimensions() {
exit 1 exit 1
;; ;;
esac esac
## poor-man's json formatter format_app_description "$app" "$profile" "$runner"
## apps/<app name>, <profile>, apps_<app name>
## third one is for github actions (they don't like slash in variables)
echo -n -e "[\"$app\", \"$profile\", \"${app//\//_}\"]"
} }
matrix() { matrix() {
first_row='yes' local sep='['
for app in ${APPS_ALL}; do for app in ${APPS_ALL}; do
row="$(dimensions "$app")" row="$(describe_app "$app")"
if [ -z "$row" ]; then if [ -z "$row" ]; then
continue continue
fi fi
if [ "$first_row" = 'yes' ]; then echo -n "${sep}${row}"
first_row='no' sep=', '
echo -n "$row"
else
echo -n ",${row}"
fi
done done
echo ']'
} }
echo -n '['
matrix matrix
echo ']'

4
scripts/relup-test/run-relup-lux.sh
View File

@ -45,8 +45,8 @@ fi
# From now on, no need for the v|e prefix # From now on, no need for the v|e prefix
OLD_VSN="${old_vsn#[e|v]}" OLD_VSN="${old_vsn#[e|v]}"
OLD_PKG="$(pwd)/_upgrade_base/${profile}-${OLD_VSN}-otp24.2.1-1-ubuntu20.04-amd64.tar.gz" OLD_PKG="$(pwd)/_upgrade_base/${profile}-${OLD_VSN}-otp24.3.4.2-1-ubuntu20.04-amd64.tar.gz"
CUR_PKG="$(pwd)/_packages/${profile}/${profile}-${cur_vsn}-otp24.2.1-1-ubuntu20.04-amd64.tar.gz" CUR_PKG="$(pwd)/_packages/${profile}/${profile}-${cur_vsn}-otp24.3.4.2-1-ubuntu20.04-amd64.tar.gz"
if [ ! -f "$OLD_PKG" ]; then if [ ! -f "$OLD_PKG" ]; then
echo "$OLD_PKG not found" echo "$OLD_PKG not found"

2
scripts/relup-test/start-relup-test-cluster.sh
View File

@ -22,7 +22,7 @@ WEBHOOK="webhook.$NET"
BENCH="bench.$NET" BENCH="bench.$NET"
COOKIE='this-is-a-secret' COOKIE='this-is-a-secret'
## Erlang image is needed to run webhook server and emqtt-bench ## Erlang image is needed to run webhook server and emqtt-bench
ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04"
# builder has emqtt-bench installed # builder has emqtt-bench installed
BENCH_IMAGE="$ERLANG_IMAGE" BENCH_IMAGE="$ERLANG_IMAGE"