yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

build: add env.sh to manage builder version in one place

This commit is contained in:
Ivan Dyachkov 2024-06-28 12:17:23 +02:00
parent 2a9c27d206
commit 6db1ed9e82
19 changed files with 325 additions and 360 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.github/actions/prepare-jmeter/action.yaml vendored
View File

@ -1,24 +1,8 @@
name: 'Prepare jmeter'
inputs:
version-emqx:
required: true
type: string
runs:
using: composite
steps:
- uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
with:
name: emqx-docker
path: /tmp
- name: load docker image
shell: bash
env:
PKG_VSN: ${{ inputs.version-emqx }}
run: |
EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g')
echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
with:
repository: emqx/emqx-fvt

90
.github/workflows/_pr_entrypoint.yaml vendored
View File

@ -11,23 +11,42 @@ on:
ref:
required: false
defaults:
run:
shell: bash
env:
IS_CI: "yes"
jobs:
init:
runs-on: ubuntu-22.04
outputs:
BUILDER_VSN: ${{ steps.env.outputs.BUILDER_VSN }}
OTP_VSN: ${{ steps.env.outputs.OTP_VSN }}
ELIXIR_VSN: ${{ steps.env.outputs.ELIXIR_VSN }}
BUILDER: ${{ steps.env.outputs.BUILDER }}
steps:
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
with:
ref: ${{ github.event.inputs.ref }}
- name: Set up environment
id: env
run: |
source ./env.sh
echo "BUILDER_VSN=$EMQX_BUILDER_VSN" | tee -a "$GITHUB_OUTPUT"
echo "OTP_VSN=$OTP_VSN" | tee -a "$GITHUB_OUTPUT"
echo "ELIXIR_VSN=$ELIXIR_VSN" | tee -a "$GITHUB_OUTPUT"
echo "BUILDER=$EMQX_BUILDER" | tee -a "$GITHUB_OUTPUT"
sanity-checks:
runs-on: ubuntu-22.04
container: "ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-ubuntu22.04"
needs: init
container: ${{ needs.init.outputs.BUILDER }}
outputs:
ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
ct-host: ${{ steps.matrix.outputs.ct-host }}
ct-docker: ${{ steps.matrix.outputs.ct-docker }}
version-emqx: ${{ steps.matrix.outputs.version-emqx }}
version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }}
builder: "ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-ubuntu22.04"
builder_vsn: "5.3-9"
otp_vsn: "26.2.5-3"
elixir_vsn: "1.15.7"
permissions:
contents: read
@ -92,35 +111,20 @@ jobs:
- name: Generate CT Matrix
id: matrix
run: |
APPS="$(./scripts/find-apps.sh --ci)"
MATRIX="$(echo "${APPS}" | jq -c '
[
(.[] | select(.profile == "emqx") | . + {
builder: "5.3-9",
otp: "26.2.5-3",
elixir: "1.15.7"
}),
(.[] | select(.profile == "emqx-enterprise") | . + {
builder: "5.3-9",
otp: ["26.2.5-3"][],
elixir: "1.15.7"
})
]
')"
MATRIX="$(./scripts/find-apps.sh --ci)"
echo "${MATRIX}" | jq
CT_MATRIX="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')"
CT_MATRIX="$(echo "${MATRIX}" | jq -c 'map({profile}) | unique')"
CT_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')"
CT_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')"
echo "ct-matrix=${CT_MATRIX}" | tee -a $GITHUB_OUTPUT
echo "ct-host=${CT_HOST}" | tee -a $GITHUB_OUTPUT
echo "ct-docker=${CT_DOCKER}" | tee -a $GITHUB_OUTPUT
echo "version-emqx=$(./pkg-vsn.sh emqx)" | tee -a $GITHUB_OUTPUT
echo "version-emqx-enterprise=$(./pkg-vsn.sh emqx-enterprise)" | tee -a $GITHUB_OUTPUT
compile:
runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral-xl","linux","x64"]') }}
container: ${{ needs.sanity-checks.outputs.builder }}
container: ${{ needs.init.outputs.BUILDER }}
needs:
- init
- sanity-checks
strategy:
matrix:
@ -156,53 +160,47 @@ jobs:
run_emqx_app_tests:
needs:
- init
- sanity-checks
- compile
uses: ./.github/workflows/run_emqx_app_tests.yaml
with:
builder: ${{ needs.sanity-checks.outputs.builder }}
builder: ${{ needs.init.outputs.BUILDER }}
before_ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
after_ref: ${{ github.sha }}
run_test_cases:
needs:
- init
- sanity-checks
- compile
uses: ./.github/workflows/run_test_cases.yaml
with:
builder: ${{ needs.sanity-checks.outputs.builder }}
builder: ${{ needs.init.outputs.BUILDER }}
ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }}
ct-host: ${{ needs.sanity-checks.outputs.ct-host }}
ct-docker: ${{ needs.sanity-checks.outputs.ct-docker }}
static_checks:
needs:
- init
- sanity-checks
- compile
uses: ./.github/workflows/static_checks.yaml
with:
builder: ${{ needs.sanity-checks.outputs.builder }}
builder: ${{ needs.init.outputs.BUILDER }}
ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }}
build_slim_packages:
needs:
- sanity-checks
uses: ./.github/workflows/build_slim_packages.yaml
with:
builder: ${{ needs.sanity-checks.outputs.builder }}
builder_vsn: ${{ needs.sanity-checks.outputs.builder_vsn }}
otp_vsn: ${{ needs.sanity-checks.outputs.otp_vsn }}
elixir_vsn: ${{ needs.sanity-checks.outputs.elixir_vsn }}
build_docker_for_test:
needs:
- init
- sanity-checks
uses: ./.github/workflows/build_docker_for_test.yaml
with:
otp_vsn: ${{ needs.sanity-checks.outputs.otp_vsn }}
elixir_vsn: ${{ needs.sanity-checks.outputs.elixir_vsn }}
version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }}
spellcheck:
needs:
@ -212,41 +210,35 @@ jobs:
run_conf_tests:
needs:
- init
- sanity-checks
- compile
uses: ./.github/workflows/run_conf_tests.yaml
with:
builder: ${{ needs.sanity-checks.outputs.builder }}
builder: ${{ needs.init.outputs.BUILDER }}
check_deps_integrity:
needs:
- init
- sanity-checks
uses: ./.github/workflows/check_deps_integrity.yaml
with:
builder: ${{ needs.sanity-checks.outputs.builder }}
builder: ${{ needs.init.outputs.BUILDER }}
run_jmeter_tests:
needs:
- sanity-checks
- build_docker_for_test
uses: ./.github/workflows/run_jmeter_tests.yaml
with:
version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
run_docker_tests:
needs:
- sanity-checks
- build_docker_for_test
uses: ./.github/workflows/run_docker_tests.yaml
with:
version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }}
run_helm_tests:
needs:
- sanity-checks
- build_docker_for_test
uses: ./.github/workflows/run_helm_tests.yaml
with:
version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }}

88
.github/workflows/_push-entrypoint.yaml vendored
View File

@ -17,13 +17,42 @@ on:
ref:
required: false
defaults:
run:
shell: bash
env:
IS_CI: 'yes'
jobs:
init:
runs-on: ubuntu-22.04
outputs:
BUILDER_VSN: ${{ steps.env.outputs.BUILDER_VSN }}
OTP_VSN: ${{ steps.env.outputs.OTP_VSN }}
ELIXIR_VSN: ${{ steps.env.outputs.ELIXIR_VSN }}
BUILDER: ${{ steps.env.outputs.BUILDER }}
BUILD_FROM: ${{ steps.env.outputs.BUILD_FROM }}
RUN_FROM: ${{ steps.env.outputs.BUILD_FROM }}
steps:
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
with:
ref: ${{ github.event.inputs.ref }}
- name: Set up environment
id: env
run: |
source env.sh
echo "BUILDER_VSN=$EMQX_BUILDER_VSN" >> "$GITHUB_OUTPUT"
echo "OTP_VSN=$OTP_VSN" >> "$GITHUB_OUTPUT"
echo "ELIXIR_VSN=$ELIXIR_VSN" >> "$GITHUB_OUTPUT"
echo "BUILDER=$EMQX_BUILDER" >> "$GITHUB_OUTPUT"
echo "BUILD_FROM=$EMQX_DOCKER_BUILD_FROM" >> "$GITHUB_OUTPUT"
echo "RUN_FROM=$EMQX_DOCKER_RUN_FROM" >> "$GITHUB_OUTPUT"
prepare:
runs-on: ubuntu-22.04
container: 'ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-ubuntu22.04'
needs: init
container: ${{ needs.init.outputs.BUILDER }}
outputs:
profile: ${{ steps.parse-git-ref.outputs.profile }}
release: ${{ steps.parse-git-ref.outputs.release }}
@ -31,10 +60,6 @@ jobs:
ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
ct-host: ${{ steps.matrix.outputs.ct-host }}
ct-docker: ${{ steps.matrix.outputs.ct-docker }}
builder: 'ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-ubuntu22.04'
builder_vsn: '5.3-9'
otp_vsn: '26.2.5-3'
elixir_vsn: '1.15.7'
permissions:
contents: read
@ -61,23 +86,9 @@ jobs:
- name: Build matrix
id: matrix
run: |
APPS="$(./scripts/find-apps.sh --ci)"
MATRIX="$(echo "${APPS}" | jq -c '
[
(.[] | select(.profile == "emqx") | . + {
builder: "5.3-9",
otp: "26.2.5-3",
elixir: "1.15.7"
}),
(.[] | select(.profile == "emqx-enterprise") | . + {
builder: "5.3-9",
otp: ["26.2.5-3"][],
elixir: "1.15.7"
})
]
')"
MATRIX="$(./scripts/find-apps.sh --ci)"
echo "${MATRIX}" | jq
CT_MATRIX="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')"
CT_MATRIX="$(echo "${MATRIX}" | jq -c 'map({profile}) | unique')"
CT_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')"
CT_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')"
echo "ct-matrix=${CT_MATRIX}" | tee -a $GITHUB_OUTPUT
@ -87,46 +98,44 @@ jobs:
build_packages:
if: needs.prepare.outputs.release == 'true'
needs:
- init
- prepare
uses: ./.github/workflows/build_packages.yaml
with:
profile: ${{ needs.prepare.outputs.profile }}
publish: true
otp_vsn: ${{ needs.prepare.outputs.otp_vsn }}
elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }}
builder_vsn: ${{ needs.prepare.outputs.builder_vsn }}
otp_vsn: ${{ needs.init.outputs.OTP_VSN }}
elixir_vsn: ${{ needs.init.outputs.ELIXIR_VSN }}
builder_vsn: ${{ needs.init.outputs.BUILDER_VSN }}
secrets: inherit
build_and_push_docker_images:
if: needs.prepare.outputs.release == 'true'
needs:
- init
- prepare
uses: ./.github/workflows/build_and_push_docker_images.yaml
with:
profile: ${{ needs.prepare.outputs.profile }}
publish: true
latest: ${{ needs.prepare.outputs.latest }}
otp_vsn: ${{ needs.prepare.outputs.otp_vsn }}
elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }}
builder_vsn: ${{ needs.prepare.outputs.builder_vsn }}
build_from: ${{ needs.init.outputs.BUILD_FROM }}
run_from: ${{ needs.init.outputs.RUN_FROM }}
secrets: inherit
build_slim_packages:
if: needs.prepare.outputs.release != 'true'
needs:
- init
- prepare
uses: ./.github/workflows/build_slim_packages.yaml
with:
builder: ${{ needs.prepare.outputs.builder }}
builder_vsn: ${{ needs.prepare.outputs.builder_vsn }}
otp_vsn: ${{ needs.prepare.outputs.otp_vsn }}
elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }}
compile:
if: needs.prepare.outputs.release != 'true'
runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
container: ${{ needs.prepare.outputs.builder }}
container: ${{ needs.init.outputs.BUILDER }}
needs:
- init
- prepare
strategy:
matrix:
@ -162,22 +171,23 @@ jobs:
run_emqx_app_tests:
needs:
- prepare
- init
- compile
uses: ./.github/workflows/run_emqx_app_tests.yaml
with:
builder: ${{ needs.prepare.outputs.builder }}
builder: ${{ needs.init.outputs.BUILDER }}
before_ref: ${{ github.event.before }}
after_ref: ${{ github.sha }}
run_test_cases:
if: needs.prepare.outputs.release != 'true'
needs:
- init
- prepare
- compile
uses: ./.github/workflows/run_test_cases.yaml
with:
builder: ${{ needs.prepare.outputs.builder }}
builder: ${{ needs.init.outputs.BUILDER }}
ct-matrix: ${{ needs.prepare.outputs.ct-matrix }}
ct-host: ${{ needs.prepare.outputs.ct-host }}
ct-docker: ${{ needs.prepare.outputs.ct-docker }}
@ -185,18 +195,20 @@ jobs:
run_conf_tests:
if: needs.prepare.outputs.release != 'true'
needs:
- init
- prepare
- compile
uses: ./.github/workflows/run_conf_tests.yaml
with:
builder: ${{ needs.prepare.outputs.builder }}
builder: ${{ needs.init.outputs.BUILDER }}
static_checks:
if: needs.prepare.outputs.release != 'true'
needs:
- init
- prepare
- compile
uses: ./.github/workflows/static_checks.yaml
with:
builder: ${{ needs.prepare.outputs.builder }}
builder: ${{ needs.init.outputs.BUILDER }}
ct-matrix: ${{ needs.prepare.outputs.ct-matrix }}

39
.github/workflows/build_and_push_docker_images.yaml vendored
View File

@ -16,13 +16,10 @@ on:
publish:
required: true
type: boolean
otp_vsn:
build_from:
required: true
type: string
elixir_vsn:
required: true
type: string
builder_vsn:
run_from:
required: true
type: string
secrets:
@ -50,18 +47,12 @@ on:
required: false
type: boolean
default: false
otp_vsn:
build_from:
required: false
type: string
default: '26.2.5-3'
elixir_vsn:
required: false
type: string
default: '1.15.7'
builder_vsn:
required: false
type: string
default: '5.3-9'
default: ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-debian12
run_from:
default: public.ecr.aws/debian/debian:stable-20240612-slim
permissions:
contents: read
@ -69,7 +60,7 @@ permissions:
jobs:
build:
runs-on: ${{ github.repository_owner == 'emqx' && fromJSON(format('["self-hosted","ephemeral","linux","{0}"]', matrix.arch)) || 'ubuntu-22.04' }}
container: "ghcr.io/emqx/emqx-builder/${{ inputs.builder_vsn }}:${{ inputs.elixir_vsn }}-${{ inputs.otp_vsn }}-debian12"
container: ${{ inputs.build_from }}
outputs:
PKG_VSN: ${{ steps.build.outputs.PKG_VSN }}
@ -164,14 +155,9 @@ jobs:
DOCKER_LATEST: ${{ inputs.latest }}
DOCKER_PUSH: false
DOCKER_BUILD_NOCACHE: true
DOCKER_PLATFORMS: linux/amd64
DOCKER_LOAD: true
EMQX_RUNNER: 'public.ecr.aws/debian/debian:stable-20240612-slim'
EMQX_DOCKERFILE: 'deploy/docker/Dockerfile'
BUILD_FROM: ${{ inputs.build_from }}
RUN_FROM: ${{ inputs.run_from }}
PKG_VSN: ${{ needs.build.outputs.PKG_VSN }}
EMQX_BUILDER_VERSION: ${{ inputs.builder_vsn }}
OTP_VSN: ${{ inputs.otp_vsn }}
ELIXIR_VSN: ${{ inputs.elixir_vsn }}
EMQX_SOURCE_TYPE: tgz
run: |
./build ${PROFILE} docker
@ -215,12 +201,9 @@ jobs:
DOCKER_BUILD_NOCACHE: false
DOCKER_PLATFORMS: linux/amd64,linux/arm64
DOCKER_LOAD: false
EMQX_RUNNER: 'public.ecr.aws/debian/debian:stable-20240612-slim'
EMQX_DOCKERFILE: 'deploy/docker/Dockerfile'
BUILD_FROM: ${{ inputs.build_from }}
RUN_FROM: ${{ inputs.run_from }}
PKG_VSN: ${{ needs.build.outputs.PKG_VSN }}
EMQX_BUILDER_VERSION: ${{ inputs.builder_vsn }}
OTP_VSN: ${{ inputs.otp_vsn }}
ELIXIR_VSN: ${{ inputs.elixir_vsn }}
EMQX_SOURCE_TYPE: tgz
run: |
./build ${PROFILE} docker

24
.github/workflows/build_docker_for_test.yaml vendored
View File

@ -6,19 +6,6 @@ concurrency:
on:
workflow_call:
inputs:
otp_vsn:
required: true
type: string
elixir_vsn:
required: true
type: string
version-emqx:
required: true
type: string
version-emqx-enterprise:
required: true
type: string
permissions:
contents: read
@ -28,9 +15,6 @@ jobs:
runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
env:
EMQX_NAME: ${{ matrix.profile }}
PKG_VSN: ${{ matrix.profile == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }}
OTP_VSN: ${{ inputs.otp_vsn }}
ELIXIR_VSN: ${{ inputs.elixir_vsn }}
strategy:
fail-fast: false
@ -42,7 +26,13 @@ jobs:
- emqx-enterprise-elixir
steps:
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Set up environment
id: env
run: |
source env.sh
PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh "$EMQX_NAME")
echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
- name: build and export to Docker
id: build
run: |

57
.github/workflows/build_packages_cron.yaml vendored
View File

@ -16,19 +16,22 @@ jobs:
linux:
if: github.repository_owner == 'emqx'
runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
container:
image: "ghcr.io/emqx/emqx-builder/${{ matrix.profile[2] }}-${{ matrix.os }}"
strategy:
fail-fast: false
matrix:
profile:
- ['emqx', 'master', '5.3-9:1.15.7-26.2.5-3']
- ['emqx', 'release-57', '5.3-9:1.15.7-26.2.5-3']
- ['emqx', 'master']
- ['emqx', 'release-57']
os:
- ubuntu22.04
- amzn2023
env:
PROFILE: ${{ matrix.profile[0] }}
OS: ${{ matrix.os }}
BUILDER_SYSTEM: force_docker
defaults:
run:
shell: bash
@ -38,34 +41,19 @@ jobs:
with:
ref: ${{ matrix.profile[1] }}
fetch-depth: 0
- name: fix workdir
- name: Set up environment
id: env
run: |
set -eu
git config --global --add safe.directory "$GITHUB_WORKSPACE"
# Align path for CMake caches
if [ ! "$PWD" = "/emqx" ]; then
ln -s $PWD /emqx
cd /emqx
fi
echo "pwd is $PWD"
- name: build emqx packages
env:
PROFILE: ${{ matrix.profile[0] }}
ACLOCAL_PATH: "/usr/share/aclocal:/usr/local/share/aclocal"
source env.sh
BUILDER="ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VSN}:${ELIXIR_VSN}-${OTP_VSN}-${OS}"
echo "BUILDER=$BUILDER" >> "$GITHUB_ENV"
- name: build tgz
run: |
set -eu
make "${PROFILE}-tgz"
make "${PROFILE}-pkg"
- name: test emqx packages
env:
PROFILE: ${{ matrix.profile[0] }}
./scripts/buildx.sh --profile "$PROFILE" --pkgtype tgz --builder "$BUILDER"
- name: build pkg
run: |
set -eu
./scripts/pkg-tests.sh "${PROFILE}-tgz"
./scripts/pkg-tests.sh "${PROFILE}-pkg"
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
./scripts/buildx.sh --profile "$PROFILE" --pkgtype pkg --builder "$BUILDER"
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
if: success()
with:
name: ${{ matrix.profile[0] }}-${{ matrix.os }}-${{ github.ref_name }}
@ -91,20 +79,23 @@ jobs:
- emqx
branch:
- master
otp:
- 26.2.5-3
os:
- macos-12-arm64
- macos-14-arm64
steps:
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
with:
ref: ${{ matrix.branch }}
fetch-depth: 0
- name: Set up environment
id: env
run: |
source env.sh
echo "OTP_VSN=$OTP_VSN" >> "$GITHUB_OUTPUT"
- uses: ./.github/actions/package-macos
with:
profile: ${{ matrix.profile }}
otp: ${{ matrix.otp }}
otp: ${{ steps.env.outputs.OTP_VSN }}
os: ${{ matrix.os }}
apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}

90
.github/workflows/build_slim_packages.yaml vendored
View File

@ -6,97 +6,50 @@ concurrency:
on:
workflow_call:
inputs:
builder:
required: true
type: string
builder_vsn:
required: true
type: string
otp_vsn:
required: true
type: string
elixir_vsn:
required: true
type: string
workflow_dispatch:
inputs:
ref:
required: false
builder:
required: false
type: string
default: 'ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-ubuntu22.04'
builder_vsn:
required: false
type: string
default: '5.3-9'
otp_vsn:
required: false
type: string
default: '26.2.5-3'
elixir_vsn:
required: false
type: string
default: '1.15.7'
permissions:
contents: read
jobs:
linux:
runs-on: ${{ github.repository_owner == 'emqx' && fromJSON(format('["self-hosted","ephemeral","linux","{0}"]', matrix.profile[4])) || 'ubuntu-22.04' }}
runs-on: ${{ github.repository_owner == 'emqx' && fromJSON(format('["self-hosted","ephemeral","linux","{0}"]', matrix.profile[2])) || 'ubuntu-22.04' }}
env:
EMQX_NAME: ${{ matrix.profile[0] }}
PROFILE: ${{ matrix.profile[0] }}
ELIXIR: ${{ matrix.profile[1] == 'elixir' && 'yes' || 'no' }}
ARCH: ${{ matrix.profile[2] == 'x64' && 'amd64' || 'arm64' }}
BUILDER_SYSTEM: force_docker
strategy:
fail-fast: false
matrix:
profile:
- ["emqx", "26.2.5-3", "ubuntu22.04", "elixir", "x64"]
- ["emqx", "26.2.5-3", "ubuntu22.04", "elixir", "arm64"]
- ["emqx-enterprise", "26.2.5-3", "ubuntu22.04", "erlang", "x64"]
container: "ghcr.io/emqx/emqx-builder/${{ inputs.builder_vsn }}:${{ inputs.elixir_vsn }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"
- ["emqx", "elixir", "x64"]
- ["emqx", "elixir", "arm64"]
- ["emqx-enterprise", "erlang", "x64"]
steps:
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
with:
fetch-depth: 0
- name: Work around https://github.com/actions/checkout/issues/766
- name: build tgz
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
- name: build and test tgz package
if: matrix.profile[3] == 'erlang'
./scripts/buildx.sh --profile $PROFILE --pkgtype tgz --elixir $ELIXIR --arch $ARCH
- name: build pkg
run: |
make ${EMQX_NAME}-tgz
./scripts/pkg-tests.sh ${EMQX_NAME}-tgz
- name: build and test deb/rpm packages
if: matrix.profile[3] == 'erlang'
run: |
make ${EMQX_NAME}-pkg
./scripts/pkg-tests.sh ${EMQX_NAME}-pkg
- name: build and test tgz package (Elixir)
if: matrix.profile[3] == 'elixir'
run: |
make ${EMQX_NAME}-elixir-tgz
./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-tgz
- name: build and test deb/rpm packages (Elixir)
if: matrix.profile[3] == 'elixir'
run: |
make ${EMQX_NAME}-elixir-pkg
./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-pkg
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
./scripts/buildx.sh --profile $PROFILE --pkgtype pkg --elixir $ELIXIR --arch $ARCH
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
with:
name: "${{ matrix.profile[0] }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}-${{ matrix.profile[3] }}-${{ matrix.profile[4] }}"
name: "${{ matrix.profile[0] }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"
path: _packages/${{ matrix.profile[0] }}/*
retention-days: 7
compression-level: 0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: "${{ matrix.profile[0] }}-schema-dump-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}-${{ matrix.profile[3] }}-${{ matrix.profile[4] }}"
name: "${{ matrix.profile[0] }}-schema-dump-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"
path: |
scripts/spellcheck
_build/docgen/${{ matrix.profile[0] }}/schema-en.json
@ -108,21 +61,24 @@ jobs:
matrix:
profile:
- emqx
otp:
- ${{ inputs.otp_vsn }}
os:
- macos-14
- macos-14-arm64
runs-on: ${{ matrix.os }}
env:
EMQX_NAME: ${{ matrix.profile }}
steps:
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Set up environment
id: env
run: |
source env.sh
echo "OTP_VSN=$OTP_VSN" >> "$GITHUB_OUTPUT"
- uses: ./.github/actions/package-macos
with:
profile: ${{ matrix.profile }}
otp: ${{ matrix.otp }}
otp: ${{ steps.env.outputs.OTP_VSN }}
os: ${{ matrix.os }}
apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}

16
.github/workflows/codeql.yaml vendored
View File

@ -17,8 +17,6 @@ jobs:
permissions:
actions: read
security-events: write
container:
image: ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-ubuntu22.04
strategy:
fail-fast: false
@ -36,11 +34,6 @@ jobs:
with:
ref: ${{ matrix.branch }}
- name: Ensure git safe dir
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
make ensure-rebar3
- name: Initialize CodeQL
uses: github/codeql-action/init@7e187e1c529d80bac7b87a16e7a792427f65cf02 # v2.15.5
with:
@ -51,14 +44,7 @@ jobs:
env:
PROFILE: emqx-enterprise
run: |
make emqx-enterprise-compile
- name: Fetch deps
if: matrix.language == 'python'
env:
PROFILE: emqx-enterprise
run: |
make deps-emqx-enterprise
./scripts/buildx.sh --profile emqx-enterprise --pkgtype rel
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@7e187e1c529d80bac7b87a16e7a792427f65cf02 # v2.15.5

29
.github/workflows/run_docker_tests.yaml vendored
View File

@ -6,13 +6,6 @@ concurrency:
on:
workflow_call:
inputs:
version-emqx:
required: true
type: string
version-emqx-enterprise:
required: true
type: string
permissions:
contents: read
@ -32,12 +25,17 @@ jobs:
env:
EMQX_NAME: ${{ matrix.profile[0] }}
PKG_VSN: ${{ matrix.profile[0] == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }}
EMQX_IMAGE_OLD_VERSION_TAG: ${{ matrix.profile[1] }}
steps:
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Set up environment
id: env
run: |
source env.sh
PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh "$EMQX_NAME")
echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
with:
name: ${{ env.EMQX_NAME }}-docker
path: /tmp
@ -71,7 +69,6 @@ jobs:
shell: bash
env:
EMQX_NAME: ${{ matrix.profile }}
PKG_VSN: ${{ matrix.profile == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }}
_EMQX_TEST_DB_BACKEND: ${{ matrix.cluster_db_backend }}
strategy:
@ -85,8 +82,14 @@ jobs:
- mnesia
- rlog
steps:
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Set up environment
id: env
run: |
source env.sh
PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh "$EMQX_NAME")
echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
with:
name: ${{ env.EMQX_NAME }}-docker
path: /tmp

17
.github/workflows/run_helm_tests.yaml vendored
View File

@ -6,13 +6,6 @@ concurrency:
on:
workflow_call:
inputs:
version-emqx:
required: true
type: string
version-emqx-enterprise:
required: true
type: string
permissions:
contents: read
@ -25,7 +18,6 @@ jobs:
shell: bash
env:
EMQX_NAME: ${{ matrix.profile }}
EMQX_TAG: ${{ matrix.profile == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }}
REPOSITORY: "emqx/${{ matrix.profile }}"
strategy:
@ -45,7 +37,14 @@ jobs:
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
with:
path: source
- uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
- name: Set up environment
id: env
run: |
cd source
source env.sh
PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh "$EMQX_NAME")
echo "EMQX_TAG=$PKG_VSN" >> "$GITHUB_ENV"
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
with:
name: "${{ env.EMQX_NAME }}-docker"
path: /tmp

99
.github/workflows/run_jmeter_tests.yaml vendored
View File

@ -2,10 +2,6 @@ name: JMeter integration tests
on:
workflow_call:
inputs:
version-emqx:
required: true
type: string
permissions:
contents: read
@ -55,10 +51,23 @@ jobs:
needs: jmeter_artifact
steps:
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- uses: ./.github/actions/prepare-jmeter
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Set up environment
id: env
run: |
source env.sh
PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
- uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
with:
version-emqx: ${{ inputs.version-emqx }}
name: emqx-docker
path: /tmp
- name: load docker image
shell: bash
run: |
EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g')
echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV
- uses: ./.github/actions/prepare-jmeter
- name: docker compose up
timeout-minutes: 5
run: |
@ -111,10 +120,23 @@ jobs:
needs: jmeter_artifact
steps:
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- uses: ./.github/actions/prepare-jmeter
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Set up environment
id: env
run: |
source env.sh
PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
- uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
with:
version-emqx: ${{ inputs.version-emqx }}
name: emqx-docker
path: /tmp
- name: load docker image
shell: bash
run: |
EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g')
echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV
- uses: ./.github/actions/prepare-jmeter
- name: docker compose up
timeout-minutes: 5
env:
@ -175,10 +197,23 @@ jobs:
needs: jmeter_artifact
steps:
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- uses: ./.github/actions/prepare-jmeter
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Set up environment
id: env
run: |
source env.sh
PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
- uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
with:
version-emqx: ${{ inputs.version-emqx }}
name: emqx-docker
path: /tmp
- name: load docker image
shell: bash
run: |
EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g')
echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV
- uses: ./.github/actions/prepare-jmeter
- name: docker compose up
timeout-minutes: 5
env:
@ -231,10 +266,23 @@ jobs:
needs: jmeter_artifact
steps:
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- uses: ./.github/actions/prepare-jmeter
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Set up environment
id: env
run: |
source env.sh
PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
- uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
with:
version-emqx: ${{ inputs.version-emqx }}
name: emqx-docker
path: /tmp
- name: load docker image
shell: bash
run: |
EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g')
echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV
- uses: ./.github/actions/prepare-jmeter
- name: docker compose up
timeout-minutes: 5
run: |
@ -284,10 +332,23 @@ jobs:
needs: jmeter_artifact
steps:
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- uses: ./.github/actions/prepare-jmeter
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Set up environment
id: env
run: |
source env.sh
PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
- uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
with:
version-emqx: ${{ inputs.version-emqx }}
name: emqx-docker
path: /tmp
- name: load docker image
shell: bash
run: |
EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g')
echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV
- uses: ./.github/actions/prepare-jmeter
- name: docker compose up
timeout-minutes: 5
run: |

16
.github/workflows/run_test_cases.yaml vendored
View File

@ -35,12 +35,12 @@ jobs:
defaults:
run:
shell: bash
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
container: ${{ inputs.builder }}
env:
PROFILE: ${{ matrix.profile }}
ENABLE_COVER_COMPILE: 1
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}
permissions:
contents: read
@ -100,7 +100,7 @@ jobs:
# produces $PROFILE-<app-name>-<otp-vsn>-sg<suitegroup>.coverdata
- name: run common tests
env:
DOCKER_CT_RUNNER_IMAGE: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
DOCKER_CT_RUNNER_IMAGE: ${{ inputs.builder }}
MONGO_TAG: "5"
MYSQL_TAG: "8"
PGSQL_TAG: "13"
@ -111,7 +111,7 @@ jobs:
MINIO_TAG: "RELEASE.2023-03-20T20-16-18Z"
SUITEGROUP: ${{ matrix.suitegroup }}
ENABLE_COVER_COMPILE: 1
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-sg${{ matrix.suitegroup }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }} --keep-up
@ -136,7 +136,7 @@ jobs:
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: failure()
with:
name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-sg${{ matrix.suitegroup }}
path: logs.tar.gz
compression-level: 0
retention-days: 7
@ -149,7 +149,7 @@ jobs:
matrix:
include: ${{ fromJson(inputs.ct-host) }}
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
container: ${{ inputs.builder }}
defaults:
run:
shell: bash
@ -161,7 +161,7 @@ jobs:
PROFILE: ${{ matrix.profile }}
SUITEGROUP: ${{ matrix.suitegroup }}
ENABLE_COVER_COMPILE: 1
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-sg${{ matrix.suitegroup }}
steps:
- uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
@ -196,7 +196,7 @@ jobs:
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: failure()
with:
name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-sg${{ matrix.suitegroup }}
path: logs.tar.gz
compression-level: 0
retention-days: 7

8
.github/workflows/static_checks.yaml vendored
View File

@ -28,7 +28,7 @@ jobs:
fail-fast: false
matrix:
include: ${{ fromJson(inputs.ct-matrix) }}
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
container: "${{ inputs.builder }}"
steps:
- uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
with:
@ -39,10 +39,10 @@ jobs:
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1
with:
path: "emqx_dialyzer_${{ matrix.otp }}_plt"
key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-${{ hashFiles('rebar.*', 'apps/*/rebar.*') }}
path: "emqx_dialyzer_${{ matrix.profile }}_plt"
key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ hashFiles('rebar.*', 'apps/*/rebar.*') }}
restore-keys: |
rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-
rebar3-dialyzer-plt-${{ matrix.profile }}-
- run: cat .env | tee -a $GITHUB_ENV
- name: run static checks
run: make static_checks

19
Makefile
View File

@ -6,23 +6,16 @@ endif
REBAR = $(CURDIR)/rebar3
BUILD = $(CURDIR)/build
SCRIPTS = $(CURDIR)/scripts
export EMQX_RELUP ?= true
export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-debian12
export EMQX_DEFAULT_RUNNER = public.ecr.aws/debian/debian:stable-20240612-slim
export EMQX_REL_FORM ?= tgz
export QUICER_DOWNLOAD_FROM_RELEASE = 1
ifeq ($(OS),Windows_NT)
export REBAR_COLOR=none
FIND=/usr/bin/find
else
FIND=find
endif
include env.sh
# Dashboard version
# from https://github.com/emqx/emqx-dashboard5
export EMQX_DASHBOARD_VERSION ?= v1.9.1
export EMQX_EE_DASHBOARD_VERSION ?= e1.7.1
export EMQX_RELUP ?= true
export EMQX_REL_FORM ?= tgz
-include default-profile.mk
PROFILE ?= emqx
REL_PROFILES := emqx emqx-enterprise
@ -196,8 +189,8 @@ $(PROFILES:%=clean-%):
@if [ -d _build/$(@:clean-%=%) ]; then \
rm -f rebar.lock; \
rm -rf _build/$(@:clean-%=%)/rel; \
$(FIND) _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
$(FIND) _build/$(@:clean-%=%) -type l -delete; \
find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
find _build/$(@:clean-%=%) -type l -delete; \
fi
.PHONY: clean-all

18
build
View File

@ -397,12 +397,14 @@ function is_ecr_and_enterprise() {
## Build the default docker image based on debian 12.
make_docker() {
local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.3-9}"
local EMQX_BUILDER_PLATFORM="${EMQX_BUILDER_PLATFORM:-debian12}"
local OTP_VSN="${OTP_VSN:-26.2.5-3}"
local ELIXIR_VSN="${ELIXIR_VSN:-1.15.7}"
local EMQX_BUILDER=${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VERSION}:${ELIXIR_VSN}-${OTP_VSN}-${EMQX_BUILDER_PLATFORM}}
local EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}"
# shellcheck disable=SC1091
source ./env.sh
local BUILD_FROM="${BUILD_FROM:-${EMQX_DOCKER_BUILD_FROM}}"
# shellcheck disable=SC2155
local OTP_VSN="$(docker run --rm "${BUILD_FROM}" erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell)"
# shellcheck disable=SC2155
local ELIXIR_VSN="$(docker run --rm "${BUILD_FROM}" elixir --short-version)"
local RUN_FROM="${RUN_FROM:-${EMQX_DOCKER_RUN_FROM}}"
local EMQX_DOCKERFILE="${EMQX_DOCKERFILE:-deploy/docker/Dockerfile}"
local EMQX_SOURCE_TYPE="${EMQX_SOURCE_TYPE:-src}"
# shellcheck disable=SC2155
@ -446,8 +448,8 @@ make_docker() {
GIT_REVISION="$(git rev-parse HEAD)"
export BUILDX_NO_DEFAULT_ATTESTATIONS=1
local DOCKER_BUILDX_ARGS=(
--build-arg BUILD_FROM="${EMQX_BUILDER}" \
--build-arg RUN_FROM="${EMQX_RUNNER}" \
--build-arg BUILD_FROM="${BUILD_FROM}" \
--build-arg RUN_FROM="${RUN_FROM}" \
--build-arg SOURCE_TYPE="${EMQX_SOURCE_TYPE}" \
--build-arg PROFILE="${PROFILE%%-elixir}" \
--build-arg IS_ELIXIR="$([[ "$PROFILE" = *-elixir ]] && echo yes || echo no)" \

8
env.sh Normal file
View File

@ -0,0 +1,8 @@
# https://github.com/emqx/emqx-builder
export EMQX_BUILDER_VSN=5.3-9
export OTP_VSN=26.2.5-3
export ELIXIR_VSN=1.15.7
export EMQX_BUILDER=ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VSN}:${ELIXIR_VSN}-${OTP_VSN}-ubuntu22.04
export EMQX_DOCKER_BUILD_FROM=ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VSN}:${ELIXIR_VSN}-${OTP_VSN}-debian12
export EMQX_DOCKER_RUN_FROM=public.ecr.aws/debian/debian:stable-20240612-slim
export QUICER_DOWNLOAD_FROM_RELEASE=1

39
scripts/buildx.sh
View File

@ -8,8 +8,7 @@
## i.e. will not work if docker command has to be executed with sudo
## example:
## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \
## --builder ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-debian12
## ./scripts/buildx.sh --profile emqx --pkgtype tgz
set -euo pipefail
@ -17,14 +16,14 @@ help() {
echo
echo "-h|--help: To display this usage information"
echo "--profile <PROFILE>: EMQX profile to build (emqx|emqx-enterprise)"
echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz,"
echo " pkg for .rpm or .deb"
echo "--pkgtype tgz|pkg|rel: Specify which package to build, tgz for .tar.gz,"
echo " pkg for .rpm or .deb, rel for release only"
echo "--elixir: Specify if the release should be built with Elixir, "
echo " defaults to 'no'."
echo "--arch amd64|arm64: Target arch to build the EMQX package for"
echo "--src_dir <SRC_DIR>: EMQX source code in this dir, default to PWD"
echo "--builder <BUILDER>: Builder image to pull"
echo " E.g. ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-debian12"
echo "--builder <BUILDER>: Docker image to use for building"
echo " E.g. ghcr.io/emqx/emqx-builder/5.3-8:1.15.7-26.2.5-2-debian12"
}
die() {
@ -34,6 +33,11 @@ die() {
exit 1
}
# ensure dir
cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
# shellcheck disable=SC1091
source ./env.sh
while [ "$#" -gt 0 ]; do
case $1 in
-h|--help)
@ -53,7 +57,7 @@ while [ "$#" -gt 0 ]; do
shift 2
;;
--builder)
BUILDER="$2"
EMQX_BUILDER="$2"
shift 2
;;
--arch)
@ -103,24 +107,21 @@ ARCH="${ARCH:-${NATIVE_ARCH:-}}"
[ -z "${PROFILE:-}" ] && die "missing --profile"
[ -z "${PKGTYPE:-}" ] && die "missing --pkgtype"
[ -z "${BUILDER:-}" ] && die "missing --builder"
[ -z "${EMQX_BUILDER:-}" ] && die "missing --builder"
[ -z "${ARCH:-}" ] && die "missing --arch"
# ensure dir
cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
set -x
if [ -z "${IS_ELIXIR:-}" ]; then
IS_ELIXIR=no
fi
case "$PKGTYPE" in
tgz|pkg)
case "${PKGTYPE:-}" in
tgz|pkg|rel)
true
;;
*)
echo "Bad --pkgtype option, should be tgz or pkg"
echo "Bad --pkgtype option, should be tgz, pkg or rel"
exit 1
;;
esac
@ -135,9 +136,13 @@ else
fi
HOST_SYSTEM="$(./scripts/get-distro.sh)"
BUILDER_SYSTEM="${BUILDER_SYSTEM:-$(echo "$BUILDER" | awk -F'-' '{print $NF}')}"
BUILDER_SYSTEM="${BUILDER_SYSTEM:-$(echo "$EMQX_BUILDER" | awk -F'-' '{print $NF}')}"
CMD_RUN="make ${MAKE_TARGET} && ./scripts/pkg-tests.sh ${MAKE_TARGET}"
if [ "${PKGTYPE}" != 'rel' ]; then
CMD_RUN="make ${MAKE_TARGET} && ./scripts/pkg-tests.sh ${MAKE_TARGET}"
else
CMD_RUN="make ${MAKE_TARGET}"
fi
IS_NATIVE_SYSTEM='no'
if [[ "$BUILDER_SYSTEM" != "force_docker" ]]; then
@ -163,7 +168,7 @@ elif docker info; then
--workdir /emqx \
--platform="linux/$ARCH" \
--env ACLOCAL_PATH="/usr/share/aclocal:/usr/local/share/aclocal" \
"$BUILDER" \
"$EMQX_BUILDER" \
bash -euc "git config --global --add safe.directory /emqx && $CMD_RUN"
else
echo "Error: Docker not available on unsupported platform"

8
scripts/pr-sanity-checks.sh
View File

@ -12,11 +12,9 @@ if ! type "yq" > /dev/null; then
exit 1
fi
EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.3-9}
OTP_VSN=${OTP_VSN:-26.2.5-3}
ELIXIR_VSN=${ELIXIR_VSN:-1.15.7}
EMQX_BUILDER_PLATFORM=${EMQX_BUILDER_PLATFORM:-ubuntu22.04}
EMQX_BUILDER=${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VERSION}:${ELIXIR_VSN}-${OTP_VSN}-${EMQX_BUILDER_PLATFORM}}
cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
# shellcheck disable=SC1091
source ./env.sh
commands=$(yq ".jobs.sanity-checks.steps[].run" .github/workflows/_pr_entrypoint.yaml | grep -v null)

4
scripts/relup-test/start-relup-test-cluster.sh
View File

@ -9,6 +9,8 @@ set -euo pipefail
# ensure dir
cd -P -- "$(dirname -- "$0")/../.."
# shellcheck disable=SC1091
source ./env.sh
set -x
@ -22,7 +24,7 @@ WEBHOOK="webhook.$NET"
BENCH="bench.$NET"
COOKIE='this-is-a-secret'
## Erlang image is needed to run webhook server and emqtt-bench
ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-ubuntu22.04"
ERLANG_IMAGE="${EMQX_BUILDER}"
# builder has emqtt-bench installed
BENCH_IMAGE="$ERLANG_IMAGE"