yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

build: change package name scheme 

`${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip`
This commit is contained in:
Zaiming Shi 2021-11-18 18:09:11 +01:00
parent 25d6a98aa4
commit a705bc1fc7
21 changed files with 297 additions and 342 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml
View File

@ -1,5 +1,4 @@
x-default-emqx: &default-emqx x-default-emqx: &default-emqx
image: $TARGET:$EMQX_TAG
env_file: env_file:
- conf.cluster.env - conf.cluster.env
healthcheck: healthcheck:
@ -15,8 +14,8 @@ services:
restart: on-failure restart: on-failure
environment: environment:
- "EMQX_HOST=node1.emqx.io" - "EMQX_HOST=node1.emqx.io"
- "EMQX_CLUSTER__DB_BACKEND=rlog" - "EMQX_DB__BACKEND=rlog"
- "EMQX_CLUSTER__RLOG__ROLE=core" - "EMQX_DB__ROLE=core"
- "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io]" - "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io]"
- "EMQX_LISTENERS__TCP__DEFAULT__PROXY_PROTOCOL=false" - "EMQX_LISTENERS__TCP__DEFAULT__PROXY_PROTOCOL=false"
- "EMQX_LISTENERS__WS__DEFAULT__PROXY_PROTOCOL=false" - "EMQX_LISTENERS__WS__DEFAULT__PROXY_PROTOCOL=false"
@ -29,9 +28,9 @@ services:
restart: on-failure restart: on-failure
environment: environment:
- "EMQX_HOST=node2.emqx.io" - "EMQX_HOST=node2.emqx.io"
- "EMQX_CLUSTER__DB_BACKEND=rlog" - "EMQX_DB__BACKEND=rlog"
- "EMQX_CLUSTER__RLOG__ROLE=replicant" - "EMQX_DB__ROLE=replicant"
- "EMQX_CLUSTER__RLOG__CORE_NODES=emqx@node1.emqx.io" - "EMQX_DB__CORE_NODES=emqx@node1.emqx.io"
- "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io]" - "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io]"
- "EMQX_LISTENERS__TCP__DEFAULT__PROXY_PROTOCOL=false" - "EMQX_LISTENERS__TCP__DEFAULT__PROXY_PROTOCOL=false"
- "EMQX_LISTENERS__WS__DEFAULT__PROXY_PROTOCOL=false" - "EMQX_LISTENERS__WS__DEFAULT__PROXY_PROTOCOL=false"

2
.ci/docker-compose-file/docker-compose-emqx-cluster.yaml
View File

@ -1,7 +1,7 @@
version: '3.9' version: '3.9'
x-default-emqx: &default-emqx x-default-emqx: &default-emqx
image: $TARGET:$EMQX_TAG image: ${_EMQX_DOCKER_IMAGE_TAG}
env_file: env_file:
- conf.cluster.env - conf.cluster.env
healthcheck: healthcheck:

4
.ci/docker-compose-file/docker-compose.yaml
View File

@ -3,7 +3,7 @@ version: '3.9'
services: services:
erlang23: erlang23:
container_name: erlang23 container_name: erlang23
image: ghcr.io/emqx/emqx-builder/5.0:23.2.7.2-emqx-2-ubuntu20.04 image: ghcr.io/emqx/emqx-builder/5.0-2:23.3.4.9-3-ubuntu20.04
env_file: env_file:
- conf.env - conf.env
environment: environment:
@ -23,7 +23,7 @@ services:
erlang24: erlang24:
container_name: erlang24 container_name: erlang24
image: ghcr.io/emqx/emqx-builder/5.0:24.1.1-emqx-1-ubuntu20.04 image: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-ubuntu20.04
env_file: env_file:
- conf.env - conf.env
environment: environment:

6
.ci/docker-compose-file/scripts/run-emqx.sh
View File

@ -1,7 +1,11 @@
#!/bin/bash #!/bin/bash
set -euxo pipefail set -euxo pipefail
if [ "$EMQX_TEST_DB_BACKEND" = "rlog" ] # _EMQX_DOCKER_IMAGE_TAG is shared with docker-compose file
export _EMQX_DOCKER_IMAGE_TAG="$1"
_EMQX_TEST_DB_BACKEND="${2:-${_EMQX_TEST_DB_BACKEND:-mnesia}}"
if [ "$_EMQX_TEST_DB_BACKEND" = "rlog" ]
then then
CLUSTER_OVERRIDES="-f .ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml" CLUSTER_OVERRIDES="-f .ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml"
else else

196
.github/workflows/build_packages.yaml vendored
View File

@ -14,89 +14,63 @@ on:
jobs: jobs:
prepare: prepare:
strategy:
matrix:
otp:
- "23.2.7.2-emqx-2"
- "24.1.1-emqx-1"
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-ubuntu20.04" # prepare source with any OTP version, no need for a matrix
container: "ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-ubuntu20.04"
outputs: outputs:
profiles: ${{ steps.set_profile.outputs.profiles }} old_vsns: ${{ steps.find_old_versons.outputs.old_vsns }}
old_vsns: ${{ steps.set_profile.outputs.old_vsns }}
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
path: source path: source
fetch-depth: 0 fetch-depth: 0
- name: set profile - name: find old versions
id: set_profile id: find_old_versons
shell: bash shell: bash
working-directory: source working-directory: source
run: | run: |
vsn="$(./pkg-vsn.sh)" vsn="$(./pkg-vsn.sh)"
pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
if make emqx-ee --dry-run > /dev/null 2>&1; then old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")" echo "::set-output name=old_vsns::$old_vsns"
echo "::set-output name=old_vsns::$old_vsns" - name: get_all_deps
echo "::set-output name=profiles::[\"emqx-ee\"]" if: endsWith(github.repository, 'emqx')
else
old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
echo "::set-output name=old_vsns::$old_vsns"
echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
fi
- name: get otp version
id: get_otp_version
run: | run: |
otp="$(erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell)" make -C source deps-all
echo "::set-output name=otp::$otp" zip -ryq source.zip source/* source/.[^.]*
- name: set get token
if: endsWith(github.repository, 'enterprise')
run: |
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
- name: get deps
working-directory: source
run: |
make ensure-rebar3
./rebar3 as default get-deps
rm -rf rebar.lock
- name: gen zip file
run: zip -ryq source-${{ steps.get_otp_version.outputs.otp }}.zip source/* source/.[^.]*
- uses: actions/upload-artifact@v2 - uses: actions/upload-artifact@v2
with: with:
name: source-${{ steps.get_otp_version.outputs.otp }} name: source
path: source-${{ steps.get_otp_version.outputs.otp }}.zip path: source.zip
windows: windows:
runs-on: windows-2019 runs-on: windows-2019
needs: prepare needs: prepare
if: endsWith(github.repository, 'emqx')
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}} profile: # only CE for windows
exclude: - emqx
- profile: emqx-edge otp:
- 23.2
steps: steps:
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v2
with: with:
name: source-23.2.7.2-emqx-2 name: source
path: . path: .
- name: unzip source code - name: unzip source code
run: Expand-Archive -Path source-23.2.7.2-emqx-2.zip -DestinationPath ./ run: Expand-Archive -Path source.zip -DestinationPath ./
- uses: ilammy/msvc-dev-cmd@v1 - uses: ilammy/msvc-dev-cmd@v1
- uses: gleam-lang/setup-erlang@v1.1.2 - uses: gleam-lang/setup-erlang@v1.1.2
id: install_erlang id: install_erlang
## gleam-lang/setup-erlang does not yet support the installation of otp24 on windows ## gleam-lang/setup-erlang does not yet support the installation of otp24 on windows
with: with:
otp-version: 23.2 otp-version: ${{ matrix.otp }}
- name: build - name: build
env: env:
PYTHON: python PYTHON: python
@ -108,10 +82,10 @@ jobs:
$version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" ) $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" )
if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") { if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") {
$regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+" $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+"
$pkg_name = "${{ matrix.profile }}-windows-$([regex]::matches($version, $regex).value).zip" $pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.zip"
} }
else { else {
$pkg_name = "${{ matrix.profile }}-windows-$($version -replace '/').zip" $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip"
} }
## We do not build/release bcrypt and quic for windows package ## We do not build/release bcrypt and quic for windows package
Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/ Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/
@ -140,34 +114,32 @@ jobs:
- uses: actions/upload-artifact@v1 - uses: actions/upload-artifact@v1
if: startsWith(github.ref, 'refs/tags/') if: startsWith(github.ref, 'refs/tags/')
with: with:
name: ${{ matrix.profile }}-23.2.7.2-emqx-2 name: ${{ matrix.profile }}
path: source/_packages/${{ matrix.profile }}/. path: source/_packages/${{ matrix.profile }}/.
mac: mac:
needs: prepare needs: prepare
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}} profile: # no EDGE for mac
- emqx
- emqx-ee
otp:
- 24.1.5-2
macos: macos:
- macos-11 - macos-11
- macos-10.15 - macos-10.15
otp:
- 24.1.1-emqx-1
exclude: exclude:
- profile: emqx-edge - profile: emqx-edge
runs-on: ${{ matrix.macos }} runs-on: ${{ matrix.macos }}
steps: steps:
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v2
with: with:
name: source-${{ matrix.otp }} name: source
path: . path: .
- name: unzip source code - name: unzip source code
run: unzip -q source-${{ matrix.otp }}.zip run: unzip -q source.zip
- name: prepare - name: prepare
run: | run: |
brew update brew update
@ -182,8 +154,12 @@ jobs:
- name: build erlang - name: build erlang
if: steps.cache.outputs.cache-hit != 'true' if: steps.cache.outputs.cache-hit != 'true'
timeout-minutes: 60 timeout-minutes: 60
env:
KERL_BUILD_BACKEND: git
OTP_GITHUB_URL: https://github.com/emqx/otp
run: | run: |
kerl build git https://github.com/emqx/otp.git OTP-${{ matrix.otp }} ${{ matrix.otp }} kerl update releases
kerl build ${{ matrix.otp }}
kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }} kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }}
- name: build - name: build
working-directory: source working-directory: source
@ -191,11 +167,12 @@ jobs:
. $HOME/.kerl/${{ matrix.otp }}/activate . $HOME/.kerl/${{ matrix.otp }}/activate
make ensure-rebar3 make ensure-rebar3
sudo cp rebar3 /usr/local/bin/rebar3 sudo cp rebar3 /usr/local/bin/rebar3
rm -rf _build/${{ matrix.profile }}/lib
make ${{ matrix.profile }}-zip make ${{ matrix.profile }}-zip
- name: test - name: test
working-directory: source working-directory: source
run: | run: |
pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip | head) pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip)
unzip -q $pkg_name unzip -q $pkg_name
# gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
@ -230,7 +207,15 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}} profile: ## all editions for linux
- emqx-edge
- emqx
- emqx-ee
otp:
- 24.1.5-2 # we test with OTP 23, but only build package on OTP 24 versions
arch:
- amd64
- arm64
os: os:
- ubuntu20.04 - ubuntu20.04
- ubuntu18.04 - ubuntu18.04
@ -240,18 +225,9 @@ jobs:
# - opensuse # - opensuse
- centos8 - centos8
- centos7 - centos7
- centos6
- raspbian10 - raspbian10
# - raspbian9 # - raspbian9
arch:
- amd64
- arm64
otp:
- 23.2.7.2-emqx-2
- 24.1.1-emqx-1
exclude: exclude:
- os: centos6
arch: arm64
- os: raspbian9 - os: raspbian9
arch: amd64 arch: amd64
- os: raspbian10 - os: raspbian10
@ -277,12 +253,13 @@ jobs:
platforms: all platforms: all
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v2
with: with:
name: source-${{ matrix.otp }} name: source
path: . path: .
- name: unzip source code - name: unzip source code
run: unzip -q source-${{ matrix.otp }}.zip run: unzip -q source.zip
- name: downloads old emqx zip packages - name: downloads old emqx zip packages
env: env:
OTP_VSN: ${{ matrix.otp }}
PROFILE: ${{ matrix.profile }} PROFILE: ${{ matrix.profile }}
ARCH: ${{ matrix.arch }} ARCH: ${{ matrix.arch }}
SYSTEM: ${{ matrix.os }} SYSTEM: ${{ matrix.os }}
@ -302,10 +279,11 @@ jobs:
cd _upgrade_base cd _upgrade_base
old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
for tag in ${old_vsns[@]}; do for tag in ${old_vsns[@]}; do
if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip) | grep -oE "^[23]+")" ]; then
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256 wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip
echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1 wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip.sha256
echo "$(cat $package_name.zip.sha256) $package_name.zip" | sha256sum -c || exit 1
fi fi
done done
- name: build emqx packages - name: build emqx packages
@ -320,7 +298,7 @@ jobs:
-v $(pwd):/emqx \ -v $(pwd):/emqx \
--workdir /emqx \ --workdir /emqx \
--platform linux/$ARCH \ --platform linux/$ARCH \
ghcr.io/emqx/emqx-builder/5.0:$OTP-$SYSTEM \ ghcr.io/emqx/emqx-builder/5.0-2:$OTP-$SYSTEM \
bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \ bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \
make $PROFILE-pkg || cat rebar3.crashdump; \ make $PROFILE-pkg || cat rebar3.crashdump; \
EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh" EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh"
@ -349,17 +327,21 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}} profile: # all editions for docker
- emqx-edge
- emqx
- emqx-ee
# NOTE: for docker, only support latest otp version, not a matrix
otp: otp:
- 24.1.1-emqx-1 - 24.1.5-2 # update to latest
steps: steps:
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v2
with: with:
name: source-${{ matrix.otp }} name: source
path: . path: .
- name: unzip source code - name: unzip source code
run: unzip -q source-${{ matrix.otp }}.zip run: unzip -q source.zip
- uses: docker/setup-buildx-action@v1 - uses: docker/setup-buildx-action@v1
- uses: docker/setup-qemu-action@v1 - uses: docker/setup-qemu-action@v1
with: with:
@ -376,7 +358,8 @@ jobs:
type=ref,event=pr type=ref,event=pr
type=ref,event=tag type=ref,event=tag
type=semver,pattern={{version}} type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}} labels:
org.opencontainers.image.otp.version=${{ matrix.otp }}
- uses: docker/login-action@v1 - uses: docker/login-action@v1
if: github.event_name == 'release' if: github.event_name == 'release'
with: with:
@ -384,32 +367,26 @@ jobs:
password: ${{ secrets.DOCKER_HUB_TOKEN }} password: ${{ secrets.DOCKER_HUB_TOKEN }}
- uses: docker/build-push-action@v2 - uses: docker/build-push-action@v2
with: with:
push: ${{ github.event_name == 'release' }} push: ${{ github.event_name == 'release' && !github.event.release.prerelease }}
pull: true pull: true
no-cache: true no-cache: true
platforms: linux/amd64,linux/arm64 platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta.outputs.tags }} tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }} labels: ${{ steps.meta.outputs.labels }}
build-args: | build-args: |
BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-alpine3.14 BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp }}-alpine3.14
RUN_FROM=alpine:3.14 RUN_FROM=alpine:3.14
EMQX_NAME=${{ matrix.profile }} EMQX_NAME=${{ matrix.profile }}
file: source/deploy/docker/Dockerfile file: source/deploy/docker/Dockerfile
context: source context: source
delete-artifact: delete-artifact:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
strategy:
matrix:
otp:
- 23.2.7.2-emqx-2
- 24.1.1-emqx-1
needs: [prepare, mac, linux, docker] needs: [prepare, mac, linux, docker]
steps: steps:
- uses: geekyeggo/delete-artifact@v1 - uses: geekyeggo/delete-artifact@v1
with: with:
name: source-${{ matrix.otp }} name: source
upload: upload:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
@ -420,9 +397,12 @@ jobs:
strategy: strategy:
matrix: matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}} profile:
- emqx-edge
- emqx
- emqx-ee
otp: otp:
- 24.1.1-emqx-1 - 24.1.5-2
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
@ -461,17 +441,11 @@ jobs:
aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$broker/${{ env.version }} aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$broker/${{ env.version }}
aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$broker/${{ env.version }}/*" aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$broker/${{ env.version }}/*"
- uses: Rory-Z/upload-release-asset@v1 - uses: Rory-Z/upload-release-asset@v1
if: github.event_name == 'release' && matrix.profile != 'emqx-ee' if: github.event_name == 'release'
with: with:
repo: emqx repo: emqx
path: "_packages/${{ matrix.profile }}/emqx-*" path: "_packages/${{ matrix.profile }}/emqx-*"
token: ${{ github.token }} token: ${{ github.token }}
- uses: Rory-Z/upload-release-asset@v1
if: github.event_name == 'release' && matrix.profile == 'emqx-ee'
with:
repo: emqx-enterprise
path: "_packages/${{ matrix.profile }}/emqx-*"
token: ${{ github.token }}
- name: update to emqx.io - name: update to emqx.io
if: github.event_name == 'release' if: github.event_name == 'release'
run: | run: |
@ -484,32 +458,28 @@ jobs:
-d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \ -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
${{ secrets.EMQX_IO_RELEASE_API }} ${{ secrets.EMQX_IO_RELEASE_API }}
- name: update repo.emqx.io - name: update repo.emqx.io
if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee' if: github.event_name == 'release'
run: | run: |
if [ "${{ matrix. profile }}" = 'emqx-ee' ]; then
BOOL_FLAG_NAME="emqx_ee"
else
BOOL_FLAG_NAME="emqx_ce"
fi
curl --silent --show-error \ curl --silent --show-error \
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \ -H "Accept: application/vnd.github.v3+json" \
-X POST \ -X POST \
-d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \ -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\", \"${BOOL_FLAG_NAME}\": \"true\"}}" \
"https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
- name: update repo.emqx.io
if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
run: |
curl --silent --show-error \
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
-X POST \
-d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \
"https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
- name: update homebrew packages - name: update homebrew packages
if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' if: github.event_name == 'release' && matrix.profile == 'emqx'
run: | run: |
if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
curl --silent --show-error \ curl --silent --show-error \
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \ -H "Accept: application/vnd.github.v3+json" \
-X POST \ -X POST \
-d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \ -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \
"https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches" "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
fi fi
- uses: geekyeggo/delete-artifact@v1 - uses: geekyeggo/delete-artifact@v1

83
.github/workflows/build_slim_packages.yaml vendored
View File

@ -4,7 +4,6 @@ concurrency:
group: slim-${{ github.event_name }}-${{ github.ref }} group: slim-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: true cancel-in-progress: true
on: on:
push: push:
tags: tags:
@ -14,48 +13,37 @@ on:
workflow_dispatch: workflow_dispatch:
jobs: jobs:
build: linux:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
otp: profile:
- 24.1.1-emqx-1 - emqx-edge
- emqx
- emqx-ee
otp_vsn:
- 24.1.5-2
os: os:
- ubuntu20.04 - ubuntu20.04
- centos7 - centos7
container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-${{ matrix.os }}" container: "ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp_vsn }}-${{ matrix.os }}"
steps: steps:
- uses: actions/checkout@v1 - uses: actions/checkout@v1
- name: prepare - name: build zip package
run: | run: make ${{ matrix.profile }}-zip
if make emqx-ee --dry-run > /dev/null 2>&1; then
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
echo "${{ secrets.CI_GIT_TOKEN }}" >> ./scripts/git-token
echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
else
echo "EMQX_NAME=emqx" >> $GITHUB_ENV
fi
- name: build zip packages
run: make ${EMQX_NAME}-zip
- name: build deb/rpm packages - name: build deb/rpm packages
run: make ${EMQX_NAME}-pkg run: make ${{ matrix.profile }}-pkg
- uses: actions/upload-artifact@v1
if: failure()
with:
name: rebar3.crashdump
path: ./rebar3.crashdump
- name: packages test - name: packages test
run: | run: |
export CODE_PATH=$GITHUB_WORKSPACE export CODE_PATH=$GITHUB_WORKSPACE
.ci/build_packages/tests.sh EMQX_NAME=${{ matrix.profile }} .ci/build_packages/tests.sh
- uses: actions/upload-artifact@v2 - uses: actions/upload-artifact@v2
with: with:
name: ${{ matrix.os }} name: ${{ matrix.profile}}-${{ matrix.otp_vsn }}-${{ matrix.os }}
path: _packages/**/*.zip path: _packages/**/*.zip
mac: mac:
@ -63,57 +51,50 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
profile:
- emqx
- emqx-ee
otp_vsn:
- 24.1.5-2
macos: macos:
- macos-11 - macos-11
- macos-10.15 - macos-10.15
otp:
- 24.1.1-emqx-1
runs-on: ${{ matrix.macos }} runs-on: ${{ matrix.macos }}
steps: steps:
- uses: actions/checkout@v1 - uses: actions/checkout@v2
- name: prepare
run: |
if make emqx-ee --dry-run > /dev/null 2>&1; then
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
echo "${{ secrets.CI_GIT_TOKEN }}" >> ./scripts/git-token
echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
else
echo "EMQX_NAME=emqx" >> $GITHUB_ENV
fi
- name: prepare - name: prepare
run: | run: |
brew update brew update
brew install curl zip unzip gnu-sed kerl unixodbc freetds brew install curl zip unzip gnu-sed kerl unixodbc freetds
echo "/usr/local/bin" >> $GITHUB_PATH echo "/usr/local/bin" >> $GITHUB_PATH
git config --global credential.helper store echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
- uses: actions/cache@v2 - uses: actions/cache@v2
id: cache id: cache
with: with:
path: ~/.kerl path: ~/.kerl
key: otp-${{ matrix.otp }}-${{ matrix.macos }} key: otp-${{ matrix.otp_vsn }}-${{ matrix.macos }}
- name: build erlang - name: build erlang
if: steps.cache.outputs.cache-hit != 'true' if: steps.cache.outputs.cache-hit != 'true'
timeout-minutes: 60 timeout-minutes: 60
env:
KERL_BUILD_BACKEND: git
OTP_GITHUB_URL: https://github.com/emqx/otp
run: | run: |
kerl build git https://github.com/emqx/otp.git OTP-${{ matrix.otp }} ${{ matrix.otp }} kerl update releases
kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }} kerl build ${{ matrix.otp_vsn }}
- name: build kerl install ${{ matrix.otp_vsn }} $HOME/.kerl/${{ matrix.otp_vsn }}
- name: build ${{ matrix.profile }}
run: | run: |
. $HOME/.kerl/${{ matrix.otp }}/activate . $HOME/.kerl/${{ matrix.otp_vsn }}/activate
make ensure-rebar3 make ensure-rebar3
sudo cp rebar3 /usr/local/bin/rebar3 sudo cp rebar3 /usr/local/bin/rebar3
make ${EMQX_NAME}-zip make ${{ matrix.profile }}-zip
- uses: actions/upload-artifact@v1
if: failure()
with:
name: rebar3.crashdump
path: ./rebar3.crashdump
- name: test - name: test
run: | run: |
unzip -q $(find _packages/${EMQX_NAME} -mindepth 1 -maxdepth 1 -iname \*.zip | head) pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip)
unzip -q $pkg_name
# gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
ready='no' ready='no'

2
.github/workflows/check_deps_integrity.yaml vendored
View File

@ -5,7 +5,7 @@ on: [pull_request]
jobs: jobs:
check_deps_integrity: check_deps_integrity:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0:24.1.1-emqx-1-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-ubuntu20.04"
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2

12
.github/workflows/code_style_check.yaml vendored
View File

@ -9,19 +9,9 @@ jobs:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
fetch-depth: 1000 fetch-depth: 1000
- name: Set git token
if: endsWith(github.repository, 'enterprise')
run: |
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
- name: Run elvis check - name: Run elvis check
run: | run: |
set -e ./scripts/elvis-check.sh $GITHUB_BASE_REF
if [ -f EMQX_ENTERPRISE ]; then
./scripts/elvis-check.sh $GITHUB_BASE_REF emqx-enterprise
else
./scripts/elvis-check.sh $GITHUB_BASE_REF emqx
fi
- name: Check line-break at EOF - name: Check line-break at EOF
run: | run: |
./scripts/check-nl-at-eof.sh ./scripts/check-nl-at-eof.sh

6
.github/workflows/run_emqx_app_tests.yaml vendored
View File

@ -12,11 +12,11 @@ jobs:
strategy: strategy:
matrix: matrix:
otp: otp:
- "23.2.7.2-emqx-2" - 23.3.4.9-3
- "24.1.1-emqx-1" - 24.1.5-2
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp }}-ubuntu20.04"
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2

84
.github/workflows/run_fvt_tests.yaml vendored
View File

@ -8,58 +8,27 @@ on:
push: push:
tags: tags:
- v* - v*
- e*
pull_request: pull_request:
jobs: jobs:
prepare: prepare:
strategy:
matrix:
otp:
- "23.2.7.2-emqx-2"
- "24.1.1-emqx-1"
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-ubuntu20.04" # prepare source with any OTP version, no need for a matrix
container: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14
outputs:
profile: ${{ steps.profile.outputs.profile }}
steps: steps:
- name: get otp version
id: get_otp_version
run: |
otp="$(erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell)"
echo "::set-output name=otp::$otp"
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
path: source path: source
fetch-depth: 0 fetch-depth: 0
- name: set profile
id: profile
shell: bash
working-directory: source
run: |
vsn="$(./pkg-vsn.sh)"
if make emqx-ee --dry-run > /dev/null 2>&1; then
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
echo "::set-output name=profile::emqx-ee"
else
echo "::set-output name=profile::emqx"
fi
- name: get deps - name: get deps
working-directory: source
run: | run: |
make ensure-rebar3 make -C source deps-all
./rebar3 as default get-deps zip -ryq source.zip source/* source/.[^.]*
rm -rf rebar.lock
- name: gen zip file
run: zip -ryq source-${{ steps.get_otp_version.outputs.otp }}.zip source/* source/.[^.]*
- uses: actions/upload-artifact@v2 - uses: actions/upload-artifact@v2
with: with:
name: source-${{ steps.get_otp_version.outputs.otp }} name: source
path: source-${{ steps.get_otp_version.outputs.otp }}.zip path: source.zip
docker_test: docker_test:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
@ -68,35 +37,34 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
otp: profile:
- 23.2.7.2-emqx-2 - emqx-edge
- 24.1.1-emqx-1 - emqx
- emqx-ee
cluster_db_backend: cluster_db_backend:
- "mnesia" - mnesia
- "rlog" - rlog
steps: steps:
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v2
with: with:
name: source-${{ matrix.otp }} name: source
path: . path: .
- name: unzip source code - name: unzip source code
run: unzip -q source-${{ matrix.otp }}.zip run: unzip -q source.zip
- name: make docker image - name: make docker image
working-directory: source working-directory: source
env: env:
OTP: ${{ matrix.otp }} EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14
run: | run: |
make ${{ needs.prepare.outputs.profile }}-docker make ${{ matrix.profile }}-docker
echo "TARGET=emqx/${{ needs.prepare.outputs.profile }}" >> $GITHUB_ENV
echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
- name: run emqx - name: run emqx
timeout-minutes: 5 timeout-minutes: 5
working-directory: source working-directory: source
run: | run: |
set -x set -x
export EMQX_TEST_DB_BACKEND="${{ matrix.cluster_db_backend }}" IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh)
./.ci/docker-compose-file/scripts/run-emqx.sh ./.ci/docker-compose-file/scripts/run-emqx.sh $IMAGE ${{ matrix.cluster_db_backend }}
- name: make paho tests - name: make paho tests
run: | run: |
if ! docker exec -i python /scripts/pytest.sh "${{ matrix.cluster_db_backend }}"; then if ! docker exec -i python /scripts/pytest.sh "${{ matrix.cluster_db_backend }}"; then
@ -118,24 +86,24 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
otp: profile:
- 23.2.7.2-emqx-2 - emqx
- 24.1.1-emqx-1 - emqx-ee
steps: steps:
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v2
with: with:
name: source-${{ matrix.otp }} name: source
path: . path: .
- name: unzip source code - name: unzip source code
run: unzip -q source-${{ matrix.otp }}.zip run: unzip -q source.zip
- name: make docker image - name: make docker image
working-directory: source working-directory: source
env: env:
OTP: ${{ matrix.otp }} EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14
run: | run: |
make ${{ needs.prepare.outputs.profile }}-docker make ${{ matrix.profile }}-docker
echo "TARGET=emqx/${{ needs.prepare.outputs.profile }}" >> $GITHUB_ENV echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV
echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
- run: minikube start - run: minikube start
- name: run emqx on chart - name: run emqx on chart

36
.github/workflows/run_relup_tests.yaml vendored
View File

@ -15,12 +15,14 @@ jobs:
relup_test: relup_test:
strategy: strategy:
matrix: matrix:
otp: profile:
- "23.2.7.2-emqx-2" - emqx
- "24.1.1-emqx-1" - emqx-ee
otp_vsn:
- 24.1.5-2
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp_vsn }}-ubuntu20.04"
defaults: defaults:
run: run:
@ -43,7 +45,7 @@ jobs:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
repository: emqx/emqtt-bench repository: emqx/emqtt-bench
ref: 0.3.4 ref: 0.3.4
path: emqtt-bench path: emqtt-bench
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
@ -55,26 +57,18 @@ jobs:
repository: ${{ github.repository }} repository: ${{ github.repository }}
path: emqx path: emqx
fetch-depth: 0 fetch-depth: 0
- name: prepare
run: |
if make -C emqx emqx-ee --dry-run > /dev/null 2>&1; then
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
echo "${{ secrets.CI_GIT_TOKEN }}" >> emqx/scripts/git-token
echo "PROFILE=emqx-ee" >> $GITHUB_ENV
else
echo "PROFILE=emqx" >> $GITHUB_ENV
fi
- name: get version - name: get version
run: | run: |
set -e -x -u set -e -x -u
cd emqx cd emqx
export PROFILE=${{ matrix.profile }}
export OTP_VSN=${{ matrix.otp_vsn }}
echo "PROFILE=$PROFILE" >> $GITHUB_ENV
echo "OTP_VSN=$OTP_VSN" >> $GITHUB_ENV
if [ $PROFILE = "emqx" ];then if [ $PROFILE = "emqx" ];then
broker="emqx-ce" broker="emqx-ce"
edition='opensource'
else else
broker="emqx-ee" broker="emqx-ee"
edition='enterprise'
fi fi
echo "BROKER=$broker" >> $GITHUB_ENV echo "BROKER=$broker" >> $GITHUB_ENV
@ -82,11 +76,7 @@ jobs:
echo "VSN=$vsn" >> $GITHUB_ENV echo "VSN=$vsn" >> $GITHUB_ENV
pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
if [ $PROFILE = "emqx" ]; then old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
else
old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")"
fi
echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV
- name: download emqx - name: download emqx
run: | run: |
@ -95,7 +85,7 @@ jobs:
cd emqx/_upgrade_base cd emqx/_upgrade_base
old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
for old_vsn in ${old_vsns[@]}; do for old_vsn in ${old_vsns[@]}; do
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-amd64.zip wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${OTP_VSN}-ubuntu20.04-amd64.zip
done done
- name: build emqx - name: build emqx
run: make -C emqx ${PROFILE}-zip run: make -C emqx ${PROFILE}-zip

33
.github/workflows/run_test_cases.yaml vendored
View File

@ -15,12 +15,11 @@ jobs:
run_static_analysis: run_static_analysis:
strategy: strategy:
matrix: matrix:
otp: emqx_builder:
- "23.2.7.2-emqx-2" - 5.0-2:24.1.5-2 # run dialyzer on latest OTP
- "24.1.1-emqx-1"
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/${{ matrix.emqx_builder }}-ubuntu20.04"
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
@ -38,12 +37,11 @@ jobs:
run_proper_test: run_proper_test:
strategy: strategy:
matrix: matrix:
otp: emqx_builder:
- "23.2.7.2-emqx-2" - 5.0-2:24.1.5-2
- "24.1.1-emqx-1"
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/${{ matrix.emqx_builder }}-ubuntu20.04"
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
@ -67,32 +65,13 @@ jobs:
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
- name: set edition
id: set_edition
run: |
if make emqx-ee --dry-run > /dev/null 2>&1; then
echo "EDITION=enterprise" >> $GITHUB_ENV
else
echo "EDITION=opensource" >> $GITHUB_ENV
fi
- name: docker compose up - name: docker compose up
if: env.EDITION == 'opensource'
env: env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: | run: |
docker-compose \ docker-compose \
-f .ci/docker-compose-file/docker-compose.yaml \ -f .ci/docker-compose-file/docker-compose.yaml \
up -d --build up -d --build
- name: docker compose up
if: env.EDITION == 'enterprise'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
timeout-minutes: 20
run: |
docker-compose \
-f .ci/docker-compose-file/docker-compose.yaml \
-f .ci/docker-compose-file/docker-compose-enterprise.yaml \
up -d --build
- name: run eunit - name: run eunit
run: | run: |
docker exec -i ${{ matrix.otp_release }} bash -c "make eunit" docker exec -i ${{ matrix.otp_release }} bash -c "make eunit"

22
Makefile
View File

@ -3,10 +3,13 @@ REBAR_VERSION = 3.16.1-emqx-1
REBAR = $(CURDIR)/rebar3 REBAR = $(CURDIR)/rebar3
BUILD = $(CURDIR)/build BUILD = $(CURDIR)/build
SCRIPTS = $(CURDIR)/scripts SCRIPTS = $(CURDIR)/scripts
export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14
export EMQX_DEFAULT_RUNNER = alpine:3.14
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
export EMQX_DESC ?= EMQ X
export EMQX_DASHBOARD_VERSION ?= v5.0.0-beta.18 export EMQX_DASHBOARD_VERSION ?= v5.0.0-beta.18
export DOCKERFILE := deploy/docker/Dockerfile
export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing
ifeq ($(OS),Windows_NT) ifeq ($(OS),Windows_NT)
export REBAR_COLOR=none export REBAR_COLOR=none
endif endif
@ -86,7 +89,6 @@ coveralls: $(REBAR)
@ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send @ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send
.PHONY: $(REL_PROFILES) .PHONY: $(REL_PROFILES)
$(REL_PROFILES:%=%): $(REBAR) get-dashboard conf-segs $(REL_PROFILES:%=%): $(REBAR) get-dashboard conf-segs
@$(REBAR) as $(@) do compile,release @$(REBAR) as $(@) do compile,release
@ -99,8 +101,10 @@ $(REL_PROFILES:%=%): $(REBAR) get-dashboard conf-segs
clean: $(PROFILES:%=clean-%) clean: $(PROFILES:%=clean-%)
$(PROFILES:%=clean-%): $(PROFILES:%=clean-%):
@if [ -d _build/$(@:clean-%=%) ]; then \ @if [ -d _build/$(@:clean-%=%) ]; then \
rm rebar.lock \
rm -rf _build/$(@:clean-%=%)/rel; \ rm -rf _build/$(@:clean-%=%)/rel; \
find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \ find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
find _build/$(@:clean-%=%) -type l | xargs rm -i -f ; \
fi fi
.PHONY: clean-all .PHONY: clean-all
@ -110,6 +114,7 @@ clean-all:
.PHONY: deps-all .PHONY: deps-all
deps-all: $(REBAR) $(PROFILES:%=deps-%) deps-all: $(REBAR) $(PROFILES:%=deps-%)
@make clean # ensure clean at the end
## deps-<profile> is used in CI scripts to download deps and the ## deps-<profile> is used in CI scripts to download deps and the
## share downloads between CI steps and/or copied into containers ## share downloads between CI steps and/or copied into containers
@ -117,6 +122,7 @@ deps-all: $(REBAR) $(PROFILES:%=deps-%)
.PHONY: $(PROFILES:%=deps-%) .PHONY: $(PROFILES:%=deps-%)
$(PROFILES:%=deps-%): $(REBAR) get-dashboard $(PROFILES:%=deps-%): $(REBAR) get-dashboard
@$(REBAR) as $(@:deps-%=%) get-deps @$(REBAR) as $(@:deps-%=%) get-deps
@rm -f rebar.lock
.PHONY: xref .PHONY: xref
xref: $(REBAR) xref: $(REBAR)
@ -175,5 +181,17 @@ endef
ALL_ZIPS = $(REL_PROFILES) ALL_ZIPS = $(REL_PROFILES)
$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt)))) $(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt))))
## emqx-docker-testing
## emqx-ee-docker-testing
## is to directly copy a unzipped zip-package to a
## base image such as ubuntu20.04. Mostly for testing
.PHONY: $(REL_PROFILES:%=%-docker-testing)
define gen-docker-target-testing
$1-docker-testing: $(COMMON_DEPS)
@$(BUILD) $1 docker-testing
endef
ALL_ZIPS = $(REL_PROFILES)
$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target-testing,$(zt))))
conf-segs: conf-segs:
@scripts/merge-config.escript @scripts/merge-config.escript

8
apps/emqx/src/emqx_misc.erl
View File

@ -332,6 +332,12 @@ pad(L, Count) ->
-include_lib("eunit/include/eunit.hrl"). -include_lib("eunit/include/eunit.hrl").
ipv6_probe_test() -> ipv6_probe_test() ->
?assertEqual([{ipv6_probe, true}], ipv6_probe([])). try gen_tcp:ipv6_probe() of
true ->
?assertEqual([{ipv6_probe, true}], ipv6_probe([]))
catch
_ : _ ->
ok
end.
-endif. -endif.

90
build
View File

@ -12,23 +12,10 @@ ARTIFACT="$2"
# ensure dir # ensure dir
cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")"
PKG_VSN="$(./pkg-vsn.sh)" PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
export PKG_VSN export PKG_VSN
if [ "$(uname -s)" = 'Darwin' ]; then SYSTEM="$(./scripts/get-distro.sh)"
DIST='macos'
VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g')
SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')"
elif [ "$(uname -s)" = 'Linux' ]; then
if grep -q -i 'centos' /etc/*-release; then
DIST='centos'
VERSION_ID="$(rpm --eval '%{centos_ver}')"
else
DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')"
VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')"
fi
SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')"
fi
ARCH="$(uname -m)" ARCH="$(uname -m)"
case "$ARCH" in case "$ARCH" in
@ -48,8 +35,8 @@ export ARCH
## Support RPM and Debian based linux systems ## Support RPM and Debian based linux systems
## ##
if [ "$(uname -s)" = 'Linux' ]; then if [ "$(uname -s)" = 'Linux' ]; then
case "${DIST:-}" in case "${SYSTEM:-}" in
ubuntu|debian|raspbian) ubuntu*|debian*|raspbian*)
PKGERDIR='deb' PKGERDIR='deb'
;; ;;
*) *)
@ -109,7 +96,7 @@ make_relup() {
rm -rf "$tmp_dir" rm -rf "$tmp_dir"
fi fi
releases+=( "$base_vsn" ) releases+=( "$base_vsn" )
done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-$SYSTEM*-$ARCH.zip" -type f) done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-otp${OTP_VSN}-$SYSTEM*-$ARCH.zip" -type f)
fi fi
if [ ${#releases[@]} -eq 0 ]; then if [ ${#releases[@]} -eq 0 ]; then
log "No upgrade base found, relup ignored" log "No upgrade base found, relup ignored"
@ -149,7 +136,7 @@ make_zip() {
log "ERROR: $tarball is not found" log "ERROR: $tarball is not found"
fi fi
local zipball local zipball
zipball="${pkgpath}/${PROFILE}-${SYSTEM}-${PKG_VSN}-${ARCH}.zip" zipball="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip"
tar zxf "${tarball}" -C "${tard}/emqx" tar zxf "${tarball}" -C "${tard}/emqx"
## try to be portable for zip packages. ## try to be portable for zip packages.
## for DEB and RPM packages the dependencies are resoved by yum and apt ## for DEB and RPM packages the dependencies are resoved by yum and apt
@ -157,18 +144,60 @@ make_zip() {
(cd "${tard}" && zip -qr - emqx) > "${zipball}" (cd "${tard}" && zip -qr - emqx) > "${zipball}"
} }
## This function builds the default docker image based on alpine:3.14 (by default)
make_docker() { make_docker() {
## Build Docker image EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}"
echo "DOCKER BUILD: Build Docker image." EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}"
echo "DOCKER BUILD: build version -> $PKG_VSN." set -x
echo "DOCKER BUILD: docker repo -> emqx/$PROFILE " docker build --no-cache --pull \
--build-arg BUILD_FROM="${EMQX_BUILDER}" \
docker build --no-cache \ --build-arg RUN_FROM="${EMQX_RUNNER}" \
--build-arg BUILD_FROM="ghcr.io/emqx/emqx-builder/5.0:${OTP:-24.1.1-emqx-1}-alpine3.14" \
--build-arg RUN_FROM="alpine:3.14" \
--build-arg EMQX_NAME="$PROFILE" \ --build-arg EMQX_NAME="$PROFILE" \
--tag "emqx/$PROFILE:$PKG_VSN" \ --tag "emqx/$PROFILE:${PKG_VSN}" \
-f deploy/docker/Dockerfile . -f "${DOCKERFILE}" .
}
## This function accepts any base docker image,
## a emqx zip-image, and a image tag (for the image to be built),
## to build a docker image which runs EMQ X
##
## Export below variables to quickly build an image
##
## Name Default Example
## ---------------------------------------------------------------------
## EMQX_BASE_IMAGE current os centos:7
## EMQX_ZIP_PACKAGE _packages/<current-zip-target> /tmp/emqx-4.4.0-otp23.3.4.9-3-centos7-amd64.zip
## EMQX_IMAGE_TAG emqx/emqx:<current-vns-rel> emqx/emqx:testing-tag
##
make_docker_testing() {
if [ -z "${EMQX_BASE_IMAGE:-}" ]; then
case "$SYSTEM" in
ubuntu20*)
EMQX_BASE_IMAGE="ubuntu:20.04"
;;
centos8)
EMQX_BASE_IMAGE="centos:8"
;;
*)
echo "Unsupported testing base image for $SYSTEM"
exit 1
;;
esac
fi
EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}"
local defaultzip
defaultzip="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip"
local zip="${EMQX_ZIP_PACKAGE:-$defaultzip}"
if [ ! -f "$zip" ]; then
log "ERROR: $zip not built?"
exit 1
fi
set -x
docker build \
--build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \
--build-arg EMQX_ZIP_PACKAGE="${zip}" \
--tag "$EMQX_IMAGE_TAG" \
-f "${DOCKERFILE_TESTING}" .
} }
log "building artifact=$ARTIFACT for profile=$PROFILE" log "building artifact=$ARTIFACT for profile=$PROFILE"
@ -194,6 +223,9 @@ case "$ARTIFACT" in
docker) docker)
make_docker make_docker
;; ;;
docker-testing)
make_docker_testing
;;
*) *)
log "Unknown artifact $ARTIFACT" log "Unknown artifact $ARTIFACT"
exit 1 exit 1

2
deploy/docker/Dockerfile
View File

@ -1,4 +1,4 @@
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0:24.1.1-emqx-1-alpine3.14 ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14
ARG RUN_FROM=alpine:3.14 ARG RUN_FROM=alpine:3.14
FROM ${BUILD_FROM} AS builder FROM ${BUILD_FROM} AS builder

2
deploy/packages/deb/Makefile
View File

@ -8,7 +8,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz
SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture)
TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH)
.PHONY: all .PHONY: all
all: | $(BUILT) all: | $(BUILT)

2
deploy/packages/deb/debian/control
View File

@ -4,7 +4,7 @@ Priority: optional
Maintainer: emqx <contact@emqx.io> Maintainer: emqx <contact@emqx.io>
Build-Depends: debhelper (>=9) Build-Depends: debhelper (>=9)
Standards-Version: 3.9.6 Standards-Version: 3.9.6
Homepage: https://www.emqx.io Homepage: https://www.emqx.com
Package: emqx Package: emqx
Architecture: any Architecture: any

14
deploy/packages/rpm/Makefile
View File

@ -5,8 +5,9 @@ BUILT := $(SRCDIR)/BUILT
dash := - dash := -
none := none :=
space := $(none) $(none) space := $(none) $(none)
RPM_VSN ?= $(shell echo $(PKG_VSN) | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+)?") ## RPM does not allow '-' in version nubmer and release string, replace with '_'
RPM_REL ?= $(shell echo $(PKG_VSN) | grep -oE "(alpha|beta|rc)\.[0-9]") RPM_VSN := $(subst -,_,$(PKG_VSN))
RPM_REL := otp$(subst -,_,$(OTP_VSN))
ARCH ?= amd64 ARCH ?= amd64
ifeq ($(ARCH),mips64) ifeq ($(ARCH),mips64)
@ -16,12 +17,8 @@ endif
EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz
TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH)
ifeq ($(RPM_REL),) SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m)
# no tail
RPM_REL := 1
endif
SOURCE_PKG := emqx-$(SYSTEM)-$(RPM_VSN)-$(RPM_REL).$(shell uname -m)
SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi)
# Not $(PWD) as it does not work for make -C # Not $(PWD) as it does not work for make -C
@ -47,7 +44,6 @@ all: | $(BUILT)
--define "_service_dst $(SERVICE_DST)" \ --define "_service_dst $(SERVICE_DST)" \
--define "_post_addition $(POST_ADDITION)" \ --define "_post_addition $(POST_ADDITION)" \
--define "_preun_addition $(PREUN_ADDITION)" \ --define "_preun_addition $(PREUN_ADDITION)" \
--define "_ostype -$(SYSTEM)" \
--define "_sharedstatedir /var/lib" \ --define "_sharedstatedir /var/lib" \
emqx.spec emqx.spec
mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME)

2
deploy/packages/rpm/emqx.spec
View File

@ -5,7 +5,7 @@
%define _log_dir %{_var}/log/%{_name} %define _log_dir %{_var}/log/%{_name}
%define _lib_home /usr/lib/%{_name} %define _lib_home /usr/lib/%{_name}
%define _var_home %{_sharedstatedir}/%{_name} %define _var_home %{_sharedstatedir}/%{_name}
%define _build_name_fmt %{_arch}/%{_name}%{?_ostype}-%{_version}-%{_release}.%{_arch}.rpm %define _build_name_fmt %{_arch}/%{_name}-%{_version}-%{_release}.%{_arch}.rpm
%define _build_id_links none %define _build_id_links none
Name: %{_package_name} Name: %{_package_name}

22
scripts/get-distro.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
## This script prints Linux distro name and its version number
## e.g. macos, centos8, ubuntu20.04
set -euo pipefail
if [ "$(uname -s)" = 'Darwin' ]; then
DIST='macos'
VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g')
SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')"
elif [ "$(uname -s)" = 'Linux' ]; then
if grep -q -i 'centos' /etc/*-release; then
DIST='centos'
VERSION_ID="$(rpm --eval '%{centos_ver}')"
else
DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')"
VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')"
fi
SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')"
fi
echo "$SYSTEM"