yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #6152 from Rory-Z/chore/rename-packages-name 

Chore/rename packages name
This commit is contained in:
Rory Z 2021-11-16 19:17:56 +08:00 committed by GitHub
parent c36ffc7ccb db802ad04f
commit addf0de4be
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 185 additions and 352 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.ci/build_packages/Dockerfile
View File

@ -1,4 +1,4 @@
ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
FROM ${BUILD_FROM}
ARG EMQX_NAME=emqx

2
.ci/docker-compose-file/docker-compose.yaml
View File

@ -3,7 +3,7 @@ version: '3.9'
services:
erlang:
container_name: erlang
image: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
image: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
env_file:
- conf.env
environment:

4
.ci/fvt_tests/local_relup_test_run.sh
View File

@ -15,6 +15,8 @@ PROFILE="$1"
VSN="$2"
OLD_VSN="$3"
PACKAGE_PATH="$4"
FROM_OTP_VSN="${5:-23.3.4.9-3}"
TO_OTP_VSN="${6:-23.3.4.9-3}"
TEMPDIR=$(mktemp -d)
trap '{ rm -rf -- "$TEMPDIR"; }' EXIT
@ -37,4 +39,6 @@ exec docker run \
--var ONE_MORE_EMQX_PATH="/relup_test/one_more_emqx" \
--var VSN="$VSN" \
--var OLD_VSN="$OLD_VSN" \
--var FROM_OTP_VSN="$FROM_OTP_VSN" \
--var TO_OTP_VSN="$TO_OTP_VSN" \
relup.lux

8
.ci/fvt_tests/relup.lux
View File

@ -3,6 +3,8 @@
[config var=ONE_MORE_EMQX_PATH]
[config var=VSN]
[config var=OLD_VSN]
[config var=FROM_OTP_VSN]
[config var=TO_OTP_VSN]
[config shell_cmd=/bin/bash]
[config timeout=600000]
@ -19,7 +21,7 @@
[shell emqx]
!cd $PACKAGE_PATH
!unzip -q -o $PROFILE-ubuntu20.04-$(echo $OLD_VSN | sed -r 's/[v|e]//g')-amd64.zip
!unzip -q -o $PROFILE-$(echo $OLD_VSN | sed -r 's/[v|e]//g')-otp${FROM_OTP_VSN}-ubuntu20.04-amd64.zip
?SH-PROMPT
!cd emqx
@ -80,7 +82,7 @@
!echo "" > log/emqx.log.1
?SH-PROMPT
!cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/
!cp -f ../$PROFILE-$VSN-otp${TO_OTP_VSN}-ubuntu20.04-amd64.zip releases/
!./bin/emqx install $VSN
?Made release permanent: "$VSN"
@ -105,7 +107,7 @@
!echo "" > log/emqx.log.1
?SH-PROMPT
!cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/
!cp -f ../$PROFILE-$VSN-otp${TO_OTP_VSN}-ubuntu20.04-amd64.zip releases/
!./bin/emqx install $VSN
?Made release permanent: "$VSN"

133
.github/workflows/build_packages.yaml vendored
View File

@ -11,7 +11,7 @@ on:
jobs:
prepare:
runs-on: ubuntu-20.04
container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
outputs:
profiles: ${{ steps.set_profile.outputs.profiles}}
@ -65,6 +65,8 @@ jobs:
strategy:
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
otp:
- 23.2
exclude:
- profile: emqx-edge
@ -79,7 +81,7 @@ jobs:
- uses: gleam-lang/setup-erlang@v1.1.0
id: install_erlang
with:
otp-version: 23.2
otp-version: ${{ matrix.otp }}
- name: build
env:
PYTHON: python
@ -90,10 +92,10 @@ jobs:
$version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" )
if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") {
$regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+"
$pkg_name = "${{ matrix.profile }}-windows-$([regex]::matches($version, $regex).value).zip"
$pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.zip"
}
else {
$pkg_name = "${{ matrix.profile }}-windows-$($version -replace '/').zip"
$pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip"
}
cd source
## We do not build/release bcrypt for windows package
@ -134,7 +136,7 @@ jobs:
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
erl_otp:
- 23.2.7.2-emqx-2
- 23.3.4.9-3
exclude:
- profile: emqx-edge
@ -212,6 +214,8 @@ jobs:
strategy:
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
otp:
- 23.3.4.9-3
arch:
- amd64
- arm64
@ -248,15 +252,11 @@ jobs:
shell: bash
steps:
- name: prepare docker
run: |
mkdir -p $HOME/.docker
echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json
echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
docker info
docker buildx create --use --name mybuild
docker run --rm --privileged tonistiigi/binfmt --install all
- uses: docker/setup-buildx-action@v1
- uses: docker/setup-qemu-action@v1
with:
image: tonistiigi/binfmt:latest
platforms: all
- uses: actions/download-artifact@v2
with:
name: source
@ -265,6 +265,7 @@ jobs:
run: unzip -q source.zip
- name: downloads old emqx zip packages
env:
OTP_VSN: ${{ matrix.otp }}
PROFILE: ${{ matrix.profile }}
ARCH: ${{ matrix.arch }}
SYSTEM: ${{ matrix.os }}
@ -283,42 +284,35 @@ jobs:
cd source/_upgrade_base
old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
for tag in ${old_vsns[@]}; do
if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256
echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1
package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip) | grep -oE "^[23]+")" ]; then
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip.sha256
echo "$(cat $package_name.zip.sha256) $package_name.zip" | sha256sum -c || exit 1
fi
done
- name: build emqx packages
env:
ERL_OTP: erl23.2.7.2-emqx-2
OTP: ${{ matrix.otp }}
PROFILE: ${{ matrix.profile }}
ARCH: ${{ matrix.arch }}
SYSTEM: ${{ matrix.os }}
working-directory: source
run: |
set -e -u
cd source
docker buildx build --no-cache \
--platform=linux/$ARCH \
-t cross_build_emqx_for_$SYSTEM \
-f .ci/build_packages/Dockerfile \
--build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-$SYSTEM \
--build-arg EMQX_NAME=$PROFILE \
--output type=tar,dest=/tmp/cross-build-$PROFILE-for-$SYSTEM.tar .
mkdir -p /tmp/packages/$PROFILE
tar -xvf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar --wildcards emqx/_packages/$PROFILE/*
mv emqx/_packages/$PROFILE/* /tmp/packages/$PROFILE/
rm -rf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar
docker rm -f $(docker ps -a -q)
docker volume prune -f
docker run -i --rm \
-v $(pwd):/emqx \
--workdir /emqx \
--platform linux/$ARCH \
ghcr.io/emqx/emqx-builder/4.4-2:$OTP-$SYSTEM \
bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \
make $PROFILE-pkg || cat rebar3.crashdump; \
EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh"
- name: create sha256
env:
PROFILE: ${{ matrix.profile}}
run: |
if [ -d /tmp/packages/$PROFILE ]; then
cd /tmp/packages/$PROFILE
if [ -d _packages/$PROFILE ]; then
cd _packages/$PROFILE
for var in $(ls emqx-* ); do
bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"
done
@ -328,7 +322,7 @@ jobs:
if: startsWith(github.ref, 'refs/tags/')
with:
name: ${{ matrix.profile }}
path: /tmp/packages/${{ matrix.profile }}/.
path: _packages/${{ matrix.profile }}/.
docker:
runs-on: ubuntu-20.04
@ -338,6 +332,8 @@ jobs:
strategy:
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
otp:
- 23.3.4.9-3
arch:
- [amd64, x86_64]
- [arm64v8, aarch64]
@ -357,22 +353,42 @@ jobs:
path: .
- name: unzip source code
run: unzip -q source.zip
- name: build emqx docker image
env:
PROFILE: ${{ matrix.profile }}
ARCH: ${{ matrix.arch[0] }}
QEMU_ARCH: ${{ matrix.arch[1] }}
run: |
sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
cd source
sudo TARGET=emqx/$PROFILE ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH make docker
cd _packages/$PROFILE && for var in $(ls ${PROFILE}-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
- uses: actions/upload-artifact@v1
if: startsWith(github.ref, 'refs/tags/')
- uses: docker/setup-buildx-action@v1
- uses: docker/setup-qemu-action@v1
with:
name: ${{ matrix.profile }}
path: source/_packages/${{ matrix.profile }}/.
image: tonistiigi/binfmt:latest
platforms: all
- uses: docker/metadata-action@v3
id: meta
with:
images: ${{ github.repository_owner }}/${{ matrix.profile }}
flavor: |
latest=${{ !github.event.release.prerelease }}
tags: |
type=ref,event=branch
type=ref,event=pr
type=ref,event=tag
type=semver,pattern={{version}}-otp${{ matrix.otp }}
type=semver,pattern={{major}}.{{minor}}-otp${{ matrix.otp }}
- uses: docker/login-action@v1
if: github.event_name == 'release'
with:
username: ${{ secrets.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_HUB_TOKEN }}
- uses: docker/build-push-action@v2
with:
push: ${{ github.event_name == 'release' }}
pull: true
no-cache: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-2:${{ matrix.otp }}-alpine3.14
RUN_FROM=alpine:3.14
EMQX_NAME=${{ matrix.profile }}
file: source/deploy/docker/Dockerfile
context: source
delete-artifact:
runs-on: ubuntu-20.04
@ -452,15 +468,6 @@ jobs:
-X POST \
-d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
${{ secrets.EMQX_IO_RELEASE_API }}
- name: push docker image to docker hub
if: github.event_name == 'release'
run: |
set -e -x -u
sudo make docker-prepare
cd _packages/${{ matrix.profile }} && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd -
echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin
sudo TARGET=emqx/${{ matrix.profile }} make docker-push
sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list
- name: update repo.emqx.io
if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee'
run: |

6
.github/workflows/build_slim_packages.yaml vendored
View File

@ -15,12 +15,12 @@ jobs:
strategy:
matrix:
erl_otp:
- erl23.2.7.2-emqx-2
- 23.3.4.9-3
os:
- ubuntu20.04
- centos7
container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }}
container: ghcr.io/emqx/emqx-builder/4.4-2:${{ matrix.erl_otp }}-${{ matrix.os }}
steps:
- uses: actions/checkout@v1
@ -58,7 +58,7 @@ jobs:
strategy:
matrix:
erl_otp:
- 23.2.7.2-emqx-2
- 23.3.4.9-3
steps:
- uses: actions/checkout@v1

2
.github/workflows/check_deps_integrity.yaml vendored
View File

@ -5,7 +5,7 @@ on: [pull_request]
jobs:
check_deps_integrity:
runs-on: ubuntu-20.04
container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
steps:
- uses: actions/checkout@v2

2
.github/workflows/run_acl_migration_tests.yaml vendored
View File

@ -5,7 +5,7 @@ on: workflow_dispatch
jobs:
test:
runs-on: ubuntu-20.04
container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
strategy:
fail-fast: true
env:

56
.github/workflows/run_automate_tests.yaml vendored
View File

@ -12,12 +12,12 @@ jobs:
build:
runs-on: ubuntu-latest
outputs:
imgname: ${{ steps.build_docker.outputs.imgname}}
version: ${{ steps.build_docker.outputs.version}}
imgname: ${{ steps.prepare.outputs.imgname}}
version: ${{ steps.prepare.outputs.version}}
steps:
- uses: actions/checkout@v2
- name: build docker
id: build_docker
- name: prepare
id: prepare
run: |
if [ -f EMQX_ENTERPRISE ]; then
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
@ -25,18 +25,22 @@ jobs:
echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
make deps-emqx-ee
make clean
fi
make docker
echo "::set-output name=version::$(./pkg-vsn.sh)"
if [ -f EMQX_ENTERPRISE ]; then
echo "::set-output name=imgname::emqx-ee"
echo "::set-output name=version::$(./pkg-vsn.sh)"
else
echo "::set-output name=imgname::emqx"
echo "::set-output name=version::$(./pkg-vsn.sh)"
fi
- name: build docker image
env:
OTP_VSN: 23.3.4.9-3
run: |
make ${{ steps.prepare.outputs.imgname }}-docker
docker save emqx/${{ steps.prepare.outputs.imgname }}:${{ steps.prepare.outputs.version }} -o image.tar.gz
- uses: actions/upload-artifact@v2
with:
name: emqx-docker-image-zip
path: _packages/${{ steps.build_docker.outputs.imgname }}/${{ steps.build_docker.outputs.imgname }}-docker-${{ steps.build_docker.outputs.version }}.zip
name: image
path: image.tar.gz
webhook:
runs-on: ubuntu-latest
@ -52,15 +56,11 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/download-artifact@v2
with:
name: emqx-docker-image-zip
name: image
path: /tmp
- name: load docker image
env:
imgname: ${{ needs.build.outputs.imgname}}
version: ${{ needs.build.outputs.version }}
run: |
unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp
docker load < /tmp/${imgname}-docker-${version}
docker load < /tmp/image.tar.gz
- name: docker compose up
timeout-minutes: 5
env:
@ -152,15 +152,11 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/download-artifact@v2
with:
name: emqx-docker-image-zip
name: image
path: /tmp
- name: load docker image
env:
imgname: ${{ needs.build.outputs.imgname }}
version: ${{ needs.build.outputs.version }}
run: |
unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp
docker load < /tmp/${imgname}-docker-${version}
docker load < /tmp/image.tar.gz
- name: docker compose up
timeout-minutes: 5
env:
@ -259,15 +255,11 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/download-artifact@v2
with:
name: emqx-docker-image-zip
name: image
path: /tmp
- name: load docker image
env:
imgname: ${{ needs.build.outputs.imgname }}
version: ${{ needs.build.outputs.version }}
run: |
unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp
docker load < /tmp/${imgname}-docker-${version}
docker load < /tmp/image.tar.gz
- name: docker compose up
timeout-minutes: 5
env:
@ -355,15 +347,11 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/download-artifact@v2
with:
name: emqx-docker-image-zip
name: image
path: /tmp
- name: load docker image
env:
imgname: ${{ needs.build.outputs.imgname }}
version: ${{ needs.build.outputs.version }}
run: |
unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp
docker load < /tmp/${imgname}-docker-${version}
docker load < /tmp/image.tar.gz
- name: docker compose up
timeout-minutes: 5
env:

35
.github/workflows/run_fvt_tests.yaml vendored
View File

@ -13,10 +13,6 @@ jobs:
steps:
- uses: actions/checkout@v1
- uses: gleam-lang/setup-erlang@v1.1.2
id: install_erlang
with:
otp-version: 23.2
- name: prepare
run: |
if make emqx-ee --dry-run > /dev/null 2>&1; then
@ -24,14 +20,19 @@ jobs:
git config --global credential.helper store
echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
make deps-emqx-ee
make clean
echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
echo "PROFILE=emqx-ee" >> $GITHUB_ENV
echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
else
echo "TARGET=emqx/emqx" >> $GITHUB_ENV
echo "PROFILE=emqx" >> $GITHUB_ENV
echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
fi
- name: make emqx image
run: make docker
env:
OTP_VSN: 23.3.4.9-3
run: make ${PROFILE}-docker
- name: run emqx
timeout-minutes: 5
run: |
@ -67,10 +68,6 @@ jobs:
steps:
- uses: actions/checkout@v1
- uses: gleam-lang/setup-erlang@v1.1.2
id: install_erlang
with:
otp-version: 23.2
- name: prepare
run: |
if make emqx-ee --dry-run > /dev/null 2>&1; then
@ -78,12 +75,19 @@ jobs:
git config --global credential.helper store
echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
make deps-emqx-ee
make clean
echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
echo "PROFILE=emqx-ee" >> $GITHUB_ENV
echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
else
echo "TARGET=emqx/emqx" >> $GITHUB_ENV
echo "PROFILE=emqx" >> $GITHUB_ENV
echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
fi
- name: make emqx image
run: make docker
env:
OTP_VSN: 23.3.4.9-3
run: make ${PROFILE}-docker
- name: install k3s
env:
KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
@ -105,11 +109,10 @@ jobs:
KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
timeout-minutes: 5
run: |
version=$(./pkg-vsn.sh)
sudo docker save ${TARGET}:$version -o emqx.tar.gz
sudo docker save ${TARGET}:${EMQX_TAG} -o emqx.tar.gz
sudo k3s ctr image import emqx.tar.gz
sed -i -r "s/^appVersion: .*$/appVersion: \"${version}\"/g" deploy/charts/emqx/Chart.yaml
sed -i -r "s/^appVersion: .*$/appVersion: \"${EMQX_TAG}\"/g" deploy/charts/emqx/Chart.yaml
sed -i '/emqx_telemetry/d' deploy/charts/emqx/values.yaml
helm install emqx \
@ -180,7 +183,7 @@ jobs:
relup_test_plan:
runs-on: ubuntu-20.04
container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
outputs:
profile: ${{ steps.profile-and-versions.outputs.profile }}
vsn: ${{ steps.profile-and-versions.outputs.vsn }}
@ -229,7 +232,7 @@ jobs:
relup_test_build:
needs: relup_test_plan
runs-on: ubuntu-20.04
container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
defaults:
run:
shell: bash
@ -316,6 +319,8 @@ jobs:
--var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \
--var VSN="$VSN" \
--var OLD_VSN="$OLD_VSN" \
--var FROM_OTP_VSN="23.3.4.9-3" \
--var TO_OTP_VSN="23.3.4.9-3" \
emqx_built/.ci/fvt_tests/relup.lux
- uses: actions/upload-artifact@v2
name: Save debug data

4
.github/workflows/run_test_cases.yaml vendored
View File

@ -10,7 +10,7 @@ on:
jobs:
run_static_analysis:
runs-on: ubuntu-20.04
container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
steps:
- uses: actions/checkout@v2
@ -27,7 +27,7 @@ jobs:
run_proper_test:
runs-on: ubuntu-20.04
container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
steps:
- uses: actions/checkout@v2

2
.tool-versions
View File

@ -1 +1 @@
erlang 23.2.7.2-emqx-2
erlang 23.3.4.9-3

12
Makefile
View File

@ -3,6 +3,7 @@ REBAR_VERSION = 3.14.3-emqx-8
REBAR = $(CURDIR)/rebar3
BUILD = $(CURDIR)/build
SCRIPTS = $(CURDIR)/scripts
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
export EMQX_DESC ?= EMQ X
export EMQX_CE_DASHBOARD_VERSION ?= v4.3.3
@ -144,11 +145,18 @@ $1: $1-rel
endef
$(foreach pt,$(PKG_PROFILES),$(eval $(call gen-pkg-target,$(pt))))
## docker target is to create docker instructions
.PHONY: $(REL_PROFILES:%=%-docker)
define gen-docker-target
$1-docker: $(COMMON_DEPS)
@$(BUILD) $1 docker
endef
ALL_ZIPS = $(REL_PROFILES)
$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt))))
.PHONY: run
run: $(PROFILE) quickrun
.PHONY: quickrun
quickrun:
./_build/$(PROFILE)/rel/emqx/bin/emqx console
include docker.mk

27
build
View File

@ -76,7 +76,7 @@ make_relup() {
rm -rf "$tmp_dir"
fi
releases+=( "$base_vsn" )
done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-$SYSTEM*-$ARCH.zip" -type f)
done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-otp${OTP_VSN}-$SYSTEM*-$ARCH.zip" -type f)
fi
if [ ${#releases[@]} -eq 0 ]; then
log "No upgrade base found, relup ignored"
@ -116,7 +116,7 @@ make_zip() {
log "ERROR: $tarball is not found"
fi
local zipball
zipball="${pkgpath}/${PROFILE}-${SYSTEM}-${PKG_VSN}-${ARCH}.zip"
zipball="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip"
tar zxf "${tarball}" -C "${tard}/emqx"
## try to be portable for zip packages.
## for DEB and RPM packages the dependencies are resoved by yum and apt
@ -124,6 +124,26 @@ make_zip() {
(cd "${tard}" && zip -qr - emqx) > "${zipball}"
}
make_docker() {
EMQX_RUNNER_IMAGE='alpine:3.14'
EMQX_RUNNER_IMAGE_COMPACT="$(echo $EMQX_RUNNER_IMAGE | tr -d ':')"
EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-2:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}"
## Build Docker image
echo "DOCKER BUILD: Build Docker image."
echo "DOCKER BUILD: docker repo -> emqx/$PROFILE "
echo "DOCKER BUILD: build version -> $PKG_VSN."
echo "DOCKER BUILD: build from -> $EMQX_BUILDER."
echo "DOCKER BUILD: runner from -> $EMQX_RUNNER_IMAGE."
docker build --no-cache --pull \
--build-arg BUILD_FROM="${EMQX_BUILDER}" \
--build-arg RUN_FROM="${EMQX_RUNNER_IMAGE}" \
--build-arg EMQX_NAME="$PROFILE" \
--tag "emqx/$PROFILE:$PKG_VSN" \
-f deploy/docker/Dockerfile .
}
log "building artifact=$ARTIFACT for profile=$PROFILE"
case "$ARTIFACT" in
@ -144,6 +164,9 @@ case "$ARTIFACT" in
make -C "deploy/packages/${PKGERDIR}" clean
EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}"
;;
docker)
make_docker
;;
*)
log "Unknown artifact $ARTIFACT"
exit 1

21
deploy/docker/Dockerfile
View File

@ -1,10 +1,7 @@
ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine-amd64
ARG RUN_FROM=alpine:3.12
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14
ARG RUN_FROM=alpine:3.14
FROM ${BUILD_FROM} AS builder
ARG QEMU_ARCH=x86_64
COPY tmp/qemu-$QEMU_ARCH-stati* /usr/bin/
RUN apk add --no-cache \
git \
curl \
@ -32,21 +29,9 @@ RUN cd /emqx \
FROM $RUN_FROM
# Basic build-time metadata as defined at http://label-schema.org
LABEL org.label-schema.docker.dockerfile="Dockerfile" \
org.label-schema.license="GNU" \
org.label-schema.name="emqx" \
org.label-schema.version=${PKG_VSN} \
org.label-schema.description="EMQ (Erlang MQTT Broker) is a distributed, massively scalable, highly extensible MQTT messaging broker written in Erlang/OTP." \
org.label-schema.url="https://emqx.io" \
org.label-schema.vcs-type="Git" \
org.label-schema.vcs-url="https://github.com/emqx/emqx" \
maintainer="EMQ X Team <support@emqx.io>"
ARG QEMU_ARCH=x86_64
ARG EMQX_NAME=emqx
COPY deploy/docker/docker-entrypoint.sh tmp/qemu-$QEMU_ARCH-stati* /usr/bin/
COPY deploy/docker/docker-entrypoint.sh /usr/bin/
COPY --from=builder /emqx/_build/$EMQX_NAME/rel/emqx /opt/emqx
RUN ln -s /opt/emqx/bin/* /usr/local/bin/

2
deploy/packages/deb/Makefile
View File

@ -8,7 +8,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz
SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture)
TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH)
TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH)
.PHONY: all
all: | $(BUILT)

2
deploy/packages/deb/debian/control
View File

@ -4,7 +4,7 @@ Priority: optional
Maintainer: emqx <contact@emqx.io>
Build-Depends: debhelper (>=9)
Standards-Version: 3.9.6
Homepage: https://www.emqx.io
Homepage: https://www.emqx.com
Package: emqx
Architecture: any

9
deploy/packages/rpm/Makefile
View File

@ -6,7 +6,7 @@ dash := -
none :=
space := $(none) $(none)
RPM_VSN ?= $(shell echo $(PKG_VSN) | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+)?")
RPM_REL ?= $(shell echo $(PKG_VSN) | grep -oE "(alpha|beta|rc)\.[0-9]")
RPM_REL ?= $(shell echo $(PKG_VSN) | grep -oE "(alpha|beta|rc)\.[0-9]+")
ARCH ?= amd64
ifeq ($(ARCH),mips64)
@ -16,12 +16,12 @@ endif
EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz
TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH)
TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH)
ifeq ($(RPM_REL),)
# no tail
RPM_REL := 1
endif
SOURCE_PKG := emqx-$(SYSTEM)-$(RPM_VSN)-$(RPM_REL).$(shell uname -m)
SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL)-otp$(OTP_VSN)-$(SYSTEM).$(shell uname -m)
SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi)
# Not $(PWD) as it does not work for make -C
@ -47,7 +47,8 @@ all: | $(BUILT)
--define "_service_dst $(SERVICE_DST)" \
--define "_post_addition $(POST_ADDITION)" \
--define "_preun_addition $(PREUN_ADDITION)" \
--define "_ostype -$(SYSTEM)" \
--define "_ostype $(SYSTEM)" \
--define "_otp_vsn $(OTP_VSN)" \
--define "_sharedstatedir /var/lib" \
emqx.spec
mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME)

2
deploy/packages/rpm/emqx.spec
View File

@ -5,7 +5,7 @@
%define _log_dir %{_var}/log/%{_name}
%define _lib_home /usr/lib/%{_name}
%define _var_home %{_sharedstatedir}/%{_name}
%define _build_name_fmt %{_arch}/%{_name}%{?_ostype}-%{_version}-%{_release}.%{_arch}.rpm
%define _build_name_fmt %{_arch}/%{_name}-%{_version}-%{_release}-otp%{_otp_vsn}-%{?_ostype}.%{_arch}.rpm
%define _build_id_links none
Name: %{_package_name}

188
docker.mk
View File

@ -1,188 +0,0 @@
#!/usr/bin/make -f
# -*- makefile -*-
## default globals.
## when built with `make docker` command the default profile is either emqx or emqx-ee (for enterprise)
## or the TARGET varialbe can be set beforehand to force a different name
TARGET ?= emqx/$(PROFILE)
QEMU_ARCH ?= x86_64
ARCH ?= amd64
QEMU_VERSION ?= v5.0.0-2
OS ?= alpine
export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
ifeq ($(findstring emqx-ee, $(TARGET)), emqx-ee)
ARCH_LIST := amd64 arm64v8 arm32v7
EMQX_NAME := emqx-ee
else ifeq ($(findstring emqx-edge, $(TARGET)), emqx-edge)
ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x
EMQX_NAME := emqx-edge
else
ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x
EMQX_NAME := emqx
endif
.PHONY: docker
docker: docker-build docker-tag docker-save
.PHONY: docker-prepare
docker-prepare:
## Prepare the machine before any code installation scripts
# @echo "PREPARE: Setting up dependencies."
# @apt update -y
# @apt install --only-upgrade docker-ce -y
## Update docker configuration to enable docker manifest command
@echo "PREPARE: Updating docker configuration"
@mkdir -p $$HOME/.docker
# enable experimental to use docker manifest command
@echo '{ "experimental": "enabled" }' | tee $$HOME/.docker/config.json
# enable experimental
@echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50 }' | tee /etc/docker/daemon.json
@service docker restart
.PHONY: docker-build
docker-build:
## Build Docker image
@echo "DOCKER BUILD: Build Docker image."
@echo "DOCKER BUILD: build version -> $(PKG_VSN)."
@echo "DOCKER BUILD: arch - $(ARCH)."
@echo "DOCKER BUILD: qemu arch - $(QEMU_ARCH)."
@echo "DOCKER BUILD: docker repo - $(TARGET) "
@echo "DOCKER BUILD: emqx name - $(EMQX_NAME)."
## Prepare qemu to build images other then x86_64 on travis
@echo "PREPARE: Qemu" \
&& docker run --rm --privileged multiarch/qemu-user-static:register --reset
@mkdir -p tmp \
&& cd tmp \
&& curl -L -o qemu-$(QEMU_ARCH)-static.tar.gz https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/qemu-$(QEMU_ARCH)-static.tar.gz \
&& tar xzf qemu-$(QEMU_ARCH)-static.tar.gz \
&& cd -
@docker build --no-cache \
--build-arg PKG_VSN=$(PKG_VSN) \
--build-arg BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine-$(ARCH) \
--build-arg RUN_FROM=$(ARCH)/alpine:3.12 \
--build-arg EMQX_NAME=$(EMQX_NAME) \
--build-arg QEMU_ARCH=$(QEMU_ARCH) \
--tag $(TARGET):build-$(OS)-$(ARCH) \
-f deploy/docker/Dockerfile .
.PHONY: docker-tag
docker-tag:
@echo "DOCKER TAG: Tag Docker image."
@for arch in $(ARCH_LIST); do \
if [ -n "$$(docker images -q $(TARGET):build-$(OS)-$${arch})" ]; then \
docker tag $(TARGET):build-$(OS)-$${arch} $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \
echo "DOCKER TAG: $(TARGET):$(PKG_VSN)-$(OS)-$${arch}"; \
if [ $${arch} = amd64 ]; then \
docker tag $(TARGET):$(PKG_VSN)-$(OS)-amd64 $(TARGET):$(PKG_VSN); \
echo "DOCKER TAG: $(TARGET):$(PKG_VSN)"; \
fi; \
fi; \
done
.PHONY: docker-save
docker-save:
@echo "DOCKER SAVE: Save Docker image."
@mkdir -p _packages/$(EMQX_NAME)
@if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \
docker save $(TARGET):$(PKG_VSN) > $(EMQX_NAME)-docker-$(PKG_VSN); \
zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN).zip $(EMQX_NAME)-docker-$(PKG_VSN); \
mv ./$(EMQX_NAME)-docker-$(PKG_VSN).zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN).zip; \
fi
@for arch in $(ARCH_LIST); do \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker save $(TARGET):$(PKG_VSN)-$(OS)-$${arch} > $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \
zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \
mv ./$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip; \
fi; \
done
.PHONY: docker-push
docker-push:
@echo "DOCKER PUSH: Push Docker image.";
@echo "DOCKER PUSH: pushing - $(TARGET):$(PKG_VSN).";
@if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \
docker push $(TARGET):$(PKG_VSN); \
docker tag $(TARGET):$(PKG_VSN) $(TARGET):latest; \
docker push $(TARGET):latest; \
fi;
@for arch in $(ARCH_LIST); do \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker push $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \
fi; \
done
.PHONY: docker-manifest-list
docker-manifest-list:
version="docker manifest create --amend $(TARGET):$(PKG_VSN)"; \
latest="docker manifest create --amend $(TARGET):latest"; \
for arch in $(ARCH_LIST); do \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ];then \
version="$${version} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \
latest="$${latest} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \
fi; \
done; \
eval $$version; \
eval $$latest;
for arch in $(ARCH_LIST); do \
case $${arch} in \
"amd64") \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \
docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \
fi; \
;; \
"arm64v8") \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \
docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \
fi; \
;; \
"arm32v7") \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \
docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \
fi; \
;; \
"i386") \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \
docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \
fi; \
;; \
"s390x") \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \
docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \
fi; \
;; \
esac; \
done;
docker manifest inspect $(TARGET):$(PKG_VSN)
docker manifest push $(TARGET):$(PKG_VSN);
docker manifest inspect $(TARGET):latest
docker manifest push $(TARGET):latest;
.PHONY: docker-clean
docker-clean:
@echo "DOCKER CLEAN: Clean Docker image."
@if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then docker rmi -f $$(docker images -q $(TARGET):$(PKG_VSN)); fi
@for arch in $(ARCH_LIST); do \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker rmi -f $$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch}); \
fi \
done

5
scripts/get-otp-vsn.sh Executable file
View File

@ -0,0 +1,5 @@
#!/bin/bash
set -euo pipefail
erl -noshell -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().'

13
src/emqx_misc.erl
View File

@ -66,16 +66,9 @@ maybe_parse_ip(Host) ->
%% @doc Add `ipv6_probe' socket option if it's supported.
ipv6_probe(Opts) ->
case persistent_term:get({?MODULE, ipv6_probe_supported}, unknown) of
unknown ->
%% e.g. 23.2.7.1-emqx-2-x86_64-unknown-linux-gnu-64
OtpVsn = emqx_vm:get_otp_version(),
Bool = (match =:= re:run(OtpVsn, "emqx", [{capture, none}])),
_ = persistent_term:put({?MODULE, ipv6_probe_supported}, Bool),
ipv6_probe(Bool, Opts);
Bool ->
ipv6_probe(Bool, Opts)
end.
Bool = try gen_tcp:ipv6_probe()
catch _ : _ -> false end,
ipv6_probe(Bool, Opts).
ipv6_probe(false, Opts) -> Opts;
ipv6_probe(true, Opts) -> [{ipv6_probe, true} | Opts].