yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #6636 from emqx/elixir-packages-mkII 

ci(mix): build emqx packages with elixir
This commit is contained in:
Thales Macedo Garitezi 2022-01-27 11:10:30 -03:00 committed by GitHub
parent 8378f52d06 12f1757328
commit bd0d331b64
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 505 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
.ci/build_packages/tests.sh
View File

@ -8,13 +8,19 @@ if [ -z "${1:-}" ]; then
exit 1 exit 1
fi fi
if [ "${2:-}" != 'tgz' ] && [ "${2:-}" != 'pkg' ]; then case "${2:-}" in
tgz|pkg)
true
;;
*)
echo "Usage $0 <PACKAGE_NAME> tgz|pkg" echo "Usage $0 <PACKAGE_NAME> tgz|pkg"
exit 1 exit 1
fi ;;
esac
PACKAGE_NAME="${1}" PACKAGE_NAME="${1}"
PACKAGE_TYPE="${2}" PACKAGE_TYPE="${2}"
ARCH="${3}"
export DEBUG=1 export DEBUG=1
export CODE_PATH=${CODE_PATH:-"/emqx"} export CODE_PATH=${CODE_PATH:-"/emqx"}
@ -43,7 +49,9 @@ if ! [ -f "$PACKAGE_FILE" ]; then
exit 1 exit 1
fi fi
case "$(uname -m)" in if [ -z "$ARCH" ]
then
case "$(uname -m)" in
x86_64) x86_64)
ARCH='amd64' ARCH='amd64'
;; ;;
@ -53,7 +61,8 @@ case "$(uname -m)" in
arm*) arm*)
ARCH=arm ARCH=arm
;; ;;
esac esac
fi
export ARCH export ARCH
emqx_prepare(){ emqx_prepare(){

4
.ci/docker-compose-file/docker-compose.yaml
View File

@ -3,7 +3,7 @@ version: '3.9'
services: services:
erlang23: erlang23:
container_name: erlang23 container_name: erlang23
image: ghcr.io/emqx/emqx-builder/5.0-3:23.3.4.9-3-ubuntu20.04 image: ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-23.3.4.9-4-ubuntu20.04
env_file: env_file:
- conf.env - conf.env
environment: environment:
@ -23,7 +23,7 @@ services:
erlang24: erlang24:
container_name: erlang24 container_name: erlang24
image: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04 image: ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-ubuntu20.04
env_file: env_file:
- conf.env - conf.env
environment: environment:

16
.ci/docker-compose-file/scripts/run-emqx.sh
View File

@ -19,17 +19,23 @@ fi
} >> .ci/docker-compose-file/conf.cluster.env } >> .ci/docker-compose-file/conf.cluster.env
is_node_up() { is_node_up() {
local node local node="$1"
node="$1" if [ "${IS_ELIXIR:-no}" = "yes" ]
then
docker exec -i "$node" \
bash -c "emqx eval \"[:\\\"emqx@node1.emqx.io\\\", :\\\"emqx@node2.emqx.io\\\"] = :ekka_cluster.info()[:running_nodes]\""
else
docker exec -i "$node" \ docker exec -i "$node" \
bash -c "emqx eval \"['emqx@node1.emqx.io','emqx@node2.emqx.io'] = maps:get(running_nodes, ekka_cluster:info()).\"" > /dev/null 2>&1 bash -c "emqx eval \"['emqx@node1.emqx.io','emqx@node2.emqx.io'] = maps:get(running_nodes, ekka_cluster:info()).\"" > /dev/null 2>&1
fi
} }
is_node_listening() { is_node_listening() {
local node local node="$1"
node="$1"
docker exec -i "$node" \ docker exec -i "$node" \
emqx eval "ok = case gen_tcp:connect(\"localhost\", 1883, []) of {ok, P} -> gen_tcp:close(P), ok; _ -> exit(1) end." > /dev/null 2>&1 emqx ctl listeners | \
grep -A6 'tcp:default' | \
grep -qE 'running *: true'
} }
is_cluster_up() { is_cluster_up() {

138
.github/workflows/build_packages.yaml vendored
View File

@ -19,7 +19,7 @@ jobs:
prepare: prepare:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
# prepare source with any OTP version, no need for a matrix # prepare source with any OTP version, no need for a matrix
container: "ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-ubuntu20.04"
outputs: outputs:
ce_old_vsns: ${{ steps.find_old_versons.outputs.ce_old_vsns }} ce_old_vsns: ${{ steps.find_old_versons.outputs.ce_old_vsns }}
@ -130,7 +130,7 @@ jobs:
- emqx - emqx
- emqx-enterprise - emqx-enterprise
otp: otp:
- 24.1.5-3 - 24.1.5-4
macos: macos:
- macos-11 - macos-11
- macos-10.15 - macos-10.15
@ -236,7 +236,18 @@ jobs:
- emqx - emqx
- emqx-enterprise - emqx-enterprise
otp: otp:
- 24.1.5-3 # we test with OTP 23, but only build package on OTP 24 versions - 24.1.5-4 # we test with OTP 23, but only build package on OTP 24 versions
elixir:
- 1.13.2
# used to split elixir packages into a separate job, since the
# entire job may take a lot of time, especially on arm64
# emulation.
# we only want to build ubuntu and centos with elixir for the
# time being, so it's easier to just include those with
# `with_elixir` set.
build_elixir:
# - with_elixir
- no_elixir
arch: arch:
- amd64 - amd64
- arm64 - arm64
@ -264,6 +275,19 @@ jobs:
profile: emqx-enterprise profile: emqx-enterprise
- os: raspbian10 - os: raspbian10
profile: emqx-enterprise profile: emqx-enterprise
include:
- profile: emqx
otp: 24.1.5-4
elixir: 1.13.2
arch: amd64
build_elixir: with_elixir
os: ubuntu20.04
- profile: emqx
otp: 24.1.5-4
elixir: 1.13.2
arch: amd64
build_elixir: with_elixir
os: centos8
defaults: defaults:
run: run:
@ -289,12 +313,16 @@ jobs:
- name: load rocksdb cache - name: load rocksdb cache
uses: actions/cache@v2 uses: actions/cache@v2
with: with:
path: source/_build/default/lib/rocksdb/ path: |
source/_build/default/lib/rocksdb/
source/deps/rocksdb/
key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }} key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
- name: load quicer cache - name: load quicer cache
uses: actions/cache@v2 uses: actions/cache@v2
with: with:
path: source/_build/default/lib/quicer/ path: |
source/_build/default/lib/quicer/
source/deps/quicer/
key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
- name: download old emqx tgz packages - name: download old emqx tgz packages
env: env:
@ -334,22 +362,51 @@ jobs:
- name: build emqx packages - name: build emqx packages
env: env:
OTP: ${{ matrix.otp }} OTP: ${{ matrix.otp }}
ELIXIR: ${{ matrix.elixir }}
PROFILE: ${{ matrix.profile }} PROFILE: ${{ matrix.profile }}
ARCH: ${{ matrix.arch }} ARCH: ${{ matrix.arch }}
SYSTEM: ${{ matrix.os }} SYSTEM: ${{ matrix.os }}
if: ${{ matrix.build_elixir == 'no_elixir' }}
working-directory: source working-directory: source
run: | run: |
./scripts/buildx.sh \ ./scripts/buildx.sh \
--profile "${PROFILE}" \ --profile "${PROFILE}" \
--pkgtype "tgz" \ --pkgtype "tgz" \
--arch "${ARCH}" \ --arch "${ARCH}" \
--builder "ghcr.io/emqx/emqx-builder/5.0-3:${OTP}-${SYSTEM}" --otp "${OTP}" \
--elixir "${ELIXIR}" \
--system "${SYSTEM}" \
--builder "ghcr.io/emqx/emqx-builder/5.0-5:${ELIXIR}-${OTP}-${SYSTEM}"
## the pkg build is incremental on the tgz build ## the pkg build is incremental on the tgz build
./scripts/buildx.sh \ ./scripts/buildx.sh \
--profile "${PROFILE}" \ --profile "${PROFILE}" \
--pkgtype "pkg" \ --pkgtype "pkg" \
--arch "${ARCH}" \ --arch "${ARCH}" \
--builder "ghcr.io/emqx/emqx-builder/5.0-3:${OTP}-${SYSTEM}" --otp "${OTP}" \
--elixir "${ELIXIR}" \
--system "${SYSTEM}" \
--builder "ghcr.io/emqx/emqx-builder/5.0-5:${ELIXIR}-${OTP}-${SYSTEM}"
- name: build emqx packages (Elixir)
env:
OTP: ${{ matrix.otp }}
ELIXIR: ${{ matrix.elixir }}
PROFILE: ${{ matrix.profile }}
ARCH: ${{ matrix.arch }}
SYSTEM: ${{ matrix.os }}
working-directory: source
if: ${{ matrix.build_elixir == 'with_elixir' }}
run: |
## we currently only build tgzs for elixir
./scripts/buildx.sh \
--profile "${PROFILE}" \
--pkgtype "tgz" \
--arch "${ARCH}" \
--otp "${OTP}" \
--elixir "${ELIXIR}" \
--system "${SYSTEM}" \
--with-elixir \
--builder "ghcr.io/emqx/emqx-builder/5.0-5:${ELIXIR}-${OTP}-${SYSTEM}"
- name: create sha256 - name: create sha256
env: env:
@ -380,9 +437,23 @@ jobs:
- emqx-edge - emqx-edge
- emqx - emqx
- emqx-enterprise - emqx-enterprise
# NOTE: for docker, only support latest otp version, not a matrix # NOTE: for docker, only support latest otp and elixir
# versions, not a matrix
otp: otp:
- 24.1.5-3 # update to latest - 24.1.5-4 # update to latest
elixir:
- 1.13.2 # update to latest
arch:
- amd64
- arm64
build_elixir:
- no_elixir
include:
- profile: emqx
otp: 24.1.5-4
elixir: 1.13.2
arch: amd64
build_elixir: with_elixir
steps: steps:
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v2
@ -398,6 +469,7 @@ jobs:
platforms: all platforms: all
- uses: docker/metadata-action@v3 - uses: docker/metadata-action@v3
id: meta id: meta
if: ${{ matrix.build_elixir == 'no_elixir' }}
with: with:
images: ${{ github.repository_owner }}/${{ matrix.profile }} images: ${{ github.repository_owner }}/${{ matrix.profile }}
flavor: | flavor: |
@ -409,12 +481,30 @@ jobs:
type=semver,pattern={{version}} type=semver,pattern={{version}}
labels: labels:
org.opencontainers.image.otp.version=${{ matrix.otp }} org.opencontainers.image.otp.version=${{ matrix.otp }}
- name: docker metadata for elixir image
uses: docker/metadata-action@v3
if: ${{ matrix.build_elixir == 'with_elixir' }}
id: meta-elixir
with:
images: ${{ github.repository_owner }}/${{ matrix.profile }}
flavor: |
latest=${{ !github.event.release.prerelease }}
suffix=-elixir
tags: |
type=ref,event=branch
type=ref,event=pr
type=ref,event=tag
type=semver,pattern={{version}}
labels: |
org.opencontainers.image.otp.version=${{ matrix.otp }}
org.opencontainers.image.elixir.version=${{ matrix.elixir }}
- uses: docker/login-action@v1 - uses: docker/login-action@v1
if: github.event_name == 'release' if: github.event_name == 'release'
with: with:
username: ${{ secrets.DOCKER_HUB_USER }} username: ${{ secrets.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_HUB_TOKEN }} password: ${{ secrets.DOCKER_HUB_TOKEN }}
- uses: docker/build-push-action@v2 - uses: docker/build-push-action@v2
if: ${{ matrix.build_elixir == 'no_elixir' }}
with: with:
push: ${{ github.event_name == 'release' && !github.event.release.prerelease }} push: ${{ github.event_name == 'release' && !github.event.release.prerelease }}
pull: true pull: true
@ -423,11 +513,27 @@ jobs:
tags: ${{ steps.meta.outputs.tags }} tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }} labels: ${{ steps.meta.outputs.labels }}
build-args: | build-args: |
BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-alpine3.14 BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-alpine3.14
RUN_FROM=alpine:3.14 RUN_FROM=alpine:3.14
EMQX_NAME=${{ matrix.profile }} EMQX_NAME=${{ matrix.profile }}
file: source/deploy/docker/Dockerfile file: source/deploy/docker/Dockerfile
context: source context: source
- name: build docker image with elixir
uses: docker/build-push-action@v2
if: ${{ matrix.profile == 'emqx' && matrix.build_elixir == 'with_elixir' }}
with:
push: ${{ github.event_name == 'release' && !github.event.release.prerelease }}
pull: true
no-cache: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta-elixir.outputs.tags }}
labels: ${{ steps.meta-elixir.outputs.labels }}
build-args: |
BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-alpine3.14
RUN_FROM=alpine:3.14
EMQX_NAME=emqx-elixir
file: source/deploy/docker/Dockerfile
context: source
- uses: aws-actions/configure-aws-credentials@v1 - uses: aws-actions/configure-aws-credentials@v1
if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx'
with: with:
@ -435,13 +541,21 @@ jobs:
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_DEFAULT_REGION }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
- name: Push image to aws ecr - name: Push image to aws ecr
if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' && matrix.build_elixir == 'no_elixir'
run: | run: |
version=${GITHUB_REF##*/} version=${GITHUB_REF##*/}
docker pull emqx/emqx:${version#v} docker pull emqx/emqx:${version#v}
docker tag emqx/emqx:${version#v} public.ecr.aws/emqx/emqx:${version#v} docker tag emqx/emqx:${version#v} public.ecr.aws/emqx/emqx:${version#v}
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
docker push public.ecr.aws/emqx/emqx:${version#v} docker push public.ecr.aws/emqx/emqx:${version#v}
- name: Push image to aws ecr (elixir)
if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' && matrix.build_elixir == 'with_elixir'
run: |
version=${GITHUB_REF##*/}-elixir
docker pull emqx/emqx:${version#v}
docker tag emqx/emqx:${version#v} public.ecr.aws/emqx/emqx:${version#v}
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
docker push public.ecr.aws/emqx/emqx:${version#v}
delete-artifact: delete-artifact:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
@ -465,7 +579,7 @@ jobs:
- emqx - emqx
- emqx-enterprise - emqx-enterprise
otp: otp:
- 24.1.5-3 - 24.1.5-4
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2

33
.github/workflows/build_slim_packages.yaml vendored
View File

@ -34,12 +34,14 @@ jobs:
- emqx - emqx
- emqx-enterprise - emqx-enterprise
otp: otp:
- 24.1.5-3 - 24.1.5-4
elixir:
- 1.13.2
os: os:
- ubuntu20.04 - ubuntu20.04
- centos7 - centos7
container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}" container: "ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
steps: steps:
- uses: actions/checkout@v1 - uses: actions/checkout@v1
@ -48,6 +50,7 @@ jobs:
echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
echo "EMQX_PKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh ${{ matrix.profile }})-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV echo "EMQX_PKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh ${{ matrix.profile }})-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV
echo "EMQX_ELIXIRPKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh ${{ matrix.profile }})-elixir${{ matrix.elixir }}-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV
- name: Get deps git refs for cache - name: Get deps git refs for cache
id: deps-refs id: deps-refs
run: | run: |
@ -56,17 +59,21 @@ jobs:
- name: load rocksdb cache - name: load rocksdb cache
uses: actions/cache@v2 uses: actions/cache@v2
with: with:
path: _build/default/lib/rocksdb/ path: |
key: ${{ matrix.os }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }} _build/default/lib/rocksdb/
deps/rocksdb/
key: ${{ matrix.os }}-${{ matrix.elixir }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
- name: load quicer cache - name: load quicer cache
uses: actions/cache@v2 uses: actions/cache@v2
with: with:
path: _build/default/lib/quicer/ path: |
key: ${{ matrix.os }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} _build/default/lib/quicer/
deps/quicer/
key: ${{ matrix.os }}-${{ matrix.elixir }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
- name: build and test tgz package - name: build and test tgz package
run: | run: |
make ${EMQX_NAME}-tgz make ${EMQX_NAME}-tgz
.ci/build_packages/tests.sh "$EMQX_PKG_NAME" tgz .ci/build_packages/tests.sh "$EMQX_PKG_NAME" tgz amd64
- name: run static checks - name: run static checks
if: contains(matrix.os, 'ubuntu') if: contains(matrix.os, 'ubuntu')
run: | run: |
@ -74,7 +81,15 @@ jobs:
- name: build and test deb/rpm packages - name: build and test deb/rpm packages
run: | run: |
make ${EMQX_NAME}-pkg make ${EMQX_NAME}-pkg
.ci/build_packages/tests.sh "$EMQX_PKG_NAME" pkg .ci/build_packages/tests.sh "$EMQX_PKG_NAME" pkg amd64
- name: build and test tgz package (Elixir)
run: |
make ${EMQX_NAME}-elixir-tgz
.ci/build_packages/tests.sh "$EMQX_ELIXIRPKG_NAME" tgz amd64
- name: build and test deb/rpm packages (Elixir)
run: |
make ${EMQX_NAME}-elixirpkg
.ci/build_packages/tests.sh "$EMQX_ELIXIRPKG_NAME" pkg amd64
- uses: actions/upload-artifact@v2 - uses: actions/upload-artifact@v2
with: with:
name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }} name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }}
@ -87,7 +102,7 @@ jobs:
- emqx - emqx
- emqx-enterprise - emqx-enterprise
otp: otp:
- 24.1.5-3 - 24.1.5-4
macos: macos:
- macos-11 - macos-11
- macos-10.15 - macos-10.15

2
.github/workflows/check_deps_integrity.yaml vendored
View File

@ -5,7 +5,7 @@ on: [pull_request]
jobs: jobs:
check_deps_integrity: check_deps_integrity:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04" container: ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-ubuntu20.04
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2

2
.github/workflows/elixir_apps_check.yaml vendored
View File

@ -7,7 +7,7 @@ on: [pull_request]
jobs: jobs:
elixir_apps_check: elixir_apps_check:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: hexpm/elixir:1.13.1-erlang-24.2-alpine-3.15.0 container: hexpm/elixir:1.13.2-erlang-24.2-alpine-3.15.0
strategy: strategy:
fail-fast: false fail-fast: false

4
.github/workflows/elixir_deps_check.yaml vendored
View File

@ -7,11 +7,9 @@ on: [pull_request]
jobs: jobs:
elixir_deps_check: elixir_deps_check:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: hexpm/elixir:1.13.1-erlang-24.2-alpine-3.15.0 container: ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-ubuntu20.04
steps: steps:
- name: install
run: apk add make bash curl git
- name: Checkout - name: Checkout
uses: actions/checkout@v2.4.0 uses: actions/checkout@v2.4.0
- name: ensure rebar - name: ensure rebar

17
.github/workflows/elixir_release.yml vendored
View File

@ -12,28 +12,23 @@ on:
jobs: jobs:
build: build:
runs-on: ubuntu-latest runs-on: ubuntu-latest
container: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14 container: ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-ubuntu20.04
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v2.4.0 uses: actions/checkout@v2.4.0
- name: setup mix - name: install tools
run: | run: apt update && apt install netcat-openbsd
mix local.hex --force
mix local.rebar --force
mix deps.get
- name: produce emqx.conf.all template
run: make conf-segs
- name: elixir release - name: elixir release
run: mix release --overwrite run: make emqx-elixir
- name: start release - name: start release
run: | run: |
cd _build/dev/rel/emqx cd _build/prod/rel/emqx
bin/emqx start bin/emqx start
- name: check if started - name: check if started
run: | run: |
sleep 10 sleep 10
nc -zv localhost 1883 nc -zv localhost 1883
cd _build/dev/rel/emqx cd _build/prod/rel/emqx
bin/emqx ping bin/emqx ping
bin/emqx ctl status bin/emqx ctl status

7
.github/workflows/run_api_tests.yaml vendored
View File

@ -16,13 +16,16 @@ jobs:
strategy: strategy:
matrix: matrix:
otp: otp:
- 24.1.5-3 - 24.1.5-4
elixir:
- 1.13.2
os: os:
- ubuntu20.04 - ubuntu20.04
arch: arch:
- amd64 - amd64
runs-on: ubuntu-latest runs-on: ubuntu-latest
container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}" container: ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2

11
.github/workflows/run_emqx_app_tests.yaml vendored
View File

@ -12,15 +12,20 @@ jobs:
strategy: strategy:
matrix: matrix:
otp: otp:
- 23.3.4.9-3 - 23.3.4.9-4
- 24.1.5-3 - 24.1.5-4
# no need to use more than 1 version of Elixir, since tests
# run using only Erlang code. This is needed just to specify
# the base image.
elixir:
- 1.13.2
os: os:
- ubuntu20.04 - ubuntu20.04
arch: arch:
- amd64 - amd64
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}" container: "ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}"
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2

31
.github/workflows/run_fvt_tests.yaml vendored
View File

@ -14,7 +14,7 @@ jobs:
prepare: prepare:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
# prepare source with any OTP version, no need for a matrix # prepare source with any OTP version, no need for a matrix
container: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14 container: ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-alpine3.14
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
@ -41,13 +41,16 @@ jobs:
- emqx - emqx
- emqx-edge - emqx-edge
- emqx-enterprise - emqx-enterprise
- emqx-elixir
cluster_db_backend: cluster_db_backend:
- mnesia - mnesia
- rlog - rlog
os: os:
- alpine3.14 - alpine3.14
otp: otp:
- 24.1.5-3 - 24.1.5-4
elixir:
- 1.13.2
arch: arch:
- amd64 - amd64
exclude: exclude:
@ -71,18 +74,22 @@ jobs:
- name: load rocksdb cache - name: load rocksdb cache
uses: actions/cache@v2 uses: actions/cache@v2
with: with:
path: source/_build/default/lib/rocksdb/ path: |
key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }} source/_build/default/lib/rocksdb/
source/deps/rocksdb/
key: ${{ matrix.os }}-${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
- name: load quicer cache - name: load quicer cache
uses: actions/cache@v2 uses: actions/cache@v2
with: with:
path: source/_build/default/lib/quicer/ path: |
key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} source/_build/default/lib/quicer/
source/deps/quicer/
key: ${{ matrix.os }}-${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
- name: make docker image - name: make docker image
working-directory: source working-directory: source
env: env:
EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }} EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
run: | run: |
make ${{ matrix.profile }}-docker make ${{ matrix.profile }}-docker
- name: run emqx - name: run emqx
@ -91,6 +98,10 @@ jobs:
run: | run: |
set -x set -x
IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh ${{ matrix.profile }}) IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh ${{ matrix.profile }})
if [[ "${{ matrix.profile }}" = *-elixir ]]
then
export IS_ELIXIR=yes
fi
./.ci/docker-compose-file/scripts/run-emqx.sh $IMAGE ${{ matrix.cluster_db_backend }} ./.ci/docker-compose-file/scripts/run-emqx.sh $IMAGE ${{ matrix.cluster_db_backend }}
- name: make paho tests - name: make paho tests
run: | run: |
@ -118,7 +129,9 @@ jobs:
os: os:
- alpine3.14 - alpine3.14
otp: otp:
- 24.1.5-3 - 24.1.5-4
elixir:
- 1.13.2
arch: arch:
- amd64 - amd64
# - emqx-enterprise # TODO test enterprise # - emqx-enterprise # TODO test enterprise
@ -152,7 +165,7 @@ jobs:
- name: make docker image - name: make docker image
working-directory: source working-directory: source
env: env:
EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }} EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
run: | run: |
make ${{ matrix.profile }}-docker make ${{ matrix.profile }}-docker
echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV

9
.github/workflows/run_relup_tests.yaml vendored
View File

@ -19,14 +19,19 @@ jobs:
- emqx - emqx
- emqx-enterprise - emqx-enterprise
otp: otp:
- 24.1.5-3 - 24.1.5-4
# no need to use more than 1 version of Elixir, since tests
# run using only Erlang code. This is needed just to specify
# the base image.
elixir:
- 1.13.2
os: os:
- ubuntu20.04 - ubuntu20.04
arch: arch:
- amd64 - amd64
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}" container: "ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
defaults: defaults:
run: run:

6
.github/workflows/run_test_cases.yaml vendored
View File

@ -16,14 +16,16 @@ jobs:
strategy: strategy:
matrix: matrix:
otp: otp:
- 24.1.5-3 - 24.1.5-4
elixir:
- 1.13.2
os: os:
- ubuntu20.04 - ubuntu20.04
arch: arch:
- amd64 - amd64
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}" container: "ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2

4
.tool-versions
View File

@ -1,2 +1,2 @@
erlang 24.1.5-3 erlang 24.1.5-4
elixir 1.13.1-otp-24 elixir 1.13.2-otp-24

51
Makefile
View File

@ -3,9 +3,10 @@ REBAR_VERSION = 3.16.1-emqx-1
REBAR = $(CURDIR)/rebar3 REBAR = $(CURDIR)/rebar3
BUILD = $(CURDIR)/build BUILD = $(CURDIR)/build
SCRIPTS = $(CURDIR)/scripts SCRIPTS = $(CURDIR)/scripts
export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14 export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-alpine3.14
export EMQX_DEFAULT_RUNNER = alpine:3.14 export EMQX_DEFAULT_RUNNER = alpine:3.14
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
export EMQX_DASHBOARD_VERSION ?= v0.18.0 export EMQX_DASHBOARD_VERSION ?= v0.18.0
export DOCKERFILE := deploy/docker/Dockerfile export DOCKERFILE := deploy/docker/Dockerfile
export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing
@ -36,6 +37,22 @@ ensure-rebar3:
@$(SCRIPTS)/fail-on-old-otp-version.escript @$(SCRIPTS)/fail-on-old-otp-version.escript
@$(SCRIPTS)/ensure-rebar3.sh $(REBAR_VERSION) @$(SCRIPTS)/ensure-rebar3.sh $(REBAR_VERSION)
.PHONY: ensure-hex
ensure-hex:
@mix local.hex --if-missing --force
.PHONY: ensure-mix-rebar3
ensure-mix-rebar3: $(REBAR)
@mix local.rebar rebar3 $(CURDIR)/rebar3 --if-missing --force
.PHONY: ensure-mix-rebar
ensure-mix-rebar: $(REBAR)
@mix local.rebar --if-missing --force
.PHONY: mix-deps-get
mix-deps-get: $(ELIXIR_COMMON_DEPS)
@mix deps.get
$(REBAR): ensure-rebar3 $(REBAR): ensure-rebar3
.PHONY: get-dashboard .PHONY: get-dashboard
@ -95,7 +112,7 @@ coveralls: $(REBAR)
@ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send @ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send
.PHONY: $(REL_PROFILES) .PHONY: $(REL_PROFILES)
$(REL_PROFILES:%=%): $(REBAR) get-dashboard conf-segs $(REL_PROFILES:%=%): $(COMMON_DEPS)
@$(REBAR) as $(@) do release @$(REBAR) as $(@) do release
## Not calling rebar3 clean because ## Not calling rebar3 clean because
@ -139,6 +156,7 @@ dialyzer: $(REBAR)
@$(REBAR) as check dialyzer @$(REBAR) as check dialyzer
COMMON_DEPS := $(REBAR) get-dashboard conf-segs COMMON_DEPS := $(REBAR) get-dashboard conf-segs
ELIXIR_COMMON_DEPS := ensure-hex ensure-mix-rebar3 ensure-mix-rebar
## rel target is to create release package without relup ## rel target is to create release package without relup
.PHONY: $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel) .PHONY: $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel)
@ -179,13 +197,13 @@ quickrun:
./_build/$(PROFILE)/rel/emqx/bin/emqx console ./_build/$(PROFILE)/rel/emqx/bin/emqx console
## docker target is to create docker instructions ## docker target is to create docker instructions
.PHONY: $(REL_PROFILES:%=%-docker) .PHONY: $(REL_PROFILES:%=%-docker) $(REL_PROFILES:%=%-elixir-docker)
define gen-docker-target define gen-docker-target
$1-docker: $(COMMON_DEPS) $1-docker: $(COMMON_DEPS)
@$(BUILD) $1 docker @$(BUILD) $1 docker
endef endef
ALL_TGZS = $(REL_PROFILES) ALL_DOCKERS = $(REL_PROFILES) $(REL_PROFILES:%=%-elixir)
$(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target,$(zt)))) $(foreach zt,$(ALL_DOCKERS),$(eval $(call gen-docker-target,$(zt))))
## emqx-docker-testing ## emqx-docker-testing
## emqx-enterprise-docker-testing ## emqx-enterprise-docker-testing
@ -201,3 +219,26 @@ $(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target-testing,$(zt))))
conf-segs: conf-segs:
@scripts/merge-config.escript @scripts/merge-config.escript
## elixir target is to create release packages using Elixir's Mix
.PHONY: $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir)
$(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir): $(COMMON_DEPS) $(ELIXIR_COMMON_DEPS) mix-deps-get
@$(BUILD) $(subst -elixir,,$(@)) elixir
.PHONY: $(REL_PROFILES:%=%-elixirpkg)
define gen-elixirpkg-target
# the Elixir places the tar in a different path than Rebar3
$1-elixirpkg: $1-pkg-elixir
@env TAR_PKG_DIR=_build/prod \
IS_ELIXIR=yes \
$(BUILD) $1 pkg
endef
$(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixirpkg-target,$(pt))))
.PHONY: $(REL_PROFILES:%=%-elixir-tgz)
define gen-elixir-tgz-target
$1-elixir-tgz: $(COMMON_DEPS) $(ELIXIR_COMMON_DEPS) mix-deps-get
@env IS_ELIXIR=yes $(BUILD) $1 tgz
endef
ALL_ELIXIR_TGZS = $(REL_PROFILES)
$(foreach tt,$(ALL_ELIXIR_TGZS),$(eval $(call gen-elixir-tgz-target,$(tt))))

2
apps/emqx/src/proto/emqx_cm_proto_v1.erl
View File

@ -32,7 +32,7 @@
]). ]).
-include("bpapi.hrl"). -include("bpapi.hrl").
-include("emqx_cm.hrl"). -include("src/emqx_cm.hrl").
introduced_in() -> introduced_in() ->
"5.0.0". "5.0.0".

4
bin/emqx
View File

@ -864,10 +864,12 @@ case "${COMMAND}" in
then then
"$REL_DIR/elixir" \ "$REL_DIR/elixir" \
--hidden \ --hidden \
--name "rand-$(relx_gen_id)-$NAME" \
--cookie "$COOKIE" \ --cookie "$COOKIE" \
--boot "$REL_DIR/start_clean" \ --boot "$REL_DIR/start_clean" \
--boot-var RELEASE_LIB "$ERTS_LIB_DIR" \ --boot-var RELEASE_LIB "$ERTS_LIB_DIR" \
--vm-args "$(latest_vm_args 'EMQX_NODE__NAME')"\ --vm-args "$REL_DIR/remote.vm.args" \
--erl "-start_epmd false -epmd_module ekka_epmd" \
--rpc-eval "$NAME" "$@" --rpc-eval "$NAME" "$@"
else else
relx_nodetool "eval" "$@" relx_nodetool "eval" "$@"

91
build
View File

@ -6,6 +6,11 @@
set -euo pipefail set -euo pipefail
DEBUG="${DEBUG:-0}"
if [ "$DEBUG" -eq 1 ]; then
set -x
fi
PROFILE="$1" PROFILE="$1"
ARTIFACT="$2" ARTIFACT="$2"
@ -26,7 +31,7 @@ case "$ARCH" in
ARCH='arm64' ARCH='arm64'
;; ;;
arm*) arm*)
ARCH=arm ARCH='arm64'
;; ;;
esac esac
export ARCH export ARCH
@ -79,6 +84,11 @@ make_rel() {
fi fi
} }
make_elixir_rel() {
export_release_vars "$PROFILE"
env MIX_ENV=prod mix release --overwrite
}
## extract previous version .tar.gz files to _build/$PROFILE/rel/emqx before making relup ## extract previous version .tar.gz files to _build/$PROFILE/rel/emqx before making relup
make_relup() { make_relup() {
local rel_dir="_build/$PROFILE/rel/emqx" local rel_dir="_build/$PROFILE/rel/emqx"
@ -119,21 +129,35 @@ cp_dyn_libs() {
## Re-pack the relx assembled .tar.gz to EMQ X's package naming scheme ## Re-pack the relx assembled .tar.gz to EMQ X's package naming scheme
## It assumes the .tar.gz has been built -- relies on Makefile dependency ## It assumes the .tar.gz has been built -- relies on Makefile dependency
make_tgz() { make_tgz() {
local pkgpath="_packages/${PROFILE}"
local tarball
local target
if [ "${IS_ELIXIR:-no}" = "yes" ]
then
# ensure tarball exists
ELIXIR_MAKE_TAR=yes make_elixir_rel
local relpath="_build/prod"
target="${pkgpath}/${PROFILE}-${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz"
else
# build the tarball again to ensure relup is included # build the tarball again to ensure relup is included
# elixir does not have relup yet.
make_rel make_rel
local relpath="_build/${PROFILE}/rel/emqx"
target="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz"
fi
tarball="${relpath}/emqx-${PKG_VSN}.tar.gz"
tard="/tmp/emqx_untar_${PKG_VSN}" tard="/tmp/emqx_untar_${PKG_VSN}"
rm -rf "${tard}" rm -rf "${tard}"
mkdir -p "${tard}/emqx" mkdir -p "${tard}/emqx"
local relpath="_build/${PROFILE}/rel/emqx"
local pkgpath="_packages/${PROFILE}"
mkdir -p "${pkgpath}" mkdir -p "${pkgpath}"
local tarball="${relpath}/emqx-${PKG_VSN}.tar.gz"
if [ ! -f "$tarball" ]; then if [ ! -f "$tarball" ]; then
log "ERROR: $tarball is not found" log "ERROR: $tarball is not found"
fi fi
local target
target="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz"
tar zxf "${tarball}" -C "${tard}/emqx" tar zxf "${tarball}" -C "${tard}/emqx"
## try to be portable for tar.gz packages. ## try to be portable for tar.gz packages.
## for DEB and RPM packages the dependencies are resoved by yum and apt ## for DEB and RPM packages the dependencies are resoved by yum and apt
@ -198,6 +222,53 @@ make_docker_testing() {
-f "${DOCKERFILE_TESTING}" . -f "${DOCKERFILE_TESTING}" .
} }
# used to control the Elixir Mix Release output
# see docstring in `mix.exs`
export_release_vars() {
local profile="$1"
case "$profile" in
emqx)
export EMQX_RLEASE_TYPE=cloud \
EMQX_PACKAGE_TYPE=bin \
EMQX_EDITION_TYPE=community \
ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no}
;;
emqx-edge)
export EMQX_RLEASE_TYPE=edge \
EMQX_PACKAGE_TYPE=bin \
EMQX_EDITION_TYPE=community \
ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no}
;;
emqx-enterprise)
export EMQX_RLEASE_TYPE=cloud \
EMQX_PACKAGE_TYPE=bin \
EMQX_EDITION_TYPE=enterprise \
ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no}
;;
emqx-pkg)
export EMQX_RLEASE_TYPE=cloud \
EMQX_PACKAGE_TYPE=pkg \
EMQX_EDITION_TYPE=community \
ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes}
;;
emqx-edge-pkg)
export EMQX_RLEASE_TYPE=edge \
EMQX_PACKAGE_TYPE=pkg \
EMQX_EDITION_TYPE=community \
ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes}
;;
emqx-enterprise-pkg)
export EMQX_RLEASE_TYPE=cloud \
EMQX_PACKAGE_TYPE=pkg \
EMQX_EDITION_TYPE=enterprise \
ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes}
;;
*)
echo Invalid profile "$profile"
exit 1
esac
}
log "building artifact=$ARTIFACT for profile=$PROFILE" log "building artifact=$ARTIFACT for profile=$PROFILE"
case "$ARTIFACT" in case "$ARTIFACT" in
@ -219,7 +290,10 @@ case "$ARTIFACT" in
exit 0 exit 0
fi fi
make -C "deploy/packages/${PKGERDIR}" clean make -C "deploy/packages/${PKGERDIR}" clean
EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}" env EMQX_REL="$(pwd)" \
EMQX_BUILD="${PROFILE}" \
SYSTEM="${SYSTEM}" \
make -C "deploy/packages/${PKGERDIR}"
;; ;;
docker) docker)
make_docker make_docker
@ -227,6 +301,9 @@ case "$ARTIFACT" in
docker-testing) docker-testing)
make_docker_testing make_docker_testing
;; ;;
elixir)
make_elixir_rel
;;
*) *)
log "Unknown artifact $ARTIFACT" log "Unknown artifact $ARTIFACT"
exit 1 exit 1

22
deploy/docker/Dockerfile
View File

@ -1,4 +1,4 @@
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14 ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-alpine3.14
ARG RUN_FROM=alpine:3.14 ARG RUN_FROM=alpine:3.14
FROM ${BUILD_FROM} AS builder FROM ${BUILD_FROM} AS builder
@ -22,17 +22,23 @@ COPY . /emqx
ARG EMQX_NAME=emqx ARG EMQX_NAME=emqx
RUN cd /emqx \ RUN if [[ "$EMQX_NAME" = *-elixir ]]; then \
&& rm -rf _build/$EMQX_NAME/lib \ export EMQX_LIB_PATH="_build/prod/lib"; \
&& make $EMQX_NAME export EMQX_REL_PATH="/emqx/_build/prod/rel/emqx"; \
else \
export EMQX_LIB_PATH="_build/$EMQX_NAME/lib"; \
export EMQX_REL_PATH="/emqx/_build/$EMQX_NAME/rel/emqx"; \
fi \
&& cd /emqx \
&& rm -rf $EMQX_LIB_PATH \
&& make $EMQX_NAME \
&& mkdir -p /emqx-rel \
&& mv $EMQX_REL_PATH /emqx-rel
FROM $RUN_FROM FROM $RUN_FROM
## define ARG again after 'FROM $RUN_FROM'
ARG EMQX_NAME=emqx
COPY deploy/docker/docker-entrypoint.sh /usr/bin/ COPY deploy/docker/docker-entrypoint.sh /usr/bin/
COPY --from=builder /emqx/_build/$EMQX_NAME/rel/emqx /opt/emqx COPY --from=builder /emqx-rel/emqx /opt/emqx
RUN ln -s /opt/emqx/bin/* /usr/local/bin/ RUN ln -s /opt/emqx/bin/* /usr/local/bin/
RUN apk add --no-cache curl ncurses-libs openssl sudo libstdc++ bash RUN apk add --no-cache curl ncurses-libs openssl sudo libstdc++ bash

11
deploy/packages/deb/Makefile
View File

@ -6,9 +6,16 @@ BUILT := $(SRCDIR)/BUILT
EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz ifeq ($(IS_ELIXIR), yes)
ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN)
else
ELIXIR_PKG_VSN :=
endif
TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx
TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz
SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture)
TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)$(ELIXIR_PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH)
.PHONY: all .PHONY: all
all: | $(BUILT) all: | $(BUILT)

12
deploy/packages/rpm/Makefile
View File

@ -16,9 +16,16 @@ endif
EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz ifeq ($(IS_ELIXIR), yes)
TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN)
else
ELIXIR_PKG_VSN :=
endif
TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx
TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz
SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m)
TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)$(ELIXIR_PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH)
SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi)
# Not $(PWD) as it does not work for make -C # Not $(PWD) as it does not work for make -C
@ -55,4 +62,3 @@ $(BUILT):
clean: clean:
rm -rf $(SRCDIR) rm -rf $(SRCDIR)

38
mix.exs
View File

@ -24,13 +24,6 @@ defmodule EMQXUmbrella.MixProject do
Defaults to `community`. Defaults to `community`.
""" """
# Temporary hack while 1.13.2 is not released
System.version()
|> Version.parse!()
|> Version.compare(Version.parse!("1.13.2"))
|> Kernel.==(:lt)
|> if(do: Code.require_file("lib/mix/release.exs"))
def project() do def project() do
[ [
app: :emqx_mix, app: :emqx_mix,
@ -124,7 +117,7 @@ defmodule EMQXUmbrella.MixProject do
steps = steps =
if System.get_env("ELIXIR_MAKE_TAR") == "yes" do if System.get_env("ELIXIR_MAKE_TAR") == "yes" do
base_steps ++ [:tar] base_steps ++ [&prepare_tar_overlays/1, :tar]
else else
base_steps base_steps
end end
@ -154,7 +147,6 @@ defmodule EMQXUmbrella.MixProject do
def applications(release_type) do def applications(release_type) do
[ [
logger: :permanent,
crypto: :permanent, crypto: :permanent,
public_key: :permanent, public_key: :permanent,
asn1: :permanent, asn1: :permanent,
@ -234,16 +226,30 @@ defmodule EMQXUmbrella.MixProject do
} }
end end
#############################################################################
# Custom Steps
#############################################################################
defp copy_files(release, release_type, package_type, edition_type) do defp copy_files(release, release_type, package_type, edition_type) do
overwrite? = Keyword.get(release.options, :overwrite, false) overwrite? = Keyword.get(release.options, :overwrite, false)
bin = Path.join(release.path, "bin") bin = Path.join(release.path, "bin")
etc = Path.join(release.path, "etc") etc = Path.join(release.path, "etc")
log = Path.join(release.path, "log")
Mix.Generator.create_directory(bin) Mix.Generator.create_directory(bin)
Mix.Generator.create_directory(etc) Mix.Generator.create_directory(etc)
Mix.Generator.create_directory(log)
Mix.Generator.create_directory(Path.join(etc, "certs")) Mix.Generator.create_directory(Path.join(etc, "certs"))
Enum.each(
["mnesia", "configs", "patches", "scripts"],
fn dir ->
path = Path.join([release.path, "data", dir])
Mix.Generator.create_directory(path)
end
)
Mix.Generator.copy_file( Mix.Generator.copy_file(
"apps/emqx_authz/etc/acl.conf", "apps/emqx_authz/etc/acl.conf",
Path.join(etc, "acl.conf"), Path.join(etc, "acl.conf"),
@ -409,6 +415,18 @@ defmodule EMQXUmbrella.MixProject do
release release
end end
# The `:tar` built-in step in Mix Release does not currently add the
# `etc` directory into the resulting tarball. The workaround is to
# add those to the `:overlays` key before running `:tar`.
# See: https://hexdocs.pm/mix/1.13.2/Mix.Release.html#__struct__/0
defp prepare_tar_overlays(release) do
Map.update!(release, :overlays, &["etc", "data" | &1])
end
#############################################################################
# Helper functions
#############################################################################
defp template_vars(release, release_type, :bin = _package_type, edition_type) do defp template_vars(release, release_type, :bin = _package_type, edition_type) do
[ [
platform_bin_dir: "bin", platform_bin_dir: "bin",
@ -454,7 +472,7 @@ defmodule EMQXUmbrella.MixProject do
# FIXME: this is empty in `make emqx` ??? # FIXME: this is empty in `make emqx` ???
erl_opts: "", erl_opts: "",
emqx_description: emqx_description(release_type, edition_type), emqx_description: emqx_description(release_type, edition_type),
built_on: built_on(), built_on_arch: built_on(),
is_elixir: "yes" is_elixir: "yes"
] ]
end end

60
scripts/buildx.sh
View File

@ -11,17 +11,23 @@
## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10 ## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10
set -euo pipefail set -euo pipefail
set -x
help() { help() {
echo echo
echo "-h|--help: To display this usage information" echo "-h|--help: To display this usage information"
echo "--profile <PROFILE>: EMQ X profile to build, e.g. emqx, emqx-edge" echo "--profile <PROFILE>: EMQ X profile to build, e.g. emqx, emqx-edge"
echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz" echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz,"
echo " and pkg for .rpm or .deb" echo " pkg for .rpm or .deb"
echo "--with-elixir: Specify if the release should be built with Elixir, "
echo " defaults to false."
echo "--arch amd64|arm64: Target arch to build the EMQ X package for" echo "--arch amd64|arm64: Target arch to build the EMQ X package for"
echo "--src_dir <SRC_DIR>: EMQ X source ode in this dir, default to PWD" echo "--src_dir <SRC_DIR>: EMQ X source ode in this dir, default to PWD"
echo "--builder <BUILDER>: Builder image to pull" echo "--builder <BUILDER>: Builder image to pull"
echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10" echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10"
echo "--otp <OTP_VSN>: OTP version being used in the builder"
echo "--elixir <ELIXIR_VSN>: Elixir version being used in the builder"
echo "--system <SYSTEM>: OS used in the builder image"
} }
while [ "$#" -gt 0 ]; do while [ "$#" -gt 0 ]; do
@ -50,6 +56,22 @@ while [ "$#" -gt 0 ]; do
ARCH="$2" ARCH="$2"
shift 2 shift 2
;; ;;
--otp)
OTP_VSN="$2"
shift 2
;;
--elixir)
ELIXIR_VSN="$2"
shift 2
;;
--with-elixir)
WITH_ELIXIR=yes
shift 1
;;
--system)
SYSTEM="$2"
shift 2
;;
*) *)
echo "WARN: Unknown arg (ignored): $1" echo "WARN: Unknown arg (ignored): $1"
shift shift
@ -58,21 +80,43 @@ while [ "$#" -gt 0 ]; do
esac esac
done done
if [ -z "${PROFILE:-}" ] || [ -z "${PKGTYPE:-}" ] || [ -z "${BUILDER:-}" ] || [ -z "${ARCH:-}" ]; then if [ -z "${PROFILE:-}" ] ||
[ -z "${PKGTYPE:-}" ] ||
[ -z "${BUILDER:-}" ] ||
[ -z "${ARCH:-}" ] ||
[ -z "${OTP_VSN:-}" ] ||
[ -z "${ELIXIR_VSN:-}" ] ||
[ -z "${SYSTEM:-}" ]; then
help help
exit 1 exit 1
fi fi
if [ "$PKGTYPE" != 'tgz' ] && [ "$PKGTYPE" != 'pkg' ]; then if [ -z "${WITH_ELIXIR:-}" ]; then
WITH_ELIXIR=no
fi
case "$PKGTYPE" in
tgz|pkg)
true
;;
*)
echo "Bad --pkgtype option, should be tgz or pkg" echo "Bad --pkgtype option, should be tgz or pkg"
exit 1 exit 1
fi ;;
esac
cd "${SRC_DIR:-.}" cd "${SRC_DIR:-.}"
PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}" PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}"
OTP_VSN_SYSTEM=$(echo "$BUILDER" | cut -d ':' -f2)
PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN_SYSTEM}-${ARCH}" if [ "$WITH_ELIXIR" = "yes" ]
then
PKG_NAME="${PROFILE}-${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
MAKE_TARGET="${PROFILE}-elixir-${PKGTYPE}"
else
PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
MAKE_TARGET="${PROFILE}-${PKGTYPE}"
fi
docker info docker info
docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}" docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}"
@ -82,4 +126,4 @@ docker run -i --rm \
--platform="linux/$ARCH" \ --platform="linux/$ARCH" \
-e EMQX_NAME="$PROFILE" \ -e EMQX_NAME="$PROFILE" \
"$BUILDER" \ "$BUILDER" \
bash -euc "make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PKG_NAME $PKGTYPE" bash -euc "make ${MAKE_TARGET} && .ci/build_packages/tests.sh $PKG_NAME $PKGTYPE $ARCH"

8
scripts/get-elixir-vsn.sh Executable file
View File

@ -0,0 +1,8 @@
#!/usr/bin/env bash
set -euo pipefail
if command -v elixir &>/dev/null
then
elixir -e "System.version() |> IO.puts()"
fi