diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 75a973990..bf2b3ed6b 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -8,13 +8,19 @@ if [ -z "${1:-}" ]; then exit 1 fi -if [ "${2:-}" != 'tgz' ] && [ "${2:-}" != 'pkg' ]; then +case "${2:-}" in + tgz|pkg) + true + ;; + *) echo "Usage $0 tgz|pkg" exit 1 -fi + ;; +esac PACKAGE_NAME="${1}" PACKAGE_TYPE="${2}" +ARCH="${3}" export DEBUG=1 export CODE_PATH=${CODE_PATH:-"/emqx"} @@ -43,17 +49,20 @@ if ! [ -f "$PACKAGE_FILE" ]; then exit 1 fi -case "$(uname -m)" in +if [ -z "$ARCH" ] +then + case "$(uname -m)" in x86_64) - ARCH='amd64' - ;; + ARCH='amd64' + ;; aarch64) - ARCH='arm64' - ;; + ARCH='arm64' + ;; arm*) - ARCH=arm - ;; -esac + ARCH=arm + ;; + esac +fi export ARCH emqx_prepare(){ diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index f1cce9364..55d6fc145 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang23: container_name: erlang23 - image: ghcr.io/emqx/emqx-builder/5.0-3:23.3.4.9-3-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-23.3.4.9-4-ubuntu20.04 env_file: - conf.env environment: @@ -23,7 +23,7 @@ services: erlang24: container_name: erlang24 - image: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-ubuntu20.04 env_file: - conf.env environment: diff --git a/.ci/docker-compose-file/scripts/run-emqx.sh b/.ci/docker-compose-file/scripts/run-emqx.sh index 35119a378..51e467983 100755 --- a/.ci/docker-compose-file/scripts/run-emqx.sh +++ b/.ci/docker-compose-file/scripts/run-emqx.sh @@ -19,17 +19,23 @@ fi } >> .ci/docker-compose-file/conf.cluster.env is_node_up() { - local node - node="$1" - docker exec -i "$node" \ - bash -c "emqx eval \"['emqx@node1.emqx.io','emqx@node2.emqx.io'] = maps:get(running_nodes, ekka_cluster:info()).\"" > /dev/null 2>&1 + local node="$1" + if [ "${IS_ELIXIR:-no}" = "yes" ] + then + docker exec -i "$node" \ + bash -c "emqx eval \"[:\\\"emqx@node1.emqx.io\\\", :\\\"emqx@node2.emqx.io\\\"] = :ekka_cluster.info()[:running_nodes]\"" + else + docker exec -i "$node" \ + bash -c "emqx eval \"['emqx@node1.emqx.io','emqx@node2.emqx.io'] = maps:get(running_nodes, ekka_cluster:info()).\"" > /dev/null 2>&1 + fi } is_node_listening() { - local node - node="$1" + local node="$1" docker exec -i "$node" \ - emqx eval "ok = case gen_tcp:connect(\"localhost\", 1883, []) of {ok, P} -> gen_tcp:close(P), ok; _ -> exit(1) end." > /dev/null 2>&1 + emqx ctl listeners | \ + grep -A6 'tcp:default' | \ + grep -qE 'running *: true' } is_cluster_up() { diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 2a7d43958..3809dd776 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -19,7 +19,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-ubuntu20.04" outputs: ce_old_vsns: ${{ steps.find_old_versons.outputs.ce_old_vsns }} @@ -130,7 +130,7 @@ jobs: - emqx - emqx-enterprise otp: - - 24.1.5-3 + - 24.1.5-4 macos: - macos-11 - macos-10.15 @@ -236,7 +236,18 @@ jobs: - emqx - emqx-enterprise otp: - - 24.1.5-3 # we test with OTP 23, but only build package on OTP 24 versions + - 24.1.5-4 # we test with OTP 23, but only build package on OTP 24 versions + elixir: + - 1.13.2 + # used to split elixir packages into a separate job, since the + # entire job may take a lot of time, especially on arm64 + # emulation. + # we only want to build ubuntu and centos with elixir for the + # time being, so it's easier to just include those with + # `with_elixir` set. + build_elixir: + # - with_elixir + - no_elixir arch: - amd64 - arm64 @@ -264,6 +275,19 @@ jobs: profile: emqx-enterprise - os: raspbian10 profile: emqx-enterprise + include: + - profile: emqx + otp: 24.1.5-4 + elixir: 1.13.2 + arch: amd64 + build_elixir: with_elixir + os: ubuntu20.04 + - profile: emqx + otp: 24.1.5-4 + elixir: 1.13.2 + arch: amd64 + build_elixir: with_elixir + os: centos8 defaults: run: @@ -289,12 +313,16 @@ jobs: - name: load rocksdb cache uses: actions/cache@v2 with: - path: source/_build/default/lib/rocksdb/ + path: | + source/_build/default/lib/rocksdb/ + source/deps/rocksdb/ key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }} - name: load quicer cache uses: actions/cache@v2 with: - path: source/_build/default/lib/quicer/ + path: | + source/_build/default/lib/quicer/ + source/deps/quicer/ key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} - name: download old emqx tgz packages env: @@ -334,22 +362,51 @@ jobs: - name: build emqx packages env: OTP: ${{ matrix.otp }} + ELIXIR: ${{ matrix.elixir }} PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} + if: ${{ matrix.build_elixir == 'no_elixir' }} working-directory: source run: | ./scripts/buildx.sh \ --profile "${PROFILE}" \ --pkgtype "tgz" \ --arch "${ARCH}" \ - --builder "ghcr.io/emqx/emqx-builder/5.0-3:${OTP}-${SYSTEM}" + --otp "${OTP}" \ + --elixir "${ELIXIR}" \ + --system "${SYSTEM}" \ + --builder "ghcr.io/emqx/emqx-builder/5.0-5:${ELIXIR}-${OTP}-${SYSTEM}" ## the pkg build is incremental on the tgz build ./scripts/buildx.sh \ --profile "${PROFILE}" \ --pkgtype "pkg" \ --arch "${ARCH}" \ - --builder "ghcr.io/emqx/emqx-builder/5.0-3:${OTP}-${SYSTEM}" + --otp "${OTP}" \ + --elixir "${ELIXIR}" \ + --system "${SYSTEM}" \ + --builder "ghcr.io/emqx/emqx-builder/5.0-5:${ELIXIR}-${OTP}-${SYSTEM}" + + - name: build emqx packages (Elixir) + env: + OTP: ${{ matrix.otp }} + ELIXIR: ${{ matrix.elixir }} + PROFILE: ${{ matrix.profile }} + ARCH: ${{ matrix.arch }} + SYSTEM: ${{ matrix.os }} + working-directory: source + if: ${{ matrix.build_elixir == 'with_elixir' }} + run: | + ## we currently only build tgzs for elixir + ./scripts/buildx.sh \ + --profile "${PROFILE}" \ + --pkgtype "tgz" \ + --arch "${ARCH}" \ + --otp "${OTP}" \ + --elixir "${ELIXIR}" \ + --system "${SYSTEM}" \ + --with-elixir \ + --builder "ghcr.io/emqx/emqx-builder/5.0-5:${ELIXIR}-${OTP}-${SYSTEM}" - name: create sha256 env: @@ -380,9 +437,23 @@ jobs: - emqx-edge - emqx - emqx-enterprise - # NOTE: for docker, only support latest otp version, not a matrix + # NOTE: for docker, only support latest otp and elixir + # versions, not a matrix otp: - - 24.1.5-3 # update to latest + - 24.1.5-4 # update to latest + elixir: + - 1.13.2 # update to latest + arch: + - amd64 + - arm64 + build_elixir: + - no_elixir + include: + - profile: emqx + otp: 24.1.5-4 + elixir: 1.13.2 + arch: amd64 + build_elixir: with_elixir steps: - uses: actions/download-artifact@v2 @@ -398,6 +469,7 @@ jobs: platforms: all - uses: docker/metadata-action@v3 id: meta + if: ${{ matrix.build_elixir == 'no_elixir' }} with: images: ${{ github.repository_owner }}/${{ matrix.profile }} flavor: | @@ -409,12 +481,30 @@ jobs: type=semver,pattern={{version}} labels: org.opencontainers.image.otp.version=${{ matrix.otp }} + - name: docker metadata for elixir image + uses: docker/metadata-action@v3 + if: ${{ matrix.build_elixir == 'with_elixir' }} + id: meta-elixir + with: + images: ${{ github.repository_owner }}/${{ matrix.profile }} + flavor: | + latest=${{ !github.event.release.prerelease }} + suffix=-elixir + tags: | + type=ref,event=branch + type=ref,event=pr + type=ref,event=tag + type=semver,pattern={{version}} + labels: | + org.opencontainers.image.otp.version=${{ matrix.otp }} + org.opencontainers.image.elixir.version=${{ matrix.elixir }} - uses: docker/login-action@v1 if: github.event_name == 'release' with: username: ${{ secrets.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - uses: docker/build-push-action@v2 + if: ${{ matrix.build_elixir == 'no_elixir' }} with: push: ${{ github.event_name == 'release' && !github.event.release.prerelease }} pull: true @@ -423,11 +513,27 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-alpine3.14 + BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-alpine3.14 RUN_FROM=alpine:3.14 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile context: source + - name: build docker image with elixir + uses: docker/build-push-action@v2 + if: ${{ matrix.profile == 'emqx' && matrix.build_elixir == 'with_elixir' }} + with: + push: ${{ github.event_name == 'release' && !github.event.release.prerelease }} + pull: true + no-cache: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta-elixir.outputs.tags }} + labels: ${{ steps.meta-elixir.outputs.labels }} + build-args: | + BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-alpine3.14 + RUN_FROM=alpine:3.14 + EMQX_NAME=emqx-elixir + file: source/deploy/docker/Dockerfile + context: source - uses: aws-actions/configure-aws-credentials@v1 if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' with: @@ -435,13 +541,21 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - name: Push image to aws ecr - if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' + if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' && matrix.build_elixir == 'no_elixir' run: | version=${GITHUB_REF##*/} docker pull emqx/emqx:${version#v} docker tag emqx/emqx:${version#v} public.ecr.aws/emqx/emqx:${version#v} aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws docker push public.ecr.aws/emqx/emqx:${version#v} + - name: Push image to aws ecr (elixir) + if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' && matrix.build_elixir == 'with_elixir' + run: | + version=${GITHUB_REF##*/}-elixir + docker pull emqx/emqx:${version#v} + docker tag emqx/emqx:${version#v} public.ecr.aws/emqx/emqx:${version#v} + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws + docker push public.ecr.aws/emqx/emqx:${version#v} delete-artifact: runs-on: ubuntu-20.04 @@ -465,7 +579,7 @@ jobs: - emqx - emqx-enterprise otp: - - 24.1.5-3 + - 24.1.5-4 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index fc4a2d8f5..a17940354 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -34,12 +34,14 @@ jobs: - emqx - emqx-enterprise otp: - - 24.1.5-3 + - 24.1.5-4 + elixir: + - 1.13.2 os: - ubuntu20.04 - centos7 - container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}" + container: "ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" steps: - uses: actions/checkout@v1 @@ -48,6 +50,7 @@ jobs: echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV echo "EMQX_PKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh ${{ matrix.profile }})-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV + echo "EMQX_ELIXIRPKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh ${{ matrix.profile }})-elixir${{ matrix.elixir }}-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV - name: Get deps git refs for cache id: deps-refs run: | @@ -56,17 +59,21 @@ jobs: - name: load rocksdb cache uses: actions/cache@v2 with: - path: _build/default/lib/rocksdb/ - key: ${{ matrix.os }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }} + path: | + _build/default/lib/rocksdb/ + deps/rocksdb/ + key: ${{ matrix.os }}-${{ matrix.elixir }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }} - name: load quicer cache uses: actions/cache@v2 with: - path: _build/default/lib/quicer/ - key: ${{ matrix.os }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} + path: | + _build/default/lib/quicer/ + deps/quicer/ + key: ${{ matrix.os }}-${{ matrix.elixir }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} - name: build and test tgz package run: | make ${EMQX_NAME}-tgz - .ci/build_packages/tests.sh "$EMQX_PKG_NAME" tgz + .ci/build_packages/tests.sh "$EMQX_PKG_NAME" tgz amd64 - name: run static checks if: contains(matrix.os, 'ubuntu') run: | @@ -74,7 +81,15 @@ jobs: - name: build and test deb/rpm packages run: | make ${EMQX_NAME}-pkg - .ci/build_packages/tests.sh "$EMQX_PKG_NAME" pkg + .ci/build_packages/tests.sh "$EMQX_PKG_NAME" pkg amd64 + - name: build and test tgz package (Elixir) + run: | + make ${EMQX_NAME}-elixir-tgz + .ci/build_packages/tests.sh "$EMQX_ELIXIRPKG_NAME" tgz amd64 + - name: build and test deb/rpm packages (Elixir) + run: | + make ${EMQX_NAME}-elixirpkg + .ci/build_packages/tests.sh "$EMQX_ELIXIRPKG_NAME" pkg amd64 - uses: actions/upload-artifact@v2 with: name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }} @@ -87,7 +102,7 @@ jobs: - emqx - emqx-enterprise otp: - - 24.1.5-3 + - 24.1.5-4 macos: - macos-11 - macos-10.15 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 2318b9938..26143b807 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04" + container: ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml index 04bc8c024..617733207 100644 --- a/.github/workflows/elixir_apps_check.yaml +++ b/.github/workflows/elixir_apps_check.yaml @@ -7,7 +7,7 @@ on: [pull_request] jobs: elixir_apps_check: runs-on: ubuntu-20.04 - container: hexpm/elixir:1.13.1-erlang-24.2-alpine-3.15.0 + container: hexpm/elixir:1.13.2-erlang-24.2-alpine-3.15.0 strategy: fail-fast: false diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml index 12fc9643b..b44275635 100644 --- a/.github/workflows/elixir_deps_check.yaml +++ b/.github/workflows/elixir_deps_check.yaml @@ -7,11 +7,9 @@ on: [pull_request] jobs: elixir_deps_check: runs-on: ubuntu-20.04 - container: hexpm/elixir:1.13.1-erlang-24.2-alpine-3.15.0 + container: ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-ubuntu20.04 steps: - - name: install - run: apk add make bash curl git - name: Checkout uses: actions/checkout@v2.4.0 - name: ensure rebar diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml index 9a1d672a2..0fa97884d 100644 --- a/.github/workflows/elixir_release.yml +++ b/.github/workflows/elixir_release.yml @@ -12,28 +12,23 @@ on: jobs: build: runs-on: ubuntu-latest - container: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14 + container: ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-ubuntu20.04 steps: - name: Checkout uses: actions/checkout@v2.4.0 - - name: setup mix - run: | - mix local.hex --force - mix local.rebar --force - mix deps.get - - name: produce emqx.conf.all template - run: make conf-segs + - name: install tools + run: apt update && apt install netcat-openbsd - name: elixir release - run: mix release --overwrite + run: make emqx-elixir - name: start release run: | - cd _build/dev/rel/emqx + cd _build/prod/rel/emqx bin/emqx start - name: check if started run: | sleep 10 nc -zv localhost 1883 - cd _build/dev/rel/emqx + cd _build/prod/rel/emqx bin/emqx ping bin/emqx ctl status diff --git a/.github/workflows/run_api_tests.yaml b/.github/workflows/run_api_tests.yaml index 1935ef438..3e7e3de05 100644 --- a/.github/workflows/run_api_tests.yaml +++ b/.github/workflows/run_api_tests.yaml @@ -16,13 +16,16 @@ jobs: strategy: matrix: otp: - - 24.1.5-3 + - 24.1.5-4 + elixir: + - 1.13.2 os: - ubuntu20.04 arch: - amd64 runs-on: ubuntu-latest - container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}" + container: ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }} + steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index c204d718b..7dd19b2a7 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -12,15 +12,20 @@ jobs: strategy: matrix: otp: - - 23.3.4.9-3 - - 24.1.5-3 + - 23.3.4.9-4 + - 24.1.5-4 + # no need to use more than 1 version of Elixir, since tests + # run using only Erlang code. This is needed just to specify + # the base image. + elixir: + - 1.13.2 os: - ubuntu20.04 arch: - amd64 runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}" + container: "ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}" steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 882f4dd4b..9570b68b3 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -14,7 +14,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14 + container: ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-alpine3.14 steps: - uses: actions/checkout@v2 @@ -41,13 +41,16 @@ jobs: - emqx - emqx-edge - emqx-enterprise + - emqx-elixir cluster_db_backend: - mnesia - rlog os: - alpine3.14 otp: - - 24.1.5-3 + - 24.1.5-4 + elixir: + - 1.13.2 arch: - amd64 exclude: @@ -71,18 +74,22 @@ jobs: - name: load rocksdb cache uses: actions/cache@v2 with: - path: source/_build/default/lib/rocksdb/ - key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }} + path: | + source/_build/default/lib/rocksdb/ + source/deps/rocksdb/ + key: ${{ matrix.os }}-${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }} - name: load quicer cache uses: actions/cache@v2 with: - path: source/_build/default/lib/quicer/ - key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} + path: | + source/_build/default/lib/quicer/ + source/deps/quicer/ + key: ${{ matrix.os }}-${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} - name: make docker image working-directory: source env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }} + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }} run: | make ${{ matrix.profile }}-docker - name: run emqx @@ -91,6 +98,10 @@ jobs: run: | set -x IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh ${{ matrix.profile }}) + if [[ "${{ matrix.profile }}" = *-elixir ]] + then + export IS_ELIXIR=yes + fi ./.ci/docker-compose-file/scripts/run-emqx.sh $IMAGE ${{ matrix.cluster_db_backend }} - name: make paho tests run: | @@ -118,7 +129,9 @@ jobs: os: - alpine3.14 otp: - - 24.1.5-3 + - 24.1.5-4 + elixir: + - 1.13.2 arch: - amd64 # - emqx-enterprise # TODO test enterprise @@ -152,7 +165,7 @@ jobs: - name: make docker image working-directory: source env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }} + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }} run: | make ${{ matrix.profile }}-docker echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 0af50f599..906a5e802 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -19,14 +19,19 @@ jobs: - emqx - emqx-enterprise otp: - - 24.1.5-3 + - 24.1.5-4 + # no need to use more than 1 version of Elixir, since tests + # run using only Erlang code. This is needed just to specify + # the base image. + elixir: + - 1.13.2 os: - ubuntu20.04 arch: - amd64 runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}" + container: "ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" defaults: run: diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index e50993ded..e7bc5dafa 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -16,14 +16,16 @@ jobs: strategy: matrix: otp: - - 24.1.5-3 + - 24.1.5-4 + elixir: + - 1.13.2 os: - ubuntu20.04 arch: - amd64 runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}" + container: "ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" steps: - uses: actions/checkout@v2 diff --git a/.tool-versions b/.tool-versions index 1c687075b..54954aec2 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -erlang 24.1.5-3 -elixir 1.13.1-otp-24 +erlang 24.1.5-4 +elixir 1.13.2-otp-24 diff --git a/Makefile b/Makefile index 07a9ff419..d26a09e1b 100644 --- a/Makefile +++ b/Makefile @@ -3,9 +3,10 @@ REBAR_VERSION = 3.16.1-emqx-1 REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts -export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-alpine3.14 export EMQX_DEFAULT_RUNNER = alpine:3.14 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) +export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) export EMQX_DASHBOARD_VERSION ?= v0.18.0 export DOCKERFILE := deploy/docker/Dockerfile export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing @@ -36,6 +37,22 @@ ensure-rebar3: @$(SCRIPTS)/fail-on-old-otp-version.escript @$(SCRIPTS)/ensure-rebar3.sh $(REBAR_VERSION) +.PHONY: ensure-hex +ensure-hex: + @mix local.hex --if-missing --force + +.PHONY: ensure-mix-rebar3 +ensure-mix-rebar3: $(REBAR) + @mix local.rebar rebar3 $(CURDIR)/rebar3 --if-missing --force + +.PHONY: ensure-mix-rebar +ensure-mix-rebar: $(REBAR) + @mix local.rebar --if-missing --force + +.PHONY: mix-deps-get +mix-deps-get: $(ELIXIR_COMMON_DEPS) + @mix deps.get + $(REBAR): ensure-rebar3 .PHONY: get-dashboard @@ -95,7 +112,7 @@ coveralls: $(REBAR) @ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send .PHONY: $(REL_PROFILES) -$(REL_PROFILES:%=%): $(REBAR) get-dashboard conf-segs +$(REL_PROFILES:%=%): $(COMMON_DEPS) @$(REBAR) as $(@) do release ## Not calling rebar3 clean because @@ -139,6 +156,7 @@ dialyzer: $(REBAR) @$(REBAR) as check dialyzer COMMON_DEPS := $(REBAR) get-dashboard conf-segs +ELIXIR_COMMON_DEPS := ensure-hex ensure-mix-rebar3 ensure-mix-rebar ## rel target is to create release package without relup .PHONY: $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel) @@ -179,13 +197,13 @@ quickrun: ./_build/$(PROFILE)/rel/emqx/bin/emqx console ## docker target is to create docker instructions -.PHONY: $(REL_PROFILES:%=%-docker) +.PHONY: $(REL_PROFILES:%=%-docker) $(REL_PROFILES:%=%-elixir-docker) define gen-docker-target $1-docker: $(COMMON_DEPS) @$(BUILD) $1 docker endef -ALL_TGZS = $(REL_PROFILES) -$(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target,$(zt)))) +ALL_DOCKERS = $(REL_PROFILES) $(REL_PROFILES:%=%-elixir) +$(foreach zt,$(ALL_DOCKERS),$(eval $(call gen-docker-target,$(zt)))) ## emqx-docker-testing ## emqx-enterprise-docker-testing @@ -201,3 +219,26 @@ $(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target-testing,$(zt)))) conf-segs: @scripts/merge-config.escript + +## elixir target is to create release packages using Elixir's Mix +.PHONY: $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir) +$(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir): $(COMMON_DEPS) $(ELIXIR_COMMON_DEPS) mix-deps-get + @$(BUILD) $(subst -elixir,,$(@)) elixir + +.PHONY: $(REL_PROFILES:%=%-elixirpkg) +define gen-elixirpkg-target +# the Elixir places the tar in a different path than Rebar3 +$1-elixirpkg: $1-pkg-elixir + @env TAR_PKG_DIR=_build/prod \ + IS_ELIXIR=yes \ + $(BUILD) $1 pkg +endef +$(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixirpkg-target,$(pt)))) + +.PHONY: $(REL_PROFILES:%=%-elixir-tgz) +define gen-elixir-tgz-target +$1-elixir-tgz: $(COMMON_DEPS) $(ELIXIR_COMMON_DEPS) mix-deps-get + @env IS_ELIXIR=yes $(BUILD) $1 tgz +endef +ALL_ELIXIR_TGZS = $(REL_PROFILES) +$(foreach tt,$(ALL_ELIXIR_TGZS),$(eval $(call gen-elixir-tgz-target,$(tt)))) diff --git a/apps/emqx/src/proto/emqx_cm_proto_v1.erl b/apps/emqx/src/proto/emqx_cm_proto_v1.erl index e8f0115cb..3c673bce0 100644 --- a/apps/emqx/src/proto/emqx_cm_proto_v1.erl +++ b/apps/emqx/src/proto/emqx_cm_proto_v1.erl @@ -32,7 +32,7 @@ ]). -include("bpapi.hrl"). --include("emqx_cm.hrl"). +-include("src/emqx_cm.hrl"). introduced_in() -> "5.0.0". diff --git a/bin/emqx b/bin/emqx index 2f9fe674e..a79e5943b 100755 --- a/bin/emqx +++ b/bin/emqx @@ -864,10 +864,12 @@ case "${COMMAND}" in then "$REL_DIR/elixir" \ --hidden \ + --name "rand-$(relx_gen_id)-$NAME" \ --cookie "$COOKIE" \ --boot "$REL_DIR/start_clean" \ --boot-var RELEASE_LIB "$ERTS_LIB_DIR" \ - --vm-args "$(latest_vm_args 'EMQX_NODE__NAME')"\ + --vm-args "$REL_DIR/remote.vm.args" \ + --erl "-start_epmd false -epmd_module ekka_epmd" \ --rpc-eval "$NAME" "$@" else relx_nodetool "eval" "$@" diff --git a/build b/build index 6790522ca..d5aeeb64f 100755 --- a/build +++ b/build @@ -6,6 +6,11 @@ set -euo pipefail +DEBUG="${DEBUG:-0}" +if [ "$DEBUG" -eq 1 ]; then + set -x +fi + PROFILE="$1" ARTIFACT="$2" @@ -26,7 +31,7 @@ case "$ARCH" in ARCH='arm64' ;; arm*) - ARCH=arm + ARCH='arm64' ;; esac export ARCH @@ -79,6 +84,11 @@ make_rel() { fi } +make_elixir_rel() { + export_release_vars "$PROFILE" + env MIX_ENV=prod mix release --overwrite +} + ## extract previous version .tar.gz files to _build/$PROFILE/rel/emqx before making relup make_relup() { local rel_dir="_build/$PROFILE/rel/emqx" @@ -119,21 +129,35 @@ cp_dyn_libs() { ## Re-pack the relx assembled .tar.gz to EMQ X's package naming scheme ## It assumes the .tar.gz has been built -- relies on Makefile dependency make_tgz() { - # build the tarball again to ensure relup is included - make_rel + local pkgpath="_packages/${PROFILE}" + local tarball + local target + if [ "${IS_ELIXIR:-no}" = "yes" ] + then + # ensure tarball exists + ELIXIR_MAKE_TAR=yes make_elixir_rel + + local relpath="_build/prod" + target="${pkgpath}/${PROFILE}-${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" + else + # build the tarball again to ensure relup is included + # elixir does not have relup yet. + make_rel + + local relpath="_build/${PROFILE}/rel/emqx" + target="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" + fi + + tarball="${relpath}/emqx-${PKG_VSN}.tar.gz" tard="/tmp/emqx_untar_${PKG_VSN}" rm -rf "${tard}" mkdir -p "${tard}/emqx" - local relpath="_build/${PROFILE}/rel/emqx" - local pkgpath="_packages/${PROFILE}" + mkdir -p "${pkgpath}" - local tarball="${relpath}/emqx-${PKG_VSN}.tar.gz" if [ ! -f "$tarball" ]; then log "ERROR: $tarball is not found" fi - local target - target="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" tar zxf "${tarball}" -C "${tard}/emqx" ## try to be portable for tar.gz packages. ## for DEB and RPM packages the dependencies are resoved by yum and apt @@ -198,6 +222,53 @@ make_docker_testing() { -f "${DOCKERFILE_TESTING}" . } +# used to control the Elixir Mix Release output +# see docstring in `mix.exs` +export_release_vars() { + local profile="$1" + case "$profile" in + emqx) + export EMQX_RLEASE_TYPE=cloud \ + EMQX_PACKAGE_TYPE=bin \ + EMQX_EDITION_TYPE=community \ + ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no} + ;; + emqx-edge) + export EMQX_RLEASE_TYPE=edge \ + EMQX_PACKAGE_TYPE=bin \ + EMQX_EDITION_TYPE=community \ + ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no} + ;; + emqx-enterprise) + export EMQX_RLEASE_TYPE=cloud \ + EMQX_PACKAGE_TYPE=bin \ + EMQX_EDITION_TYPE=enterprise \ + ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no} + ;; + emqx-pkg) + export EMQX_RLEASE_TYPE=cloud \ + EMQX_PACKAGE_TYPE=pkg \ + EMQX_EDITION_TYPE=community \ + ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes} + ;; + emqx-edge-pkg) + export EMQX_RLEASE_TYPE=edge \ + EMQX_PACKAGE_TYPE=pkg \ + EMQX_EDITION_TYPE=community \ + ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes} + ;; + emqx-enterprise-pkg) + export EMQX_RLEASE_TYPE=cloud \ + EMQX_PACKAGE_TYPE=pkg \ + EMQX_EDITION_TYPE=enterprise \ + ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes} + ;; + *) + echo Invalid profile "$profile" + exit 1 + esac +} + log "building artifact=$ARTIFACT for profile=$PROFILE" case "$ARTIFACT" in @@ -219,7 +290,10 @@ case "$ARTIFACT" in exit 0 fi make -C "deploy/packages/${PKGERDIR}" clean - EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}" + env EMQX_REL="$(pwd)" \ + EMQX_BUILD="${PROFILE}" \ + SYSTEM="${SYSTEM}" \ + make -C "deploy/packages/${PKGERDIR}" ;; docker) make_docker @@ -227,6 +301,9 @@ case "$ARTIFACT" in docker-testing) make_docker_testing ;; + elixir) + make_elixir_rel + ;; *) log "Unknown artifact $ARTIFACT" exit 1 diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index a075fcf5d..68463fa5e 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-alpine3.14 ARG RUN_FROM=alpine:3.14 FROM ${BUILD_FROM} AS builder @@ -22,17 +22,23 @@ COPY . /emqx ARG EMQX_NAME=emqx -RUN cd /emqx \ - && rm -rf _build/$EMQX_NAME/lib \ - && make $EMQX_NAME +RUN if [[ "$EMQX_NAME" = *-elixir ]]; then \ + export EMQX_LIB_PATH="_build/prod/lib"; \ + export EMQX_REL_PATH="/emqx/_build/prod/rel/emqx"; \ + else \ + export EMQX_LIB_PATH="_build/$EMQX_NAME/lib"; \ + export EMQX_REL_PATH="/emqx/_build/$EMQX_NAME/rel/emqx"; \ + fi \ + && cd /emqx \ + && rm -rf $EMQX_LIB_PATH \ + && make $EMQX_NAME \ + && mkdir -p /emqx-rel \ + && mv $EMQX_REL_PATH /emqx-rel FROM $RUN_FROM -## define ARG again after 'FROM $RUN_FROM' -ARG EMQX_NAME=emqx - COPY deploy/docker/docker-entrypoint.sh /usr/bin/ -COPY --from=builder /emqx/_build/$EMQX_NAME/rel/emqx /opt/emqx +COPY --from=builder /emqx-rel/emqx /opt/emqx RUN ln -s /opt/emqx/bin/* /usr/local/bin/ RUN apk add --no-cache curl ncurses-libs openssl sudo libstdc++ bash diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index 2cb3679ee..6ba61c314 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -6,9 +6,16 @@ BUILT := $(SRCDIR)/BUILT EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) -TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz -SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) -TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) +ifeq ($(IS_ELIXIR), yes) + ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN) +else + ELIXIR_PKG_VSN := +endif + +TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx +TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz +SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) +TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)$(ELIXIR_PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) .PHONY: all all: | $(BUILT) diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index 618f94447..4bdf7f6db 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -16,9 +16,16 @@ endif EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) -TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz -TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) -SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) +ifeq ($(IS_ELIXIR), yes) + ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN) +else + ELIXIR_PKG_VSN := +endif + +TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx +TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz +SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) +TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)$(ELIXIR_PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) # Not $(PWD) as it does not work for make -C @@ -55,4 +62,3 @@ $(BUILT): clean: rm -rf $(SRCDIR) - diff --git a/mix.exs b/mix.exs index ccff14df2..4e9128b9e 100644 --- a/mix.exs +++ b/mix.exs @@ -24,13 +24,6 @@ defmodule EMQXUmbrella.MixProject do Defaults to `community`. """ - # Temporary hack while 1.13.2 is not released - System.version() - |> Version.parse!() - |> Version.compare(Version.parse!("1.13.2")) - |> Kernel.==(:lt) - |> if(do: Code.require_file("lib/mix/release.exs")) - def project() do [ app: :emqx_mix, @@ -124,7 +117,7 @@ defmodule EMQXUmbrella.MixProject do steps = if System.get_env("ELIXIR_MAKE_TAR") == "yes" do - base_steps ++ [:tar] + base_steps ++ [&prepare_tar_overlays/1, :tar] else base_steps end @@ -154,7 +147,6 @@ defmodule EMQXUmbrella.MixProject do def applications(release_type) do [ - logger: :permanent, crypto: :permanent, public_key: :permanent, asn1: :permanent, @@ -234,16 +226,30 @@ defmodule EMQXUmbrella.MixProject do } end + ############################################################################# + # Custom Steps + ############################################################################# + defp copy_files(release, release_type, package_type, edition_type) do overwrite? = Keyword.get(release.options, :overwrite, false) bin = Path.join(release.path, "bin") etc = Path.join(release.path, "etc") + log = Path.join(release.path, "log") Mix.Generator.create_directory(bin) Mix.Generator.create_directory(etc) + Mix.Generator.create_directory(log) Mix.Generator.create_directory(Path.join(etc, "certs")) + Enum.each( + ["mnesia", "configs", "patches", "scripts"], + fn dir -> + path = Path.join([release.path, "data", dir]) + Mix.Generator.create_directory(path) + end + ) + Mix.Generator.copy_file( "apps/emqx_authz/etc/acl.conf", Path.join(etc, "acl.conf"), @@ -409,6 +415,18 @@ defmodule EMQXUmbrella.MixProject do release end + # The `:tar` built-in step in Mix Release does not currently add the + # `etc` directory into the resulting tarball. The workaround is to + # add those to the `:overlays` key before running `:tar`. + # See: https://hexdocs.pm/mix/1.13.2/Mix.Release.html#__struct__/0 + defp prepare_tar_overlays(release) do + Map.update!(release, :overlays, &["etc", "data" | &1]) + end + + ############################################################################# + # Helper functions + ############################################################################# + defp template_vars(release, release_type, :bin = _package_type, edition_type) do [ platform_bin_dir: "bin", @@ -454,7 +472,7 @@ defmodule EMQXUmbrella.MixProject do # FIXME: this is empty in `make emqx` ??? erl_opts: "", emqx_description: emqx_description(release_type, edition_type), - built_on: built_on(), + built_on_arch: built_on(), is_elixir: "yes" ] end diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 2ec943149..fa8cfdce2 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -11,17 +11,23 @@ ## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10 set -euo pipefail +set -x help() { echo - echo "-h|--help: To display this usage information" - echo "--profile : EMQ X profile to build, e.g. emqx, emqx-edge" - echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz" - echo " and pkg for .rpm or .deb" - echo "--arch amd64|arm64: Target arch to build the EMQ X package for" - echo "--src_dir : EMQ X source ode in this dir, default to PWD" - echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10" + echo "-h|--help: To display this usage information" + echo "--profile : EMQ X profile to build, e.g. emqx, emqx-edge" + echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz," + echo " pkg for .rpm or .deb" + echo "--with-elixir: Specify if the release should be built with Elixir, " + echo " defaults to false." + echo "--arch amd64|arm64: Target arch to build the EMQ X package for" + echo "--src_dir : EMQ X source ode in this dir, default to PWD" + echo "--builder : Builder image to pull" + echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10" + echo "--otp : OTP version being used in the builder" + echo "--elixir : Elixir version being used in the builder" + echo "--system : OS used in the builder image" } while [ "$#" -gt 0 ]; do @@ -50,6 +56,22 @@ while [ "$#" -gt 0 ]; do ARCH="$2" shift 2 ;; + --otp) + OTP_VSN="$2" + shift 2 + ;; + --elixir) + ELIXIR_VSN="$2" + shift 2 + ;; + --with-elixir) + WITH_ELIXIR=yes + shift 1 + ;; + --system) + SYSTEM="$2" + shift 2 + ;; *) echo "WARN: Unknown arg (ignored): $1" shift @@ -58,21 +80,43 @@ while [ "$#" -gt 0 ]; do esac done -if [ -z "${PROFILE:-}" ] || [ -z "${PKGTYPE:-}" ] || [ -z "${BUILDER:-}" ] || [ -z "${ARCH:-}" ]; then +if [ -z "${PROFILE:-}" ] || + [ -z "${PKGTYPE:-}" ] || + [ -z "${BUILDER:-}" ] || + [ -z "${ARCH:-}" ] || + [ -z "${OTP_VSN:-}" ] || + [ -z "${ELIXIR_VSN:-}" ] || + [ -z "${SYSTEM:-}" ]; then help exit 1 fi -if [ "$PKGTYPE" != 'tgz' ] && [ "$PKGTYPE" != 'pkg' ]; then +if [ -z "${WITH_ELIXIR:-}" ]; then + WITH_ELIXIR=no +fi + +case "$PKGTYPE" in + tgz|pkg) + true + ;; + *) echo "Bad --pkgtype option, should be tgz or pkg" exit 1 -fi + ;; +esac cd "${SRC_DIR:-.}" PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}" -OTP_VSN_SYSTEM=$(echo "$BUILDER" | cut -d ':' -f2) -PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN_SYSTEM}-${ARCH}" + +if [ "$WITH_ELIXIR" = "yes" ] +then + PKG_NAME="${PROFILE}-${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" + MAKE_TARGET="${PROFILE}-elixir-${PKGTYPE}" +else + PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" + MAKE_TARGET="${PROFILE}-${PKGTYPE}" +fi docker info docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}" @@ -82,4 +126,4 @@ docker run -i --rm \ --platform="linux/$ARCH" \ -e EMQX_NAME="$PROFILE" \ "$BUILDER" \ - bash -euc "make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PKG_NAME $PKGTYPE" + bash -euc "make ${MAKE_TARGET} && .ci/build_packages/tests.sh $PKG_NAME $PKGTYPE $ARCH" diff --git a/scripts/get-elixir-vsn.sh b/scripts/get-elixir-vsn.sh new file mode 100755 index 000000000..db60325e7 --- /dev/null +++ b/scripts/get-elixir-vsn.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if command -v elixir &>/dev/null +then + elixir -e "System.version() |> IO.puts()" +fi