From 6dd11665bbf2d04edb4ae4b1fc2aa3b3b8f3d4c2 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 4 Jan 2022 17:56:40 -0300 Subject: [PATCH] ci(mix): build emqx packages with elixir --- .ci/build_packages/tests.sh | 13 ++++-- .github/workflows/build_packages.yaml | 37 +++++++++++++--- Makefile | 35 ++++++++++++++- build | 61 ++++++++++++++++++++++++++- deploy/packages/deb/Makefile | 13 ++++-- deploy/packages/rpm/Makefile | 14 ++++-- mix.exs | 28 +++++++++++- scripts/buildx.sh | 60 ++++++++++++++++++++------ scripts/get-elixir-vsn.sh | 5 +++ 9 files changed, 232 insertions(+), 34 deletions(-) create mode 100755 scripts/get-elixir-vsn.sh diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 75a973990..2909e6909 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -4,14 +4,19 @@ set -euo pipefail set -x if [ -z "${1:-}" ]; then - echo "Usage $0 tgz|pkg" + echo "Usage $0 tgz|pkg|elixirpkg" exit 1 fi -if [ "${2:-}" != 'tgz' ] && [ "${2:-}" != 'pkg' ]; then - echo "Usage $0 tgz|pkg" +case "${2:-}" in + tgz|pkg|elixirpkg) + true + ;; + *) + echo "Usage $0 zip|pkg|elixirpkg" exit 1 -fi + ;; +esac PACKAGE_NAME="${1}" PACKAGE_TYPE="${2}" diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 2a7d43958..aa7a521ff 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -19,7 +19,8 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04" + # FIXME: use tagged version once merged + container: "ghcr.io/emqx/emqx-builder/elixir:24.1.5-3-1.13.1-ubuntu20.04" outputs: ce_old_vsns: ${{ steps.find_old_versons.outputs.ce_old_vsns }} @@ -237,6 +238,8 @@ jobs: - emqx-enterprise otp: - 24.1.5-3 # we test with OTP 23, but only build package on OTP 24 versions + elixir: + - 1.13.1 arch: - amd64 - arm64 @@ -334,22 +337,39 @@ jobs: - name: build emqx packages env: OTP: ${{ matrix.otp }} + ELIXIR: ${{ matrix.elixir }} PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} working-directory: source + # FIXME: use tagged version once merged run: | ./scripts/buildx.sh \ --profile "${PROFILE}" \ --pkgtype "tgz" \ --arch "${ARCH}" \ - --builder "ghcr.io/emqx/emqx-builder/5.0-3:${OTP}-${SYSTEM}" + --otp "${OTP}" \ + --elixir "${ELIXIR}" \ + --system "${SYSTEM}" \ + --builder "ghcr.io/emqx/emqx-builder/elixir:${OTP}-${ELIXIR}-${SYSTEM}" ## the pkg build is incremental on the tgz build ./scripts/buildx.sh \ --profile "${PROFILE}" \ --pkgtype "pkg" \ --arch "${ARCH}" \ - --builder "ghcr.io/emqx/emqx-builder/5.0-3:${OTP}-${SYSTEM}" + --otp "${OTP}" \ + --elixir "${ELIXIR}" \ + --system "${SYSTEM}" \ + --builder "ghcr.io/emqx/emqx-builder/elixir:${OTP}-${ELIXIR}-${SYSTEM}" + + ./scripts/buildx.sh \ + --profile "${PROFILE}" \ + --pkgtype "elixirpkg" \ + --arch "${ARCH}" \ + --otp "${OTP}" \ + --elixir "${ELIXIR}" \ + --system "${SYSTEM}" \ + --builder "ghcr.io/emqx/emqx-builder/elixir:${OTP}-${ELIXIR}-${SYSTEM}" - name: create sha256 env: @@ -380,9 +400,15 @@ jobs: - emqx-edge - emqx - emqx-enterprise - # NOTE: for docker, only support latest otp version, not a matrix + # NOTE: for docker, only support latest otp and elixir + # version, not a matrix otp: - 24.1.5-3 # update to latest + elixir: + - 1.13.1 # update to latest + arch: + - amd64 + - arm64 steps: - uses: actions/download-artifact@v2 @@ -423,7 +449,8 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-alpine3.14 + # FIXME: use tagged version once merged + BUILD_FROM=ghcr.io/emqx/emqx-builder/elixir:${{ matrix.otp }}-${{ matrix.elixir }}-alpine3.14 RUN_FROM=alpine:3.14 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile diff --git a/Makefile b/Makefile index 07a9ff419..1b34ee07d 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ SCRIPTS = $(CURDIR)/scripts export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14 export EMQX_DEFAULT_RUNNER = alpine:3.14 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) +export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) export EMQX_DASHBOARD_VERSION ?= v0.18.0 export DOCKERFILE := deploy/docker/Dockerfile export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing @@ -36,6 +37,22 @@ ensure-rebar3: @$(SCRIPTS)/fail-on-old-otp-version.escript @$(SCRIPTS)/ensure-rebar3.sh $(REBAR_VERSION) +.PHONY: ensure-hex +ensure-hex: + @mix local.hex --if-missing --force + +.PHONY: ensure-mix-rebar3 +ensure-mix-rebar3: $(REBAR) + @mix local.rebar rebar3 $(CURDIR)/rebar3 --if-missing --force + +.PHONY: ensure-mix-rebar +ensure-mix-rebar: $(REBAR) + @mix local.rebar --if-missing --force + +.PHONY: mix-deps-get +mix-deps-get: $(ELIXIR_COMMON_DEPS) + @mix deps.get + $(REBAR): ensure-rebar3 .PHONY: get-dashboard @@ -95,7 +112,7 @@ coveralls: $(REBAR) @ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send .PHONY: $(REL_PROFILES) -$(REL_PROFILES:%=%): $(REBAR) get-dashboard conf-segs +$(REL_PROFILES:%=%): $(COMMON_DEPS) @$(REBAR) as $(@) do release ## Not calling rebar3 clean because @@ -139,6 +156,7 @@ dialyzer: $(REBAR) @$(REBAR) as check dialyzer COMMON_DEPS := $(REBAR) get-dashboard conf-segs +ELIXIR_COMMON_DEPS := ensure-hex ensure-mix-rebar3 ensure-mix-rebar ## rel target is to create release package without relup .PHONY: $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel) @@ -201,3 +219,18 @@ $(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target-testing,$(zt)))) conf-segs: @scripts/merge-config.escript + +## elixir target is to create release packages using Elixir's Mix +.PHONY: $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir) +$(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir): $(COMMON_DEPS) $(ELIXIR_COMMON_DEPS) mix-deps-get + @$(BUILD) $(subst -elixir,,$(@)) elixir + +.PHONY: $(REL_PROFILES:%=%-elixirpkg) +define gen-elixirpkg-target +# the Elixir places the tar in a different path than Rebar3 +$1-elixirpkg: $1-pkg-elixir + @env TAR_PKG_DIR=_build/prod \ + IS_ELIXIR=yes \ + $(BUILD) $1 pkg +endef +$(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixirpkg-target,$(pt)))) diff --git a/build b/build index 6790522ca..26da3984d 100755 --- a/build +++ b/build @@ -6,6 +6,11 @@ set -euo pipefail +DEBUG="${DEBUG:-0}" +if [ "$DEBUG" -eq 1 ]; then + set -x +fi + PROFILE="$1" ARTIFACT="$2" @@ -198,6 +203,53 @@ make_docker_testing() { -f "${DOCKERFILE_TESTING}" . } +# used to control the Elixir Mix Release output +# see docstring in `mix.exs` +export_release_vars() { + local profile="$1" + case "$profile" in + emqx) + export EMQX_RLEASE_TYPE=cloud \ + EMQX_PACKAGE_TYPE=bin \ + EMQX_EDITION_TYPE=community \ + ELIXIR_MAKE_TAR=no + ;; + emqx-edge) + export EMQX_RLEASE_TYPE=edge \ + EMQX_PACKAGE_TYPE=bin \ + EMQX_EDITION_TYPE=community \ + ELIXIR_MAKE_TAR=no + ;; + emqx-enterprise) + export EMQX_RLEASE_TYPE=cloud \ + EMQX_PACKAGE_TYPE=bin \ + EMQX_EDITION_TYPE=enterprise \ + ELIXIR_MAKE_TAR=no + ;; + emqx-pkg) + export EMQX_RLEASE_TYPE=cloud \ + EMQX_PACKAGE_TYPE=pkg \ + EMQX_EDITION_TYPE=community \ + ELIXIR_MAKE_TAR=yes + ;; + emqx-edge-pkg) + export EMQX_RLEASE_TYPE=edge \ + EMQX_PACKAGE_TYPE=pkg \ + EMQX_EDITION_TYPE=community \ + ELIXIR_MAKE_TAR=yes + ;; + emqx-enterprise-pkg) + export EMQX_RLEASE_TYPE=cloud \ + EMQX_PACKAGE_TYPE=pkg \ + EMQX_EDITION_TYPE=enterprise \ + ELIXIR_MAKE_TAR=yes + ;; + *) + echo Invalid profile "$profile" + exit 1 + esac +} + log "building artifact=$ARTIFACT for profile=$PROFILE" case "$ARTIFACT" in @@ -219,7 +271,10 @@ case "$ARTIFACT" in exit 0 fi make -C "deploy/packages/${PKGERDIR}" clean - EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}" + env EMQX_REL="$(pwd)" \ + EMQX_BUILD="${PROFILE}" \ + SYSTEM="${SYSTEM}" \ + make -C "deploy/packages/${PKGERDIR}" ;; docker) make_docker @@ -227,6 +282,10 @@ case "$ARTIFACT" in docker-testing) make_docker_testing ;; + elixir) + export_release_vars "$PROFILE" + env MIX_ENV=prod mix release --overwrite + ;; *) log "Unknown artifact $ARTIFACT" exit 1 diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index 2cb3679ee..3b51a1442 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -6,9 +6,16 @@ BUILT := $(SRCDIR)/BUILT EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) -TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz -SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) -TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) +ifeq ($(IS_ELIXIR), yes) + ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN) +else + ELIXIR_PKG_VSN := +endif + +TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx +TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz +SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) +TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)$(ELIXIR_PKG_VSN)-$(SYSTEM)-$(ARCH) .PHONY: all all: | $(BUILT) diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index acee8b51c..fbb5f899b 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -16,9 +16,16 @@ endif EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) -TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz -TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) -SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) +ifeq ($(IS_ELIXIR), yes) + ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN) +else + ELIXIR_PKG_VSN := +endif + +TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx +TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz +SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) +TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)$(ELIXIR_PKG_VSN)-$(SYSTEM)-$(ARCH) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) # Not $(PWD) as it does not work for make -C @@ -55,4 +62,3 @@ $(BUILT): clean: rm -rf $(SRCDIR) - diff --git a/mix.exs b/mix.exs index 88a243e58..1c6a8d98c 100644 --- a/mix.exs +++ b/mix.exs @@ -124,7 +124,7 @@ defmodule EMQXUmbrella.MixProject do steps = if System.get_env("ELIXIR_MAKE_TAR") == "yes" do - base_steps ++ [:tar] + base_steps ++ [&prepare_tar_overlays/1, :tar] else base_steps end @@ -234,6 +234,10 @@ defmodule EMQXUmbrella.MixProject do } end + ############################################################################# + # Custom Steps + ############################################################################# + defp copy_files(release, release_type, package_type, edition_type) do overwrite? = Keyword.get(release.options, :overwrite, false) @@ -409,6 +413,26 @@ defmodule EMQXUmbrella.MixProject do release end + # The `:tar` built-in step in Mix Release does not currently add the + # `etc` directory into the resulting tarball. The workaround is to + # add those to the `:overlays` key before running `:tar`. + # See: https://hexdocs.pm/mix/1.13.1/Mix.Release.html#__struct__/0 + defp prepare_tar_overlays(release) do + Enum.each( + ["mnesia", "configs", "patches", "scripts"], + fn dir -> + path = Path.join([release.path, "data", dir]) + File.mkdir_p!(path) + end + ) + + Map.update!(release, :overlays, &["etc", "data" | &1]) + end + + ############################################################################# + # Helper functions + ############################################################################# + defp template_vars(release, release_type, :bin = _package_type, edition_type) do [ platform_bin_dir: "bin", @@ -454,7 +478,7 @@ defmodule EMQXUmbrella.MixProject do # FIXME: this is empty in `make emqx` ??? erl_opts: "", emqx_description: emqx_description(release_type, edition_type), - built_on: built_on(), + built_on_arch: built_on(), is_elixir: "yes" ] end diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 2ec943149..74b5e8267 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -11,17 +11,21 @@ ## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10 set -euo pipefail +set -x help() { echo - echo "-h|--help: To display this usage information" - echo "--profile : EMQ X profile to build, e.g. emqx, emqx-edge" - echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz" - echo " and pkg for .rpm or .deb" - echo "--arch amd64|arm64: Target arch to build the EMQ X package for" - echo "--src_dir : EMQ X source ode in this dir, default to PWD" - echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10" + echo "-h|--help: To display this usage information" + echo "--profile : EMQ X profile to build, e.g. emqx, emqx-edge" + echo "--pkgtype tgz|pkg|elixirpkg: Specify which package to build, tgz for .tar.gz," + echo " pkg and elixirpkg for .rpm or .deb" + echo "--arch amd64|arm64: Target arch to build the EMQ X package for" + echo "--src_dir : EMQ X source ode in this dir, default to PWD" + echo "--builder : Builder image to pull" + echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10" + echo "--otp : OTP version being used in the builder" + echo "--elixir : Elixir version being used in the builder" + echo "--system : OS used in the builder image" } while [ "$#" -gt 0 ]; do @@ -50,6 +54,18 @@ while [ "$#" -gt 0 ]; do ARCH="$2" shift 2 ;; + --otp) + OTP_VSN="$2" + shift 2 + ;; + --elixir) + ELIXIR_VSN="$2" + shift 2 + ;; + --system) + SYSTEM="$2" + shift 2 + ;; *) echo "WARN: Unknown arg (ignored): $1" shift @@ -58,21 +74,37 @@ while [ "$#" -gt 0 ]; do esac done -if [ -z "${PROFILE:-}" ] || [ -z "${PKGTYPE:-}" ] || [ -z "${BUILDER:-}" ] || [ -z "${ARCH:-}" ]; then +if [ -z "${PROFILE:-}" ] || + [ -z "${PKGTYPE:-}" ] || + [ -z "${BUILDER:-}" ] || + [ -z "${ARCH:-}" ] || + [ -z "${OTP_VSN:-}" ] || + [ -z "${ELIXIR_VSN:-}" ] || + [ -z "${SYSTEM:-}" ]; then help exit 1 fi -if [ "$PKGTYPE" != 'tgz' ] && [ "$PKGTYPE" != 'pkg' ]; then - echo "Bad --pkgtype option, should be tgz or pkg" +case "$PKGTYPE" in + tgz|pkg|elixirpkg) + true + ;; + *) + echo "Bad --pkgtype option, should be zip or pkg" exit 1 -fi + ;; +esac cd "${SRC_DIR:-.}" PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}" -OTP_VSN_SYSTEM=$(echo "$BUILDER" | cut -d ':' -f2) -PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN_SYSTEM}-${ARCH}" + +if [ "$PKGTYPE" = "elixirpkg" ] +then + PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-elixir${ELIXIR_VSN}-${SYSTEM}-${ARCH}" +else + PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" +fi docker info docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}" diff --git a/scripts/get-elixir-vsn.sh b/scripts/get-elixir-vsn.sh new file mode 100755 index 000000000..c74af4bf6 --- /dev/null +++ b/scripts/get-elixir-vsn.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -euo pipefail + +elixir -e "System.version() |> IO.puts()"