yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ci(mix): build emqx packages with elixir

This commit is contained in:
Thales Macedo Garitezi 2022-01-04 17:56:40 -03:00
parent bf8d30f4f3
commit 6dd11665bb
No known key found for this signature in database
GPG Key ID: DD279F8152A9B6DD
9 changed files with 232 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.ci/build_packages/tests.sh
View File

@ -4,14 +4,19 @@ set -euo pipefail
set -x
if [ -z "${1:-}" ]; then
echo "Usage $0 <PACKAGE_NAME> tgz|pkg"
echo "Usage $0 <PACKAGE_NAME> tgz|pkg|elixirpkg"
exit 1
fi
if [ "${2:-}" != 'tgz' ] && [ "${2:-}" != 'pkg' ]; then
echo "Usage $0 <PACKAGE_NAME> tgz|pkg"
case "${2:-}" in
tgz|pkg|elixirpkg)
true
;;
*)
echo "Usage $0 <PACKAGE_NAME> zip|pkg|elixirpkg"
exit 1
fi
;;
esac
PACKAGE_NAME="${1}"
PACKAGE_TYPE="${2}"

37
.github/workflows/build_packages.yaml vendored
View File

@ -19,7 +19,8 @@ jobs:
prepare:
runs-on: ubuntu-20.04
# prepare source with any OTP version, no need for a matrix
container: "ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04"
# FIXME: use tagged version once merged
container: "ghcr.io/emqx/emqx-builder/elixir:24.1.5-3-1.13.1-ubuntu20.04"
outputs:
ce_old_vsns: ${{ steps.find_old_versons.outputs.ce_old_vsns }}
@ -237,6 +238,8 @@ jobs:
- emqx-enterprise
otp:
- 24.1.5-3 # we test with OTP 23, but only build package on OTP 24 versions
elixir:
- 1.13.1
arch:
- amd64
- arm64
@ -334,22 +337,39 @@ jobs:
- name: build emqx packages
env:
OTP: ${{ matrix.otp }}
ELIXIR: ${{ matrix.elixir }}
PROFILE: ${{ matrix.profile }}
ARCH: ${{ matrix.arch }}
SYSTEM: ${{ matrix.os }}
working-directory: source
# FIXME: use tagged version once merged
run: |
./scripts/buildx.sh \
--profile "${PROFILE}" \
--pkgtype "tgz" \
--arch "${ARCH}" \
--builder "ghcr.io/emqx/emqx-builder/5.0-3:${OTP}-${SYSTEM}"
--otp "${OTP}" \
--elixir "${ELIXIR}" \
--system "${SYSTEM}" \
--builder "ghcr.io/emqx/emqx-builder/elixir:${OTP}-${ELIXIR}-${SYSTEM}"
## the pkg build is incremental on the tgz build
./scripts/buildx.sh \
--profile "${PROFILE}" \
--pkgtype "pkg" \
--arch "${ARCH}" \
--builder "ghcr.io/emqx/emqx-builder/5.0-3:${OTP}-${SYSTEM}"
--otp "${OTP}" \
--elixir "${ELIXIR}" \
--system "${SYSTEM}" \
--builder "ghcr.io/emqx/emqx-builder/elixir:${OTP}-${ELIXIR}-${SYSTEM}"
./scripts/buildx.sh \
--profile "${PROFILE}" \
--pkgtype "elixirpkg" \
--arch "${ARCH}" \
--otp "${OTP}" \
--elixir "${ELIXIR}" \
--system "${SYSTEM}" \
--builder "ghcr.io/emqx/emqx-builder/elixir:${OTP}-${ELIXIR}-${SYSTEM}"
- name: create sha256
env:
@ -380,9 +400,15 @@ jobs:
- emqx-edge
- emqx
- emqx-enterprise
# NOTE: for docker, only support latest otp version, not a matrix
# NOTE: for docker, only support latest otp and elixir
# version, not a matrix
otp:
- 24.1.5-3 # update to latest
elixir:
- 1.13.1 # update to latest
arch:
- amd64
- arm64
steps:
- uses: actions/download-artifact@v2
@ -423,7 +449,8 @@ jobs:
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-alpine3.14
# FIXME: use tagged version once merged
BUILD_FROM=ghcr.io/emqx/emqx-builder/elixir:${{ matrix.otp }}-${{ matrix.elixir }}-alpine3.14
RUN_FROM=alpine:3.14
EMQX_NAME=${{ matrix.profile }}
file: source/deploy/docker/Dockerfile

35
Makefile
View File

@ -6,6 +6,7 @@ SCRIPTS = $(CURDIR)/scripts
export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14
export EMQX_DEFAULT_RUNNER = alpine:3.14
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
export EMQX_DASHBOARD_VERSION ?= v0.18.0
export DOCKERFILE := deploy/docker/Dockerfile
export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing
@ -36,6 +37,22 @@ ensure-rebar3:
@$(SCRIPTS)/fail-on-old-otp-version.escript
@$(SCRIPTS)/ensure-rebar3.sh $(REBAR_VERSION)
.PHONY: ensure-hex
ensure-hex:
@mix local.hex --if-missing --force
.PHONY: ensure-mix-rebar3
ensure-mix-rebar3: $(REBAR)
@mix local.rebar rebar3 $(CURDIR)/rebar3 --if-missing --force
.PHONY: ensure-mix-rebar
ensure-mix-rebar: $(REBAR)
@mix local.rebar --if-missing --force
.PHONY: mix-deps-get
mix-deps-get: $(ELIXIR_COMMON_DEPS)
@mix deps.get
$(REBAR): ensure-rebar3
.PHONY: get-dashboard
@ -95,7 +112,7 @@ coveralls: $(REBAR)
@ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send
.PHONY: $(REL_PROFILES)
$(REL_PROFILES:%=%): $(REBAR) get-dashboard conf-segs
$(REL_PROFILES:%=%): $(COMMON_DEPS)
@$(REBAR) as $(@) do release
## Not calling rebar3 clean because
@ -139,6 +156,7 @@ dialyzer: $(REBAR)
@$(REBAR) as check dialyzer
COMMON_DEPS := $(REBAR) get-dashboard conf-segs
ELIXIR_COMMON_DEPS := ensure-hex ensure-mix-rebar3 ensure-mix-rebar
## rel target is to create release package without relup
.PHONY: $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel)
@ -201,3 +219,18 @@ $(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target-testing,$(zt))))
conf-segs:
@scripts/merge-config.escript
## elixir target is to create release packages using Elixir's Mix
.PHONY: $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir)
$(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir): $(COMMON_DEPS) $(ELIXIR_COMMON_DEPS) mix-deps-get
@$(BUILD) $(subst -elixir,,$(@)) elixir
.PHONY: $(REL_PROFILES:%=%-elixirpkg)
define gen-elixirpkg-target
# the Elixir places the tar in a different path than Rebar3
$1-elixirpkg: $1-pkg-elixir
@env TAR_PKG_DIR=_build/prod \
IS_ELIXIR=yes \
$(BUILD) $1 pkg
endef
$(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixirpkg-target,$(pt))))

61
build
View File

@ -6,6 +6,11 @@
set -euo pipefail
DEBUG="${DEBUG:-0}"
if [ "$DEBUG" -eq 1 ]; then
set -x
fi
PROFILE="$1"
ARTIFACT="$2"
@ -198,6 +203,53 @@ make_docker_testing() {
-f "${DOCKERFILE_TESTING}" .
}
# used to control the Elixir Mix Release output
# see docstring in `mix.exs`
export_release_vars() {
local profile="$1"
case "$profile" in
emqx)
export EMQX_RLEASE_TYPE=cloud \
EMQX_PACKAGE_TYPE=bin \
EMQX_EDITION_TYPE=community \
ELIXIR_MAKE_TAR=no
;;
emqx-edge)
export EMQX_RLEASE_TYPE=edge \
EMQX_PACKAGE_TYPE=bin \
EMQX_EDITION_TYPE=community \
ELIXIR_MAKE_TAR=no
;;
emqx-enterprise)
export EMQX_RLEASE_TYPE=cloud \
EMQX_PACKAGE_TYPE=bin \
EMQX_EDITION_TYPE=enterprise \
ELIXIR_MAKE_TAR=no
;;
emqx-pkg)
export EMQX_RLEASE_TYPE=cloud \
EMQX_PACKAGE_TYPE=pkg \
EMQX_EDITION_TYPE=community \
ELIXIR_MAKE_TAR=yes
;;
emqx-edge-pkg)
export EMQX_RLEASE_TYPE=edge \
EMQX_PACKAGE_TYPE=pkg \
EMQX_EDITION_TYPE=community \
ELIXIR_MAKE_TAR=yes
;;
emqx-enterprise-pkg)
export EMQX_RLEASE_TYPE=cloud \
EMQX_PACKAGE_TYPE=pkg \
EMQX_EDITION_TYPE=enterprise \
ELIXIR_MAKE_TAR=yes
;;
*)
echo Invalid profile "$profile"
exit 1
esac
}
log "building artifact=$ARTIFACT for profile=$PROFILE"
case "$ARTIFACT" in
@ -219,7 +271,10 @@ case "$ARTIFACT" in
exit 0
fi
make -C "deploy/packages/${PKGERDIR}" clean
EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}"
env EMQX_REL="$(pwd)" \
EMQX_BUILD="${PROFILE}" \
SYSTEM="${SYSTEM}" \
make -C "deploy/packages/${PKGERDIR}"
;;
docker)
make_docker
@ -227,6 +282,10 @@ case "$ARTIFACT" in
docker-testing)
make_docker_testing
;;
elixir)
export_release_vars "$PROFILE"
env MIX_ENV=prod mix release --overwrite
;;
*)
log "Unknown artifact $ARTIFACT"
exit 1

13
deploy/packages/deb/Makefile
View File

@ -6,9 +6,16 @@ BUILT := $(SRCDIR)/BUILT
EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz
SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture)
TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH)
ifeq ($(IS_ELIXIR), yes)
ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN)
else
ELIXIR_PKG_VSN :=
endif
TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx
TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz
SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture)
TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)$(ELIXIR_PKG_VSN)-$(SYSTEM)-$(ARCH)
.PHONY: all
all: | $(BUILT)

14
deploy/packages/rpm/Makefile
View File

@ -16,9 +16,16 @@ endif
EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz
TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH)
SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m)
ifeq ($(IS_ELIXIR), yes)
ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN)
else
ELIXIR_PKG_VSN :=
endif
TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx
TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz
SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m)
TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)$(ELIXIR_PKG_VSN)-$(SYSTEM)-$(ARCH)
SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi)
# Not $(PWD) as it does not work for make -C
@ -55,4 +62,3 @@ $(BUILT):
clean:
rm -rf $(SRCDIR)

28
mix.exs
View File

@ -124,7 +124,7 @@ defmodule EMQXUmbrella.MixProject do
steps =
if System.get_env("ELIXIR_MAKE_TAR") == "yes" do
base_steps ++ [:tar]
base_steps ++ [&prepare_tar_overlays/1, :tar]
else
base_steps
end
@ -234,6 +234,10 @@ defmodule EMQXUmbrella.MixProject do
}
end
#############################################################################
# Custom Steps
#############################################################################
defp copy_files(release, release_type, package_type, edition_type) do
overwrite? = Keyword.get(release.options, :overwrite, false)
@ -409,6 +413,26 @@ defmodule EMQXUmbrella.MixProject do
release
end
# The `:tar` built-in step in Mix Release does not currently add the
# `etc` directory into the resulting tarball. The workaround is to
# add those to the `:overlays` key before running `:tar`.
# See: https://hexdocs.pm/mix/1.13.1/Mix.Release.html#__struct__/0
defp prepare_tar_overlays(release) do
Enum.each(
["mnesia", "configs", "patches", "scripts"],
fn dir ->
path = Path.join([release.path, "data", dir])
File.mkdir_p!(path)
end
)
Map.update!(release, :overlays, &["etc", "data" | &1])
end
#############################################################################
# Helper functions
#############################################################################
defp template_vars(release, release_type, :bin = _package_type, edition_type) do
[
platform_bin_dir: "bin",
@ -454,7 +478,7 @@ defmodule EMQXUmbrella.MixProject do
# FIXME: this is empty in `make emqx` ???
erl_opts: "",
emqx_description: emqx_description(release_type, edition_type),
built_on: built_on(),
built_on_arch: built_on(),
is_elixir: "yes"
]
end

60
scripts/buildx.sh
View File

@ -11,17 +11,21 @@
## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10
set -euo pipefail
set -x
help() {
echo
echo "-h|--help: To display this usage information"
echo "--profile <PROFILE>: EMQ X profile to build, e.g. emqx, emqx-edge"
echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz"
echo " and pkg for .rpm or .deb"
echo "--arch amd64|arm64: Target arch to build the EMQ X package for"
echo "--src_dir <SRC_DIR>: EMQ X source ode in this dir, default to PWD"
echo "--builder <BUILDER>: Builder image to pull"
echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10"
echo "-h|--help: To display this usage information"
echo "--profile <PROFILE>: EMQ X profile to build, e.g. emqx, emqx-edge"
echo "--pkgtype tgz|pkg|elixirpkg: Specify which package to build, tgz for .tar.gz,"
echo " pkg and elixirpkg for .rpm or .deb"
echo "--arch amd64|arm64: Target arch to build the EMQ X package for"
echo "--src_dir <SRC_DIR>: EMQ X source ode in this dir, default to PWD"
echo "--builder <BUILDER>: Builder image to pull"
echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10"
echo "--otp <OTP_VSN>: OTP version being used in the builder"
echo "--elixir <ELIXIR_VSN>: Elixir version being used in the builder"
echo "--system <SYSTEM>: OS used in the builder image"
}
while [ "$#" -gt 0 ]; do
@ -50,6 +54,18 @@ while [ "$#" -gt 0 ]; do
ARCH="$2"
shift 2
;;
--otp)
OTP_VSN="$2"
shift 2
;;
--elixir)
ELIXIR_VSN="$2"
shift 2
;;
--system)
SYSTEM="$2"
shift 2
;;
*)
echo "WARN: Unknown arg (ignored): $1"
shift
@ -58,21 +74,37 @@ while [ "$#" -gt 0 ]; do
esac
done
if [ -z "${PROFILE:-}" ] || [ -z "${PKGTYPE:-}" ] || [ -z "${BUILDER:-}" ] || [ -z "${ARCH:-}" ]; then
if [ -z "${PROFILE:-}" ] ||
[ -z "${PKGTYPE:-}" ] ||
[ -z "${BUILDER:-}" ] ||
[ -z "${ARCH:-}" ] ||
[ -z "${OTP_VSN:-}" ] ||
[ -z "${ELIXIR_VSN:-}" ] ||
[ -z "${SYSTEM:-}" ]; then
help
exit 1
fi
if [ "$PKGTYPE" != 'tgz' ] && [ "$PKGTYPE" != 'pkg' ]; then
echo "Bad --pkgtype option, should be tgz or pkg"
case "$PKGTYPE" in
tgz|pkg|elixirpkg)
true
;;
*)
echo "Bad --pkgtype option, should be zip or pkg"
exit 1
fi
;;
esac
cd "${SRC_DIR:-.}"
PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}"
OTP_VSN_SYSTEM=$(echo "$BUILDER" | cut -d ':' -f2)
PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN_SYSTEM}-${ARCH}"
if [ "$PKGTYPE" = "elixirpkg" ]
then
PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-elixir${ELIXIR_VSN}-${SYSTEM}-${ARCH}"
else
PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
fi
docker info
docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}"

5
scripts/get-elixir-vsn.sh Executable file
View File

@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
elixir -e "System.version() |> IO.puts()"