ci(mix): build emqx packages with elixir

.ci/build_packages/tests.sh

@@ -4,14 +4,19 @@ set -euo pipefail
 set -x
 
 if [ -z "${1:-}" ]; then
-    echo "Usage $0 <PACKAGE_NAME> tgz|pkg"
+    echo "Usage $0 <PACKAGE_NAME> tgz|pkg|elixirpkg"
     exit 1
 fi
 
-if [ "${2:-}" != 'tgz' ] && [ "${2:-}" != 'pkg' ]; then
+case "${2:-}" in
-    echo "Usage $0 <PACKAGE_NAME> tgz|pkg"
+  tgz|pkg|elixirpkg)
+    true
+    ;;
+  *)
+    echo "Usage $0 <PACKAGE_NAME> zip|pkg|elixirpkg"
     exit 1
-fi
+    ;;
+esac
 
 PACKAGE_NAME="${1}"
 PACKAGE_TYPE="${2}"

.github/workflows/build_packages.yaml

@@ -19,7 +19,8 @@ jobs:
   prepare:
     runs-on: ubuntu-20.04
     # prepare source with any OTP version, no need for a matrix
-    container: "ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04"
+    # FIXME: use tagged version once merged
+    container: "ghcr.io/emqx/emqx-builder/elixir:24.1.5-3-1.13.1-ubuntu20.04"
 
     outputs:
       ce_old_vsns: ${{ steps.find_old_versons.outputs.ce_old_vsns }}
@@ -237,6 +238,8 @@ jobs:
           - emqx-enterprise
         otp:
           - 24.1.5-3 # we test with OTP 23, but only build package on OTP 24 versions
+        elixir:
+          - 1.13.1
         arch:
           - amd64
           - arm64
@@ -334,22 +337,39 @@ jobs:
     - name: build emqx packages
       env:
         OTP: ${{ matrix.otp }}
+        ELIXIR: ${{ matrix.elixir }}
         PROFILE: ${{ matrix.profile }}
         ARCH: ${{ matrix.arch }}
         SYSTEM: ${{ matrix.os }}
       working-directory: source
+      # FIXME: use tagged version once merged
       run: |
         ./scripts/buildx.sh \
           --profile "${PROFILE}" \
           --pkgtype "tgz" \
           --arch "${ARCH}" \
-          --builder "ghcr.io/emqx/emqx-builder/5.0-3:${OTP}-${SYSTEM}"
+          --otp "${OTP}" \
+          --elixir "${ELIXIR}" \
+          --system "${SYSTEM}" \
+          --builder "ghcr.io/emqx/emqx-builder/elixir:${OTP}-${ELIXIR}-${SYSTEM}"
         ## the pkg build is incremental on the tgz build
         ./scripts/buildx.sh \
           --profile "${PROFILE}" \
           --pkgtype "pkg" \
           --arch "${ARCH}" \
-          --builder "ghcr.io/emqx/emqx-builder/5.0-3:${OTP}-${SYSTEM}"
+          --otp "${OTP}" \
+          --elixir "${ELIXIR}" \
+          --system "${SYSTEM}" \
+          --builder "ghcr.io/emqx/emqx-builder/elixir:${OTP}-${ELIXIR}-${SYSTEM}"
+
+        ./scripts/buildx.sh \
+          --profile "${PROFILE}" \
+          --pkgtype "elixirpkg" \
+          --arch "${ARCH}" \
+          --otp "${OTP}" \
+          --elixir "${ELIXIR}" \
+          --system "${SYSTEM}" \
+          --builder "ghcr.io/emqx/emqx-builder/elixir:${OTP}-${ELIXIR}-${SYSTEM}"
 
     - name: create sha256
       env:
@@ -380,9 +400,15 @@ jobs:
           - emqx-edge
           - emqx
           - emqx-enterprise
-        # NOTE: for docker, only support latest otp version, not a matrix
+        # NOTE: for docker, only support latest otp and elixir
+        # version, not a matrix
         otp:
           - 24.1.5-3 # update to latest
+        elixir:
+          - 1.13.1 # update to latest
+        arch:
+          - amd64
+          - arm64
 
     steps:
     - uses: actions/download-artifact@v2
@@ -423,7 +449,8 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
-          BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-alpine3.14
+          # FIXME: use tagged version once merged
+          BUILD_FROM=ghcr.io/emqx/emqx-builder/elixir:${{ matrix.otp }}-${{ matrix.elixir }}-alpine3.14
           RUN_FROM=alpine:3.14
           EMQX_NAME=${{ matrix.profile }}
         file: source/deploy/docker/Dockerfile

Makefile

@@ -6,6 +6,7 @@ SCRIPTS = $(CURDIR)/scripts
 export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14
 export EMQX_DEFAULT_RUNNER = alpine:3.14
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
+export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
 export EMQX_DASHBOARD_VERSION ?= v0.18.0
 export DOCKERFILE := deploy/docker/Dockerfile
 export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing
@@ -36,6 +37,22 @@ ensure-rebar3:
 	@$(SCRIPTS)/fail-on-old-otp-version.escript
 	@$(SCRIPTS)/ensure-rebar3.sh $(REBAR_VERSION)
 
+.PHONY: ensure-hex
+ensure-hex:
+	@mix local.hex --if-missing --force
+
+.PHONY: ensure-mix-rebar3
+ensure-mix-rebar3: $(REBAR)
+	@mix local.rebar rebar3 $(CURDIR)/rebar3 --if-missing --force
+
+.PHONY: ensure-mix-rebar
+ensure-mix-rebar: $(REBAR)
+	@mix local.rebar --if-missing --force
+
+.PHONY: mix-deps-get
+mix-deps-get: $(ELIXIR_COMMON_DEPS)
+	@mix deps.get
+
 $(REBAR): ensure-rebar3
 
 .PHONY: get-dashboard
@@ -95,7 +112,7 @@ coveralls: $(REBAR)
 	@ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send
 
 .PHONY: $(REL_PROFILES)
-$(REL_PROFILES:%=%): $(REBAR) get-dashboard conf-segs
+$(REL_PROFILES:%=%): $(COMMON_DEPS)
 	@$(REBAR) as $(@) do release
 
 ## Not calling rebar3 clean because
@@ -139,6 +156,7 @@ dialyzer: $(REBAR)
 	@$(REBAR) as check dialyzer
 
 COMMON_DEPS := $(REBAR) get-dashboard conf-segs
+ELIXIR_COMMON_DEPS := ensure-hex ensure-mix-rebar3 ensure-mix-rebar
 
 ## rel target is to create release package without relup
 .PHONY: $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel)
@@ -201,3 +219,18 @@ $(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target-testing,$(zt))))
 
 conf-segs:
 	@scripts/merge-config.escript
+
+## elixir target is to create release packages using Elixir's Mix
+.PHONY: $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir)
+$(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir): $(COMMON_DEPS) $(ELIXIR_COMMON_DEPS) mix-deps-get
+	@$(BUILD) $(subst -elixir,,$(@)) elixir
+
+.PHONY: $(REL_PROFILES:%=%-elixirpkg)
+define gen-elixirpkg-target
+# the Elixir places the tar in a different path than Rebar3
+$1-elixirpkg: $1-pkg-elixir
+	@env TAR_PKG_DIR=_build/prod \
+	     IS_ELIXIR=yes \
+	     $(BUILD) $1 pkg
+endef
+$(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixirpkg-target,$(pt))))

build

@@ -6,6 +6,11 @@
 
 set -euo pipefail
 
+DEBUG="${DEBUG:-0}"
+if [ "$DEBUG" -eq 1 ]; then
+    set -x
+fi
+
 PROFILE="$1"
 ARTIFACT="$2"
 
@@ -198,6 +203,53 @@ make_docker_testing() {
        -f "${DOCKERFILE_TESTING}" .
 }
 
+# used to control the Elixir Mix Release output
+# see docstring in `mix.exs`
+export_release_vars() {
+  local profile="$1"
+  case "$profile" in
+    emqx)
+      export EMQX_RLEASE_TYPE=cloud \
+             EMQX_PACKAGE_TYPE=bin \
+             EMQX_EDITION_TYPE=community \
+             ELIXIR_MAKE_TAR=no
+      ;;
+    emqx-edge)
+      export EMQX_RLEASE_TYPE=edge \
+             EMQX_PACKAGE_TYPE=bin \
+             EMQX_EDITION_TYPE=community \
+             ELIXIR_MAKE_TAR=no
+      ;;
+    emqx-enterprise)
+      export EMQX_RLEASE_TYPE=cloud \
+             EMQX_PACKAGE_TYPE=bin \
+             EMQX_EDITION_TYPE=enterprise \
+             ELIXIR_MAKE_TAR=no
+      ;;
+    emqx-pkg)
+      export EMQX_RLEASE_TYPE=cloud \
+             EMQX_PACKAGE_TYPE=pkg \
+             EMQX_EDITION_TYPE=community \
+             ELIXIR_MAKE_TAR=yes
+      ;;
+    emqx-edge-pkg)
+      export EMQX_RLEASE_TYPE=edge \
+             EMQX_PACKAGE_TYPE=pkg \
+             EMQX_EDITION_TYPE=community \
+             ELIXIR_MAKE_TAR=yes
+      ;;
+    emqx-enterprise-pkg)
+      export EMQX_RLEASE_TYPE=cloud \
+             EMQX_PACKAGE_TYPE=pkg \
+             EMQX_EDITION_TYPE=enterprise \
+             ELIXIR_MAKE_TAR=yes
+      ;;
+    *)
+      echo Invalid profile "$profile"
+      exit 1
+  esac
+}
+
 log "building artifact=$ARTIFACT for profile=$PROFILE"
 
 case "$ARTIFACT" in
@@ -219,7 +271,10 @@ case "$ARTIFACT" in
             exit 0
         fi
         make -C "deploy/packages/${PKGERDIR}" clean
-        EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}"
+        env EMQX_REL="$(pwd)" \
+            EMQX_BUILD="${PROFILE}" \
+            SYSTEM="${SYSTEM}" \
+            make -C "deploy/packages/${PKGERDIR}"
         ;;
     docker)
         make_docker
@@ -227,6 +282,10 @@ case "$ARTIFACT" in
     docker-testing)
         make_docker_testing
         ;;
+    elixir)
+        export_release_vars "$PROFILE"
+        env MIX_ENV=prod mix release --overwrite
+        ;;
     *)
         log "Unknown artifact $ARTIFACT"
         exit 1

deploy/packages/deb/Makefile

@@ -6,9 +6,16 @@ BUILT := $(SRCDIR)/BUILT
 
 EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
 
-TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz
+ifeq ($(IS_ELIXIR), yes)
+  ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN)
+else
+  ELIXIR_PKG_VSN :=
+endif
+
+TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx
+TAR_PKG     := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz
 SOURCE_PKG  := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture)
-TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH)
+TARGET_PKG  := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)$(ELIXIR_PKG_VSN)-$(SYSTEM)-$(ARCH)
 
 .PHONY: all
 all: | $(BUILT)

deploy/packages/rpm/Makefile

@@ -16,9 +16,16 @@ endif
 
 EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
 
-TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz
+ifeq ($(IS_ELIXIR), yes)
-TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH)
+  ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN)
+else
+  ELIXIR_PKG_VSN :=
+endif
+
+TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx
+TAR_PKG     := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz
 SOURCE_PKG  := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m)
+TARGET_PKG  := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)$(ELIXIR_PKG_VSN)-$(SYSTEM)-$(ARCH)
 
 SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi)
 # Not $(PWD) as it does not work for make -C
@@ -55,4 +62,3 @@ $(BUILT):
 
 clean:
 	rm -rf $(SRCDIR)
-

mix.exs

@@ -124,7 +124,7 @@ defmodule EMQXUmbrella.MixProject do
 
         steps =
           if System.get_env("ELIXIR_MAKE_TAR") == "yes" do
-            base_steps ++ [:tar]
+            base_steps ++ [&prepare_tar_overlays/1, :tar]
           else
             base_steps
           end
@@ -234,6 +234,10 @@ defmodule EMQXUmbrella.MixProject do
     }
   end
 
+  #############################################################################
+  #  Custom Steps
+  #############################################################################
+
   defp copy_files(release, release_type, package_type, edition_type) do
     overwrite? = Keyword.get(release.options, :overwrite, false)
 
@@ -409,6 +413,26 @@ defmodule EMQXUmbrella.MixProject do
     release
   end
 
+  # The `:tar` built-in step in Mix Release does not currently add the
+  # `etc` directory into the resulting tarball.  The workaround is to
+  # add those to the `:overlays` key before running `:tar`.
+  # See: https://hexdocs.pm/mix/1.13.1/Mix.Release.html#__struct__/0
+  defp prepare_tar_overlays(release) do
+    Enum.each(
+      ["mnesia", "configs", "patches", "scripts"],
+      fn dir ->
+        path = Path.join([release.path, "data", dir])
+        File.mkdir_p!(path)
+      end
+    )
+
+    Map.update!(release, :overlays, &["etc", "data" | &1])
+  end
+
+  #############################################################################
+  #  Helper functions
+  #############################################################################
+
   defp template_vars(release, release_type, :bin = _package_type, edition_type) do
     [
       platform_bin_dir: "bin",
@@ -454,7 +478,7 @@ defmodule EMQXUmbrella.MixProject do
       # FIXME: this is empty in `make emqx` ???
       erl_opts: "",
       emqx_description: emqx_description(release_type, edition_type),
-      built_on: built_on(),
+      built_on_arch: built_on(),
       is_elixir: "yes"
     ]
   end

scripts/buildx.sh

@@ -11,17 +11,21 @@
 ## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10
 
 set -euo pipefail
+set -x
 
 help() {
     echo
     echo "-h|--help:                   To display this usage information"
     echo "--profile <PROFILE>:         EMQ X profile to build, e.g. emqx, emqx-edge"
-    echo "--pkgtype tgz|pkg:   Specify which package to build, tgz for .tar.gz"
+    echo "--pkgtype tgz|pkg|elixirpkg: Specify which package to build, tgz for .tar.gz,"
-    echo "                     and pkg for .rpm or .deb"
+    echo "                             pkg and elixirpkg for .rpm or .deb"
     echo "--arch amd64|arm64:          Target arch to build the EMQ X package for"
     echo "--src_dir <SRC_DIR>:         EMQ X source ode in this dir, default to PWD"
     echo "--builder <BUILDER>:         Builder image to pull"
     echo "                             E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10"
+    echo "--otp <OTP_VSN>:             OTP version being used in the builder"
+    echo "--elixir <ELIXIR_VSN>:       Elixir version being used in the builder"
+    echo "--system <SYSTEM>:           OS used in the builder image"
 }
 
 while [ "$#" -gt 0 ]; do
@@ -50,6 +54,18 @@ while [ "$#" -gt 0 ]; do
         ARCH="$2"
         shift 2
         ;;
+    --otp)
+        OTP_VSN="$2"
+        shift 2
+        ;;
+    --elixir)
+        ELIXIR_VSN="$2"
+        shift 2
+        ;;
+    --system)
+        SYSTEM="$2"
+        shift 2
+        ;;
     *)
       echo "WARN: Unknown arg (ignored): $1"
       shift
@@ -58,21 +74,37 @@ while [ "$#" -gt 0 ]; do
   esac
 done
 
-if [ -z "${PROFILE:-}" ] || [ -z "${PKGTYPE:-}" ] || [ -z "${BUILDER:-}" ] || [ -z "${ARCH:-}" ]; then
+if [ -z "${PROFILE:-}" ]    ||
+   [ -z "${PKGTYPE:-}" ]    ||
+   [ -z "${BUILDER:-}" ]    ||
+   [ -z "${ARCH:-}" ]       ||
+   [ -z "${OTP_VSN:-}" ]    ||
+   [ -z "${ELIXIR_VSN:-}" ] ||
+   [ -z "${SYSTEM:-}" ]; then
     help
     exit 1
 fi
 
-if [ "$PKGTYPE" != 'tgz' ] && [ "$PKGTYPE" != 'pkg' ]; then
+case "$PKGTYPE" in
-    echo "Bad --pkgtype option, should be tgz or pkg"
+  tgz|pkg|elixirpkg)
+    true
+    ;;
+  *)
+    echo "Bad --pkgtype option, should be zip or pkg"
     exit 1
-fi
+    ;;
+esac
 
 cd "${SRC_DIR:-.}"
 
 PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}"
-OTP_VSN_SYSTEM=$(echo "$BUILDER" | cut -d ':' -f2)
+
-PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN_SYSTEM}-${ARCH}"
+if [ "$PKGTYPE" = "elixirpkg" ]
+then
+  PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-elixir${ELIXIR_VSN}-${SYSTEM}-${ARCH}"
+else
+  PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
+fi
 
 docker info
 docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}"

scripts/get-elixir-vsn.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+elixir -e "System.version() |> IO.puts()"