build: make use of rebar git cache in docker build

.gitignore

@@ -68,3 +68,4 @@ lux_logs/
 bom.json
 ct_run*/
 apps/emqx_conf/etc/emqx.conf.all.rendered*
+rebar-git-cache.tar

build

@@ -6,7 +6,11 @@
 
 set -euo pipefail
 
-[ "${DEBUG:-0}" -eq 1 ] && set -x
+if [ "${DEBUG:-0}" -eq 1 ]; then
+    set -x
+    # set this for rebar3
+    export DIAGNOSTIC=1
+fi
 
 PROFILE_ARG="$1"
 ARTIFACT="$2"
@@ -449,17 +453,33 @@ make_docker() {
     if [ "${DOCKER_PUSH:-false}" = true ]; then
         DOCKER_BUILDX_ARGS+=(--push)
     fi
+    if [ -d "${REBAR_GIT_CACHE_DIR:-}" ]; then
+        cache_tar="$(pwd)/rebar-git-cache.tar"
+        if [ ! -f "${cache_tar}" ]; then
+            pushd "${REBAR_GIT_CACHE_DIR}" >/dev/null
+            tar -cf "${cache_tar}" .
+            popd >/dev/null
+        fi
+    fi
+    if [ -n "${DEBUG:-}" ]; then
+        DOCKER_BUILDX_ARGS+=(--build-arg DEBUG="${DEBUG}" --progress=plain)
+    fi
+
     # shellcheck disable=SC2015
     [ -f ./.dockerignore ] && mv ./.dockerignore ./.dockerignore.bak || true
     trap docker_cleanup EXIT
     {
-        echo '/_build'
+        echo '_build/'
-        echo '/deps'
+        echo 'deps/'
-        echo '/*.lock'
+        echo '*.lock'
+        echo '_packages/'
+        echo '.vs/'
+        echo '.vscode/'
+        echo 'lux_logs/'
+        echo '_upgrade_base/'
     } >> ./.dockerignore
-    set -x
+    echo "Docker build args: ${DOCKER_BUILDX_ARGS[*]}"
     docker buildx build "${DOCKER_BUILDX_ARGS[@]}" .
-    [[ "${DEBUG:-}" -eq 1 ]] || set +x
     echo "${EMQX_IMAGE_TAG}" > ./.docker_image_tag
 }
 

deploy/docker/Dockerfile

@@ -1,23 +1,35 @@
 ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-debian11
 ARG RUN_FROM=public.ecr.aws/debian/debian:11-slim
 FROM ${BUILD_FROM} AS builder
+ARG DEBUG=0
 
 COPY . /emqx
 
 ARG EMQX_NAME=emqx
 ARG PKG_VSN
-ENV EMQX_RELUP=false
 
-RUN export PROFILE=${EMQX_NAME%%-elixir} \
+ENV EMQX_RELUP=false
-    && export EMQX_NAME1=$EMQX_NAME \
+ENV DEBUG=${DEBUG}
-    && export EMQX_NAME=$PROFILE \
+ENV EMQX_REL_FORM='docker'
-    && export EMQX_REL_PATH="/emqx/_build/$EMQX_NAME/rel/emqx" \
+
-    && export EMQX_REL_FORM='docker' \
+WORKDIR /emqx/
-    && cd /emqx \
+
-    && make $EMQX_NAME1 \
+RUN git config --global --add safe.directory '*'
-    && rm -f $EMQX_REL_PATH/*.tar.gz \
+
+RUN if [ -f rebar-git-cache.tar ]; then \
+        mkdir .cache && \
+        tar -xf rebar-git-cache.tar -C .cache && \
+        export REBAR_GIT_CACHE_DIR='/emqx/.cache' && \
+        export REBAR_GIT_CACHE_REF_AUTOFILL=0 ;\
+    fi \
+    && export PROFILE=${EMQX_NAME%%-elixir} \
+    && export EMQX_NAME1="${EMQX_NAME}" \
+    && export EMQX_NAME=${PROFILE} \
+    && export EMQX_REL_PATH="/emqx/_build/${EMQX_NAME}/rel/emqx" \
+    && make ${EMQX_NAME1} \
+    && rm -f ${EMQX_REL_PATH}/*.tar.gz \
     && mkdir -p /emqx-rel \
-    && mv $EMQX_REL_PATH /emqx-rel
+    && mv ${EMQX_REL_PATH} /emqx-rel
 
 FROM $RUN_FROM
 ARG EXTRA_DEPS=''

scripts/ensure-rebar3.sh

@@ -2,6 +2,8 @@
 
 set -euo pipefail
 
+[ "${DEBUG:-0}" -eq 1 ] && set -x
+
 ## rebar3 tag 3.19.0-emqx-1 is compiled using latest official OTP-24 image.
 ## we have to use an otp24-compiled rebar3 because the defination of record #application{}
 ## in systools.hrl is changed in otp24.
@@ -14,7 +16,7 @@ case ${OTP_VSN} in
         VERSION="3.18.0-emqx-1"
         ;;
     25*)
-        VERSION="3.19.0-emqx-8"
+        VERSION="3.19.0-emqx-9"
         ;;
     *)
         echo "Unsupporetd Erlang/OTP version $OTP_VSN"

scripts/pre-compile.sh

@@ -2,6 +2,8 @@
 
 set -euo pipefail
 
+[ "${DEBUG:-0}" -eq 1 ] && set -x
+
 # NOTE: PROFILE_STR may not be exactly PROFILE (emqx or emqx-enterprise)
 # it might be with suffix such as -pkg etc.
 PROFILE_STR="${1}"
@@ -28,5 +30,6 @@ curl -L --silent --show-error \
      --output "apps/emqx_dashboard/priv/desc.zh.hocon" \
     'https://raw.githubusercontent.com/emqx/emqx-i18n/main/desc.zh.hocon'
 
-# generate sbom
+# TODO
-./scripts/update-bom.sh "$PROFILE_STR" ./rel
+# make sbom a build artifcat
+# ./scripts/update-bom.sh "$PROFILE_STR" ./rel

scripts/update-bom.sh

@@ -8,4 +8,3 @@ PROFILE="$1"
 REL_DIR="$2"
 
 ./rebar3 as "$PROFILE" sbom -f -o "$REL_DIR/bom.json"
-