From 30a02c74dc208238d513e8dd39edde8af9545cf2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 14 Sep 2022 20:17:47 +0200 Subject: [PATCH 1/3] build: allow passing ssh agent to docker run in buildx.sh --- scripts/buildx.sh | 48 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 98386efa0..5f4599765 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -20,10 +20,13 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQ X package for" echo "--src_dir : EMQ X source ode in this dir, default to PWD" echo "--builder : Builder image to pull" + echo " E.g. ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-debian11" echo "--system : The target OS system the package is being built for, ex: debian11" - echo " E.g. ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-debian10" + echo "--ssh: Pass ssh agent to the builder." + echo " Also configures git in container to use ssh instead of https to clone deps" } +USE_SSH='no' while [ "$#" -gt 0 ]; do case $1 in -h|--help) @@ -54,6 +57,10 @@ while [ "$#" -gt 0 ]; do SYSTEM="$2" shift 2 ;; + --ssh) + USE_SSH='yes' + shift + ;; *) echo "WARN: Unknown arg (ignored): $1" shift @@ -72,18 +79,45 @@ if [ "$PKGTYPE" != 'zip' ] && [ "$PKGTYPE" != 'pkg' ]; then exit 1 fi +## Although we have 'deterministic' set in 'erl_opts', and foced overriding at project level, +## still, some of the beams might be compiled (e.g. by erlang.mk) without this flag +## longer file path means larger beam files +## i.e. Keep the path to work dir short! +DOCKER_WORKDIR='/emqx' + cd "${SRC_DIR:-.}" -set -x -# $SYSTEM below is used by the `relup-base-vsns.escript` to correctly -# output the list of relup base versions. +cat <.gitconfig.tmp +[core] + sshCommand = ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no +[safe] + directory = $DOCKER_WORKDIR +EOF + +if [ "$USE_SSH" = 'yes' ]; then + cat <>.gitconfig.tmp +[url "ssh://git@github.com/"] + insteadOf = https://github.com/ +EOF + # when passing ssh agent, we assume this command is executed locally not in ci, so add '-t' option + SSH_AGENT_OPTION="-t -e SSH_AUTH_SOCK=/ssh-agent -v ${SSH_AUTH_SOCK}:/ssh-agent" +else + SSH_AGENT_OPTION='' +fi + docker info docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}" + +# $SYSTEM below is used by the `relup-base-vsns.escript` to correctly +# output the list of relup base versions. +# shellcheck disable=SC2086 docker run -i --rm \ - -v "$(pwd)":/emqx \ - --workdir /emqx \ + -v "$(pwd)":$DOCKER_WORKDIR \ + -v "$(pwd)/.gitconfig.tmp":/root/.gitconfig \ + --workdir $DOCKER_WORKDIR \ --platform="linux/$ARCH" \ --user root \ -e SYSTEM="$SYSTEM" \ + $SSH_AGENT_OPTION \ "$BUILDER" \ - bash -euc "git config --global --add safe.directory /emqx && chown -R root:root _build && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" + bash -euc "mkdir -p _build && chown -R root:root _build && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" From 20ddd4557f7f767da0484be63fd5e4b2de985b64 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 14 Sep 2022 20:43:29 +0200 Subject: [PATCH 2/3] refactor: no need for the --system flag for buildx.sh --- .github/workflows/build_packages.yaml | 1 - scripts/buildx.sh | 8 -------- scripts/relup-base-vsns.escript | 10 +++++++++- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index b82fe03c5..9392735b5 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -262,7 +262,6 @@ jobs: --profile "${PROFILE}" \ --pkgtype "${PACKAGE}" \ --arch "${ARCH}" \ - --system "${SYSTEM}" \ --builder "ghcr.io/emqx/emqx-builder/4.4-19:${OTP}-${SYSTEM}" - uses: actions/upload-artifact@v1 with: diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 5f4599765..87e548e7e 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -21,7 +21,6 @@ help() { echo "--src_dir : EMQ X source ode in this dir, default to PWD" echo "--builder : Builder image to pull" echo " E.g. ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-debian11" - echo "--system : The target OS system the package is being built for, ex: debian11" echo "--ssh: Pass ssh agent to the builder." echo " Also configures git in container to use ssh instead of https to clone deps" } @@ -53,10 +52,6 @@ while [ "$#" -gt 0 ]; do ARCH="$2" shift 2 ;; - --system) - SYSTEM="$2" - shift 2 - ;; --ssh) USE_SSH='yes' shift @@ -108,8 +103,6 @@ fi docker info docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}" -# $SYSTEM below is used by the `relup-base-vsns.escript` to correctly -# output the list of relup base versions. # shellcheck disable=SC2086 docker run -i --rm \ -v "$(pwd)":$DOCKER_WORKDIR \ @@ -117,7 +110,6 @@ docker run -i --rm \ --workdir $DOCKER_WORKDIR \ --platform="linux/$ARCH" \ --user root \ - -e SYSTEM="$SYSTEM" \ $SSH_AGENT_OPTION \ "$BUILDER" \ bash -euc "mkdir -p _build && chown -R root:root _build && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" diff --git a/scripts/relup-base-vsns.escript b/scripts/relup-base-vsns.escript index 6c618ca45..157f604cb 100755 --- a/scripts/relup-base-vsns.escript +++ b/scripts/relup-base-vsns.escript @@ -160,7 +160,7 @@ fetch_version(Vsn, VsnMap) -> filter_froms(Froms0, AvailableVersionsIndex) -> Froms1 = - case os:getenv("SYSTEM") of + case get_system() of %% we do not support relup for windows "windows" -> []; @@ -178,6 +178,14 @@ filter_froms(Froms0, AvailableVersionsIndex) -> fun(V) -> maps:get(V, AvailableVersionsIndex, false) end, Froms1). +get_system() -> + case os:getenv("SYSTEM") of + false -> + string:trim(os:cmd("./scripts/get-distro.sh")); + System -> + System + end. + %% assumes that's X.Y.Z, without pre-releases parse_vsn(VsnBin) -> {match, [Major0, Minor0, Patch0]} = re:run(VsnBin, "([0-9]+)\\.([0-9]+)\\.([0-9]+)", From bceafe509442b05dd757aac2f28a8410122eb5b0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 14 Sep 2022 20:49:21 +0200 Subject: [PATCH 3/3] chore: gitignore .gitconfig.tmp --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 13217dc00..c3f1ea6d1 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,4 @@ erlang_ls.config # For direnv .envrc mix.lock +.gitconfig.tmp