From d863609f4382f7eb9667372a6eed7c8db06d43b7 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 9 Feb 2022 14:04:19 +0100 Subject: [PATCH 1/3] build: use rockylinux and 'rhel' for package names --- .github/workflows/build_packages.yaml | 17 +++++++++-------- build | 6 +++--- scripts/buildx.sh | 6 ++++-- scripts/get-distro.sh | 6 +++--- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index c5b734f56..cf5f5b902 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -240,17 +240,12 @@ jobs: - ubuntu16.04 - debian10 - debian9 - # - opensuse - - centos8 + - rockylinux8 - centos7 - - centos6 - raspbian10 - # - raspbian9 exclude: - package: pkg otp: 23.3.4.9-3 - - os: centos6 - arch: arm64 - os: raspbian9 arch: amd64 - os: raspbian10 @@ -298,12 +293,18 @@ jobs: if [ ! -z "$(echo $SYSTEM | grep -oE 'raspbian')" ]; then export ARCH="arm" fi - + if [ "$SYSTEM" = 'centos7' ]; then + DISTRO='rhel7' + elif [ "$SYSTEM" = 'rockylinux8' ]; then + DISTRO='rhel8' + else + DISTRO=${SYSTEM} + fi mkdir -p _upgrade_base cd _upgrade_base old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for tag in ${old_vsns[@]}; do - package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" + package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${DISTRO}-${ARCH}" if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip) | grep -oE "^[23]+")" ]; then wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip.sha256 diff --git a/build b/build index 7b5e6645e..3d5d3b265 100755 --- a/build +++ b/build @@ -150,7 +150,7 @@ make_docker() { ## Name Default Example ## --------------------------------------------------------------------- ## EMQX_BASE_IMAGE current os centos:7 -## EMQX_ZIP_PACKAGE _packages/ /tmp/emqx-4.4.0-otp24.1.5-3-centos7-amd64.zip +## EMQX_ZIP_PACKAGE _packages/ /tmp/emqx-4.4.0-otp24.1.5-3-rhel7-amd64.zip ## EMQX_IMAGE_TAG emqx/emqx: emqx/emqx:testing-tag ## make_docker_testing() { @@ -159,8 +159,8 @@ make_docker_testing() { ubuntu20*) EMQX_BASE_IMAGE="ubuntu:20.04" ;; - centos8) - EMQX_BASE_IMAGE="centos:8" + rhel8) + EMQX_BASE_IMAGE="rockylinux:8" ;; *) echo "Unsupported testing base image for $SYSTEM" diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 6e3ef8160..e9ebcf6e1 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -69,9 +69,11 @@ fi cd "${SRC_DIR:-.}" +set -x PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" -OTP_VSN_SYSTEM=$(echo "$BUILDER" | cut -d ':' -f2) -PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN_SYSTEM}-${ARCH}" +OTP_VSN="$(docker run -v $(pwd):/src --rm "$BUILDER" bash /src/scripts/get-otp-vsn.sh)" +DISTRO="$(docker run -v $(pwd):/src --rm "$BUILDER" bash /src/scripts/get-distro.sh)" +PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${DISTRO}-${ARCH}" docker info docker run --rm --privileged tonistiigi/binfmt:latest --install ${ARCH} diff --git a/scripts/get-distro.sh b/scripts/get-distro.sh index 89eafc4ee..c28698af1 100755 --- a/scripts/get-distro.sh +++ b/scripts/get-distro.sh @@ -10,9 +10,9 @@ if [ "$(uname -s)" = 'Darwin' ]; then VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g') SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')" elif [ "$(uname -s)" = 'Linux' ]; then - if grep -q -i 'centos' /etc/*-release; then - DIST='centos' - VERSION_ID="$(rpm --eval '%{centos_ver}')" + if grep -q -i 'rhel' /etc/*-release; then + DIST='rhel' + VERSION_ID="$(rpm --eval '%{rhel}')" else DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')" From 4f6876b6e91b5a27e58b7c2a4d93b5c2ec834f8d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 9 Feb 2022 22:16:39 +0100 Subject: [PATCH 2/3] ci: package slim-build on rockylinux --- .github/workflows/build_slim_packages.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index df02bc625..b7b948fd6 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -25,7 +25,7 @@ jobs: - 24.1.5-3 os: - ubuntu20.04 - - centos7 + - rockylinux8 container: ghcr.io/emqx/emqx-builder/4.4-4:${{ matrix.erl_otp }}-${{ matrix.os }} @@ -53,7 +53,8 @@ jobs: - name: packages test run: | PKG_VSN="$(./pkg-vsn.sh)" - PKG_NAME="${EMQX_NAME}-${PKG_VSN}-otp${{ matrix.erl_otp }}-${{ matrix.os }}-amd64" + DISTRO="$(./scripts/get-distro.sh)" + PKG_NAME="${EMQX_NAME}-${PKG_VSN}-otp${{ matrix.erl_otp }}-${DISTRO}-amd64" export CODE_PATH="$GITHUB_WORKSPACE" .ci/build_packages/tests.sh "$PKG_NAME" zip .ci/build_packages/tests.sh "$PKG_NAME" pkg From fd75756d5ce51e00978b856ae4fa5bba8e574e54 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 9 Feb 2022 22:49:08 +0100 Subject: [PATCH 3/3] ci: update to emqx-builder 4.4-5 (rockylinux) --- .ci/build_packages/Dockerfile | 2 +- .ci/docker-compose-file/docker-compose.yaml | 2 +- .github/workflows/build_packages.yaml | 8 ++++---- .github/workflows/build_slim_packages.yaml | 2 +- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/run_acl_migration_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 4 ++-- .github/workflows/run_test_cases.yaml | 2 +- Makefile | 2 +- deploy/docker/Dockerfile | 2 +- scripts/buildx.sh | 4 ++-- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile index b1edd8409..3f842dfb8 100644 --- a/.ci/build_packages/Dockerfile +++ b/.ci/build_packages/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 FROM ${BUILD_FROM} ARG EMQX_NAME=emqx diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index bd4f5f391..396f5081d 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 env_file: - conf.env environment: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index cf5f5b902..503f0a15b 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -16,7 +16,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles }} @@ -324,7 +324,7 @@ jobs: --profile "${PROFILE}" \ --pkgtype "${PACKAGE}" \ --arch "${ARCH}" \ - --builder "ghcr.io/emqx/emqx-builder/4.4-4:${OTP}-${SYSTEM}" + --builder "ghcr.io/emqx/emqx-builder/4.4-5:${OTP}-${SYSTEM}" - name: create sha256 working-directory: source env: @@ -393,7 +393,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-4:${{ matrix.otp }}-alpine3.14 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-5:${{ matrix.otp }}-alpine3.14 RUN_FROM=alpine:3.14 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile @@ -408,7 +408,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-4:${{ matrix.otp }}-alpine3.14 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-5:${{ matrix.otp }}-alpine3.14 RUN_FROM=alpine:3.14 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile.enterprise diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index b7b948fd6..07b18a10a 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -27,7 +27,7 @@ jobs: - ubuntu20.04 - rockylinux8 - container: ghcr.io/emqx/emqx-builder/4.4-4:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-5:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index d09270e65..98c2e8327 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml index 1ecbdead1..68a06f11b 100644 --- a/.github/workflows/run_acl_migration_tests.yaml +++ b/.github/workflows/run_acl_migration_tests.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 strategy: fail-fast: true env: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index ca4462467..1229d8414 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -226,7 +226,7 @@ jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 outputs: profile: ${{ steps.profile-and-versions.outputs.profile }} vsn: ${{ steps.profile-and-versions.outputs.vsn }} @@ -277,7 +277,7 @@ jobs: otp: - 24.1.5-3 runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 defaults: run: shell: bash diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index c32c13531..16f35be6b 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -10,7 +10,7 @@ on: jobs: run_proper_test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/Makefile b/Makefile index 9df141302..9bb3c2db3 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ $(shell $(CURDIR)/scripts/git-hooks-init.sh) REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts -export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-alpine3.14 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-alpine3.14 export EMQX_DEFAULT_RUNNER = alpine:3.14 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index cb8d83309..9456706cd 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-alpine3.14 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-alpine3.14 ARG RUN_FROM=alpine:3.14 FROM ${BUILD_FROM} AS builder diff --git a/scripts/buildx.sh b/scripts/buildx.sh index e9ebcf6e1..9e803d377 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -8,7 +8,7 @@ ## i.e. will not work if docker command has to be executed with sudo ## example: -## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10 --arch arm64 +## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-debian10 --arch arm64 set -euo pipefail @@ -20,7 +20,7 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQ X package for" echo "--src_dir : EMQ X source ode in this dir, default to PWD" echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10" + echo " E.g. ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-debian10" } while [ "$#" -gt 0 ]; do