fix(docker): use 'make $PROFILE-docker' in CI

This commit is contained in:
Ivan Dyachkov 2023-08-09 14:05:36 +02:00
parent c35ad653f3
commit 8601f8f83b
5 changed files with 113 additions and 124 deletions

View File

@ -23,7 +23,6 @@ jobs:
container: 'ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04'
outputs:
profile: ${{ steps.parse-git-ref.outputs.profile }}
edition: ${{ steps.parse-git-ref.outputs.edition }}
release: ${{ steps.parse-git-ref.outputs.release }}
latest: ${{ steps.parse-git-ref.outputs.latest }}
version: ${{ steps.parse-git-ref.outputs.version }}
@ -50,12 +49,10 @@ jobs:
run: |
JSON="$(./scripts/parse-git-ref.sh $GITHUB_REF)"
PROFILE=$(echo "$JSON" | jq -cr '.profile')
EDITION=$(echo "$JSON" | jq -cr '.edition')
RELEASE=$(echo "$JSON" | jq -cr '.release')
LATEST=$(echo "$JSON" | jq -cr '.latest')
VERSION="$(./pkg-vsn.sh "$PROFILE")"
echo "profile=$PROFILE" | tee -a $GITHUB_OUTPUT
echo "edition=$EDITION" | tee -a $GITHUB_OUTPUT
echo "release=$RELEASE" | tee -a $GITHUB_OUTPUT
echo "latest=$LATEST" | tee -a $GITHUB_OUTPUT
echo "version=$VERSION" | tee -a $GITHUB_OUTPUT
@ -107,8 +104,8 @@ jobs:
publish: ${{ needs.prepare.outputs.release }}
otp_vsn: ${{ needs.prepare.outputs.otp_vsn }}
elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }}
runner: ${{ needs.prepare.outputs.runner }}
builder_vsn: ${{ needs.prepare.outputs.builder_vsn }}
runner: ${{ needs.prepare.outputs.runner }}
secrets: inherit
build_and_push_docker_images:
@ -118,15 +115,13 @@ jobs:
uses: ./.github/workflows/build_and_push_docker_images.yaml
with:
profile: ${{ needs.prepare.outputs.profile }}
edition: ${{ needs.prepare.outputs.edition }}
version: ${{ needs.prepare.outputs.version }}
latest: ${{ needs.prepare.outputs.latest }}
publish: ${{ needs.prepare.outputs.release }}
latest: ${{ needs.prepare.outputs.latest }}
otp_vsn: ${{ needs.prepare.outputs.otp_vsn }}
elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }}
# workaround: self-hosted runners do not have access to org-level secrets?
runner: ubuntu-22.04
builder_vsn: ${{ needs.prepare.outputs.builder_vsn }}
runner: ${{ needs.prepare.outputs.runner }}
secrets: inherit
compile:

View File

@ -10,9 +10,6 @@ on:
profile:
required: true
type: string
edition:
required: true
type: string
version:
required: true
type: string
@ -28,10 +25,10 @@ on:
elixir_vsn:
required: true
type: string
runner:
builder_vsn:
required: true
type: string
builder_vsn:
runner:
required: true
type: string
secrets:
@ -47,17 +44,12 @@ on:
inputs:
ref:
required: false
version:
required: true
type: string
profile:
required: false
type: string
default: 'emqx'
edition:
required: false
type: string
default: 'Opensource'
version:
required: true
latest:
required: false
type: boolean
@ -74,14 +66,14 @@ on:
required: false
type: string
default: '1.14.5'
runner:
required: false
type: string
default: 'ubuntu-22.04'
builder_vsn:
required: false
type: string
default: '5.1-3'
runner:
required: false
type: string
default: 'ubuntu-22.04'
jobs:
docker:
@ -92,18 +84,12 @@ jobs:
matrix:
profile:
- ${{ inputs.profile }}
- ${{ inputs.profile }}-elixir
registry:
- 'docker.io'
- 'public.ecr.aws'
os:
- [debian11, "debian:11-slim", "deploy/docker/Dockerfile"]
builder:
- ${{ inputs.builder_vsn }}
otp:
- ${{ inputs.otp_vsn }}
elixir:
- 'no_elixir'
- ${{ inputs.elixir_vsn }}
exclude:
- profile: emqx-enterprise-elixir
steps:
- uses: actions/checkout@v3
@ -130,53 +116,19 @@ jobs:
password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
ecr: true
- name: prepare for docker/metadata-action
id: pre-meta
shell: bash
- name: Build docker image
env:
PROFILE: ${{ matrix.profile }}
DOCKER_REGISTRY: ${{ matrix.registry }}
DOCKER_ORG: ${{ github.repository_owner }}
DOCKER_LATEST: ${{ inputs.latest }}
DOCKER_PUSH: ${{ inputs.publish == 'true' || inputs.publish || github.repository_owner != 'emqx' }}
DOCKER_BUILD_NOCACHE: true
DOCKER_PLATFORMS: linux/amd64,linux/arm64
EMQX_RUNNER: 'debian:11-slim'
PKG_VSN: ${{ inputs.version }}
EMQX_BUILDER_VSN: ${{ inputs.builder_vsn }}
EMQX_OTP_VSN: ${{ inputs.otp_vsn }}
EMQX_ELIXIR_VSN: ${{ inputs.elixir_vsn }}
run: |
extra_labels=
img_suffix=
if [ "${{ matrix.elixir }}" != 'no_elixir' ]; then
img_suffix="-elixir"
extra_labels="org.opencontainers.image.elixir.version=${{ matrix.elixir }}"
fi
extra_deps=
if [[ "${{ matrix.profile }}" = *enterprise* ]]; then
extra_deps='libsasl2-2,libsasl2-modules-gssapi-mit'
fi
echo "img_suffix=$img_suffix" >> $GITHUB_OUTPUT
echo "extra_labels=$extra_labels" >> $GITHUB_OUTPUT
echo "extra_deps=$extra_deps" >> $GITHUB_OUTPUT
- uses: docker/metadata-action@v4
id: meta
with:
images: |
${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }}
flavor: |
latest=${{ matrix.elixir == 'no_elixir' }}
suffix=${{ steps.pre-meta.outputs.img_suffix }}
tags: |
type=semver,pattern={{major}}.{{minor}},value=${{ inputs.version }}
type=semver,pattern={{version}},value=${{ inputs.version }}
type=raw,value=${{ inputs.version }}
type=raw,value=latest,enable=${{ inputs.latest }}
labels: |
org.opencontainers.image.otp.version=${{ matrix.otp }}
org.opencontainers.image.edition=${{ inputs.edition }}
${{ steps.pre-meta.outputs.extra_labels }}
- uses: docker/build-push-action@v3
with:
push: ${{ inputs.publish == 'true' || inputs.publish || github.repository_owner != 'emqx' }}
pull: true
no-cache: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
EMQX_NAME=${{ matrix.profile }}${{ steps.pre-meta.outputs.img_suffix }}
EXTRA_DEPS=${{ steps.pre-meta.outputs.extra_deps }}
PKG_VSN=${{ inputs.version }}
file: ${{ matrix.os[2] }}
./build ${PROFILE} docker

85
build
View File

@ -368,21 +368,78 @@ docker_cleanup() {
}
## Build the default docker image based on debian 11.
## NOTE: docker image build in github action does not call this
## function, see build_and_push_docker_images.yaml
make_docker() {
EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}"
EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}"
EMQX_DOCKERFILE="${EMQX_DOCKERFILE:-deploy/docker/Dockerfile}"
local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.1-3}"
local EMQX_BUILDER_PLATFORM="${EMQX_BUILDER_PLATFORM:-debian11}"
local EMQX_BUILDER_OTP="${EMQX_BUILDER_OTP:-25.3.2-1}"
local EMQX_BUILDER_ELIXIR="${EMQX_BUILDER_ELIXIR:-1.14.5}"
local EMQX_BUILDER=${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VERSION}:${EMQX_BUILDER_ELIXIR}-${EMQX_BUILDER_OTP}-${EMQX_BUILDER_PLATFORM}}
local EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}"
local EMQX_DOCKERFILE="${EMQX_DOCKERFILE:-deploy/docker/Dockerfile}"
local PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
local VSN_MAJOR="$(echo "$PKG_VSN" | cut -d . -f 1)"
local VSN_MINOR="$(echo "$PKG_VSN" | cut -d . -f 2)"
local VSN_PATCH="$(echo "$PKG_VSN" | cut -d . -f 3)"
local SUFFIX=''
if [[ "$PROFILE" = *-elixir ]]; then
PKG_VSN="$PKG_VSN-elixir"
SUFFIX="-elixir"
fi
local default_tag="emqx/${PROFILE%%-elixir}:${PKG_VSN}"
EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-$default_tag}"
local DOCKER_REGISTRY="${REGISTRY:-docker.io}"
local DOCKER_ORG="${DOCKER_ORG:-emqx}"
local EMQX_BASE_DOCKER_TAG="${DOCKER_REGISTRY}/${DOCKER_ORG}/${PROFILE%%-elixir}"
local default_tag="${EMQX_BASE_DOCKER_TAG}:${PKG_VSN}${SUFFIX}"
local EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-$default_tag}"
local EDITION=Opensource
local LICENSE='Apache-2.0'
## extra_deps is a comma separated list of debian 11 package names
local extra_deps=''
local EXTRA_DEPS=''
if [[ "$PROFILE" = *enterprise* ]]; then
extra_deps='libsasl2-2,libsasl2-modules-gssapi-mit'
EXTRA_DEPS='libsasl2-2,libsasl2-modules-gssapi-mit'
EDITION=Enterprise
LICENSE='(Apache-2.0 AND BSL-1.1)'
fi
local ISO_8601_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
local GIT_REVISION="$(git rev-parse HEAD)"
local TAGS=()
local DOCKER_BUILDX_ARGS=(
--build-arg BUILD_FROM="${EMQX_BUILDER}" \
--build-arg RUN_FROM="${EMQX_RUNNER}" \
--build-arg EMQX_NAME="${PROFILE}" \
--build-arg EXTRA_DEPS="${EXTRA_DEPS}" \
--build-arg PKG_VSN="${PKG_VSN}" \
--file deploy/docker/Dockerfile \
--label org.opencontainers.image.title="${PROFILE}" \
--label org.opencontainers.image.edition="${EDITION}" \
--label org.opencontainers.image.version="${PKG_VSN}" \
--label org.opencontainers.image.revision="${GIT_REVISION}" \
--label org.opencontainers.image.created="${ISO_8601_DATE}" \
--label org.opencontainers.image.source='https://github.com/emqx/emqx' \
--label org.opencontainers.image.url='https://github.com/emqx/emqx' \
--label org.opencontainers.image.documentation='https://www.emqx.io/docs/en/latest/' \
--label org.opencontainers.image.description='The most scalable open-source MQTT broker for IoT, IIoT, and connected vehicles' \
--label org.opencontainers.image.licenses="${LICENSE}" \
--label org.opencontainers.image.otp.version="${EMQX_BUILDER_OTP}" \
--tag "${EMQX_IMAGE_TAG}" \
--tag "${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}${SUFFIX}" \
--tag "${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}.${VSN_PATCH}${SUFFIX}" \
--tag "${EMQX_BASE_DOCKER_TAG}:${PKG_VSN}${SUFFIX}" \
--provenance false \
--pull
)
if [ ${DOCKER_BUILD_NOCACHE:-false} = true ]; then
DOCKER_BUILDX_ARGS+=(--no-cache)
fi
if [ "${SUFFIX}" = '-elixir' ]; then
DOCKER_BUILDX_ARGS+=(--label org.opencontainers.image.elixir.version="${EMQX_BUILDER_ELIXIR}")
fi
if [ ${DOCKER_LATEST:-false} = true ]; then
DOCKER_BUILDX_ARGS+=(--tag "${DOCKER_REGISTRY}/${DOCKER_ORG}/${PROFILE}:latest")
fi
if [ "${DOCKER_PLATFORMS:-default}" != 'default' ]; then
DOCKER_BUILDX_ARGS+=(--platform "${DOCKER_PLATFORMS}")
fi
if [ ${DOCKER_PUSH-false} = true ]; then
DOCKER_BUILDX_ARGS+=(--push)
fi
# shellcheck disable=SC2015
[ -f ./.dockerignore ] && mv ./.dockerignore ./.dockerignore.bak || true
@ -393,13 +450,7 @@ make_docker() {
echo '/*.lock'
} >> ./.dockerignore
set -x
docker build --no-cache --pull \
--build-arg BUILD_FROM="${EMQX_BUILDER}" \
--build-arg RUN_FROM="${EMQX_RUNNER}" \
--build-arg EMQX_NAME="${PROFILE}" \
--build-arg EXTRA_DEPS="${extra_deps}" \
--tag "${EMQX_IMAGE_TAG}" \
-f "${EMQX_DOCKERFILE}" .
docker buildx build "${DOCKER_BUILDX_ARGS[@]}" .
[[ "${DEBUG:-}" -eq 1 ]] || set +x
echo "${EMQX_IMAGE_TAG}" > ./.docker_image_tag
}

View File

@ -18,32 +18,26 @@ is_latest() {
if [[ $1 =~ ^refs/tags/v[5-9]+\.[0-9]+\.[0-9]+$ ]]; then
PROFILE=emqx
EDITION=Opensource
RELEASE=true
LATEST=$(is_latest "$1")
elif [[ $1 =~ ^refs/tags/v[5-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
PROFILE=emqx
EDITION=Opensource
RELEASE=true
LATEST=$(is_latest "$1")
elif [[ $1 =~ ^refs/tags/e[5-9]+\.[0-9]+\.[0-9]+$ ]]; then
PROFILE=emqx-enterprise
EDITION=Enterprise
RELEASE=true
LATEST=$(is_latest "$1")
elif [[ $1 =~ ^refs/tags/e[5-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
PROFILE=emqx-enterprise
EDITION=Enterprise
RELEASE=true
LATEST=$(is_latest "$1")
elif [[ $1 =~ ^refs/tags/v[5-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$ ]]; then
PROFILE=emqx
EDITION=Opensource
RELEASE=true
LATEST=false
elif [[ $1 =~ ^refs/tags/e[5-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$ ]]; then
PROFILE=emqx-enterprise
EDITION=Enterprise
RELEASE=true
LATEST=false
elif [[ $1 =~ ^refs/tags/.+ ]]; then
@ -51,17 +45,14 @@ elif [[ $1 =~ ^refs/tags/.+ ]]; then
exit 1
elif [[ $1 =~ ^refs/heads/master$ ]]; then
PROFILE=emqx
EDITION=Opensource
RELEASE=false
LATEST=false
elif [[ $1 =~ ^refs/heads/release-[5-9][0-9]+$ ]]; then
PROFILE=emqx-enterprise
EDITION=Enterprise
RELEASE=false
LATEST=false
elif [[ $1 =~ ^refs/heads/ci/.* ]]; then
PROFILE=emqx
EDITION=Opensource
RELEASE=false
LATEST=false
else
@ -70,5 +61,5 @@ else
fi
cat <<EOF
{"profile": "$PROFILE", "edition": "$EDITION", "release": $RELEASE, "latest": $LATEST}
{"profile": "$PROFILE", "release": $RELEASE, "latest": $LATEST}
EOF

View File

@ -10,17 +10,17 @@ Unrecognized git ref: v5.2.0
./parse-git-ref.sh refs/tags/v5.1.0
>>>
{"profile": "emqx", "edition": "Opensource", "release": true, "latest": false}
{"profile": "emqx", "release": true, "latest": false}
>>>= 0
./parse-git-ref.sh refs/tags/v5.1.5.1
>>>
{"profile": "emqx", "edition": "Opensource", "release": true, "latest": false}
{"profile": "emqx", "release": true, "latest": false}
>>>= 0
./parse-git-ref.sh refs/tags/v5.2.0-alpha.1
>>>
{"profile": "emqx", "edition": "Opensource", "release": true, "latest": false}
{"profile": "emqx", "release": true, "latest": false}
>>>= 0
./parse-git-ref.sh refs/tags/v5.2.0-alpha-1
@ -30,62 +30,62 @@ Unrecognized tag: refs/tags/v5.2.0-alpha-1
./parse-git-ref.sh refs/tags/v5.2.0-beta.1
>>>
{"profile": "emqx", "edition": "Opensource", "release": true, "latest": false}
{"profile": "emqx", "release": true, "latest": false}
>>>= 0
./parse-git-ref.sh refs/tags/v5.2.0-rc.1
>>>
{"profile": "emqx", "edition": "Opensource", "release": true, "latest": false}
{"profile": "emqx", "release": true, "latest": false}
>>>= 0
./parse-git-ref.sh refs/tags/e5.1.0
>>>
{"profile": "emqx-enterprise", "edition": "Enterprise", "release": true, "latest": false}
{"profile": "emqx-enterprise", "release": true, "latest": false}
>>>= 0
./parse-git-ref.sh refs/tags/e5.1.5.1
>>>
{"profile": "emqx-enterprise", "edition": "Enterprise", "release": true, "latest": false}
{"profile": "emqx-enterprise", "release": true, "latest": false}
>>>= 0
./parse-git-ref.sh refs/tags/e5.2.0-alpha.1
>>>
{"profile": "emqx-enterprise", "edition": "Enterprise", "release": true, "latest": false}
{"profile": "emqx-enterprise", "release": true, "latest": false}
>>>= 0
./parse-git-ref.sh refs/tags/e5.2.0-beta.1
>>>
{"profile": "emqx-enterprise", "edition": "Enterprise", "release": true, "latest": false}
{"profile": "emqx-enterprise", "release": true, "latest": false}
>>>= 0
./parse-git-ref.sh refs/tags/e5.2.0-rc.1
>>>
{"profile": "emqx-enterprise", "edition": "Enterprise", "release": true, "latest": false}
{"profile": "emqx-enterprise", "release": true, "latest": false}
>>>= 0
./parse-git-ref.sh refs/tags/e5.1.99
>>>
{"profile": "emqx-enterprise", "edition": "Enterprise", "release": true, "latest": true}
{"profile": "emqx-enterprise", "release": true, "latest": true}
>>>= 0
./parse-git-ref.sh refs/tags/v5.1.99
>>>
{"profile": "emqx", "edition": "Opensource", "release": true, "latest": true}
{"profile": "emqx", "release": true, "latest": true}
>>>= 0
./parse-git-ref.sh refs/heads/master
>>>
{"profile": "emqx", "edition": "Opensource", "release": false, "latest": false}
{"profile": "emqx", "release": false, "latest": false}
>>>= 0
./parse-git-ref.sh refs/heads/release-51
>>>
{"profile": "emqx-enterprise", "edition": "Enterprise", "release": false, "latest": false}
{"profile": "emqx-enterprise", "release": false, "latest": false}
>>>= 0
./parse-git-ref.sh refs/heads/ci/foobar
>>>
{"profile": "emqx", "edition": "Opensource", "release": false, "latest": false}
{"profile": "emqx", "release": false, "latest": false}
>>>= 0
./parse-git-ref.sh refs/heads/release-44