From c587a753da34406b1aaff61e244539309b92ec80 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 9 Aug 2023 14:05:36 +0200 Subject: [PATCH] fix(docker): use 'make $PROFILE-docker' in CI --- .github/workflows/_push-entrypoint.yaml | 11 +-- .../build_and_push_docker_images.yaml | 99 +++++-------------- build | 88 +++++++++++++---- scripts/parse-git-ref.sh | 11 +-- scripts/shelltest/parse-git-ref.test | 30 +++--- 5 files changed, 115 insertions(+), 124 deletions(-) diff --git a/.github/workflows/_push-entrypoint.yaml b/.github/workflows/_push-entrypoint.yaml index c1589d6db..a65df1234 100644 --- a/.github/workflows/_push-entrypoint.yaml +++ b/.github/workflows/_push-entrypoint.yaml @@ -23,7 +23,6 @@ jobs: container: 'ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04' outputs: profile: ${{ steps.parse-git-ref.outputs.profile }} - edition: ${{ steps.parse-git-ref.outputs.edition }} release: ${{ steps.parse-git-ref.outputs.release }} latest: ${{ steps.parse-git-ref.outputs.latest }} version: ${{ steps.parse-git-ref.outputs.version }} @@ -50,12 +49,10 @@ jobs: run: | JSON="$(./scripts/parse-git-ref.sh $GITHUB_REF)" PROFILE=$(echo "$JSON" | jq -cr '.profile') - EDITION=$(echo "$JSON" | jq -cr '.edition') RELEASE=$(echo "$JSON" | jq -cr '.release') LATEST=$(echo "$JSON" | jq -cr '.latest') VERSION="$(./pkg-vsn.sh "$PROFILE")" echo "profile=$PROFILE" | tee -a $GITHUB_OUTPUT - echo "edition=$EDITION" | tee -a $GITHUB_OUTPUT echo "release=$RELEASE" | tee -a $GITHUB_OUTPUT echo "latest=$LATEST" | tee -a $GITHUB_OUTPUT echo "version=$VERSION" | tee -a $GITHUB_OUTPUT @@ -107,8 +104,8 @@ jobs: publish: ${{ needs.prepare.outputs.release }} otp_vsn: ${{ needs.prepare.outputs.otp_vsn }} elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }} - runner: ${{ needs.prepare.outputs.runner }} builder_vsn: ${{ needs.prepare.outputs.builder_vsn }} + runner: ${{ needs.prepare.outputs.runner }} secrets: inherit build_and_push_docker_images: @@ -118,15 +115,13 @@ jobs: uses: ./.github/workflows/build_and_push_docker_images.yaml with: profile: ${{ needs.prepare.outputs.profile }} - edition: ${{ needs.prepare.outputs.edition }} version: ${{ needs.prepare.outputs.version }} - latest: ${{ needs.prepare.outputs.latest }} publish: ${{ needs.prepare.outputs.release }} + latest: ${{ needs.prepare.outputs.latest }} otp_vsn: ${{ needs.prepare.outputs.otp_vsn }} elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }} - # workaround: self-hosted runners do not have access to org-level secrets? - runner: ubuntu-22.04 builder_vsn: ${{ needs.prepare.outputs.builder_vsn }} + runner: ${{ needs.prepare.outputs.runner }} secrets: inherit compile: diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 6c7bdbeff..b0d6aa481 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -10,9 +10,6 @@ on: profile: required: true type: string - edition: - required: true - type: string version: required: true type: string @@ -28,10 +25,10 @@ on: elixir_vsn: required: true type: string - runner: + builder_vsn: required: true type: string - builder_vsn: + runner: required: true type: string secrets: @@ -47,17 +44,12 @@ on: inputs: ref: required: false - version: - required: true - type: string profile: required: false type: string default: 'emqx' - edition: - required: false - type: string - default: 'Opensource' + version: + required: true latest: required: false type: boolean @@ -74,14 +66,14 @@ on: required: false type: string default: '1.14.5' - runner: - required: false - type: string - default: 'ubuntu-22.04' builder_vsn: required: false type: string default: '5.1-3' + runner: + required: false + type: string + default: 'ubuntu-22.04' jobs: docker: @@ -92,18 +84,10 @@ jobs: matrix: profile: - ${{ inputs.profile }} + - ${{ inputs.profile }}-elixir registry: - 'docker.io' - 'public.ecr.aws' - os: - - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] - builder: - - ${{ inputs.builder_vsn }} - otp: - - ${{ inputs.otp_vsn }} - elixir: - - 'no_elixir' - - ${{ inputs.elixir_vsn }} steps: - uses: actions/checkout@v3 @@ -130,53 +114,20 @@ jobs: password: ${{ secrets.AWS_SECRET_ACCESS_KEY }} ecr: true - - name: prepare for docker/metadata-action - id: pre-meta - shell: bash + - name: Build docker image + env: + PROFILE: ${{ matrix.profile }} + DOCKER_REGISTRY: ${{ matrix.registry }} + DOCKER_ORG: ${{ github.repository_owner }} + DOCKER_LATEST: ${{ inputs.latest }} + DOCKER_PUSH: ${{ inputs.publish == 'true' || inputs.publish || github.repository_owner != 'emqx' }} + DOCKER_BUILD_NOCACHE: true + DOCKER_PLATFORMS: linux/amd64,linux/arm64 + EMQX_RUNNER: 'debian:11-slim' + EMQX_DOCKERFILE: 'deploy/docker/Dockerfile' + PKG_VSN: ${{ inputs.version }} + EMQX_BUILDER_VSN: ${{ inputs.builder_vsn }} + EMQX_OTP_VSN: ${{ inputs.otp_vsn }} + EMQX_ELIXIR_VSN: ${{ inputs.elixir_vsn }} run: | - extra_labels= - img_suffix= - if [ "${{ matrix.elixir }}" != 'no_elixir' ]; then - img_suffix="-elixir" - extra_labels="org.opencontainers.image.elixir.version=${{ matrix.elixir }}" - fi - extra_deps= - if [[ "${{ matrix.profile }}" = *enterprise* ]]; then - extra_deps='libsasl2-2,libsasl2-modules-gssapi-mit' - fi - - echo "img_suffix=$img_suffix" >> $GITHUB_OUTPUT - echo "extra_labels=$extra_labels" >> $GITHUB_OUTPUT - echo "extra_deps=$extra_deps" >> $GITHUB_OUTPUT - - - uses: docker/metadata-action@v4 - id: meta - with: - images: | - ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }} - flavor: | - latest=${{ matrix.elixir == 'no_elixir' }} - suffix=${{ steps.pre-meta.outputs.img_suffix }} - tags: | - type=semver,pattern={{major}}.{{minor}},value=${{ inputs.version }} - type=semver,pattern={{version}},value=${{ inputs.version }} - type=raw,value=${{ inputs.version }} - type=raw,value=latest,enable=${{ inputs.latest }} - labels: | - org.opencontainers.image.otp.version=${{ matrix.otp }} - org.opencontainers.image.edition=${{ inputs.edition }} - ${{ steps.pre-meta.outputs.extra_labels }} - - - uses: docker/build-push-action@v3 - with: - push: ${{ inputs.publish == 'true' || inputs.publish || github.repository_owner != 'emqx' }} - pull: true - no-cache: true - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - EMQX_NAME=${{ matrix.profile }}${{ steps.pre-meta.outputs.img_suffix }} - EXTRA_DEPS=${{ steps.pre-meta.outputs.extra_deps }} - PKG_VSN=${{ inputs.version }} - file: ${{ matrix.os[2] }} + ./build ${PROFILE} docker diff --git a/build b/build index fb0d213f2..df4cb19e8 100755 --- a/build +++ b/build @@ -368,21 +368,81 @@ docker_cleanup() { } ## Build the default docker image based on debian 11. -## NOTE: docker image build in github action does not call this -## function, see build_and_push_docker_images.yaml make_docker() { - EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}" - EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}" - EMQX_DOCKERFILE="${EMQX_DOCKERFILE:-deploy/docker/Dockerfile}" + local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.1-3}" + local EMQX_BUILDER_PLATFORM="${EMQX_BUILDER_PLATFORM:-debian11}" + local EMQX_BUILDER_OTP="${EMQX_BUILDER_OTP:-25.3.2-1}" + local EMQX_BUILDER_ELIXIR="${EMQX_BUILDER_ELIXIR:-1.14.5}" + local EMQX_BUILDER=${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VERSION}:${EMQX_BUILDER_ELIXIR}-${EMQX_BUILDER_OTP}-${EMQX_BUILDER_PLATFORM}} + local EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}" + local EMQX_DOCKERFILE="${EMQX_DOCKERFILE:-deploy/docker/Dockerfile}" + local PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" + # shellcheck disable=SC2155 + local VSN_MAJOR="$(echo "$PKG_VSN" | cut -d . -f 1)" + # shellcheck disable=SC2155 + local VSN_MINOR="$(echo "$PKG_VSN" | cut -d . -f 2)" + # shellcheck disable=SC2155 + local VSN_PATCH="$(echo "$PKG_VSN" | cut -d . -f 3)" + local SUFFIX='' if [[ "$PROFILE" = *-elixir ]]; then - PKG_VSN="$PKG_VSN-elixir" + SUFFIX="-elixir" fi - local default_tag="emqx/${PROFILE%%-elixir}:${PKG_VSN}" - EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-$default_tag}" + local DOCKER_REGISTRY="${REGISTRY:-docker.io}" + local DOCKER_ORG="${DOCKER_ORG:-emqx}" + local EMQX_BASE_DOCKER_TAG="${DOCKER_REGISTRY}/${DOCKER_ORG}/${PROFILE%%-elixir}" + local default_tag="${EMQX_BASE_DOCKER_TAG}:${PKG_VSN}${SUFFIX}" + local EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-$default_tag}" + local EDITION=Opensource + local LICENSE='Apache-2.0' ## extra_deps is a comma separated list of debian 11 package names - local extra_deps='' + local EXTRA_DEPS='' if [[ "$PROFILE" = *enterprise* ]]; then - extra_deps='libsasl2-2,libsasl2-modules-gssapi-mit' + EXTRA_DEPS='libsasl2-2,libsasl2-modules-gssapi-mit' + EDITION=Enterprise + LICENSE='(Apache-2.0 AND BSL-1.1)' + fi + # shellcheck disable=SC2155 + local ISO_8601_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + # shellcheck disable=SC2155 + local GIT_REVISION="$(git rev-parse HEAD)" + local DOCKER_BUILDX_ARGS=( + --build-arg BUILD_FROM="${EMQX_BUILDER}" \ + --build-arg RUN_FROM="${EMQX_RUNNER}" \ + --build-arg EMQX_NAME="${PROFILE}" \ + --build-arg EXTRA_DEPS="${EXTRA_DEPS}" \ + --build-arg PKG_VSN="${PKG_VSN}" \ + --file "${EMQX_DOCKERFILE}" \ + --label org.opencontainers.image.title="${PROFILE}" \ + --label org.opencontainers.image.edition="${EDITION}" \ + --label org.opencontainers.image.version="${PKG_VSN}" \ + --label org.opencontainers.image.revision="${GIT_REVISION}" \ + --label org.opencontainers.image.created="${ISO_8601_DATE}" \ + --label org.opencontainers.image.source='https://github.com/emqx/emqx' \ + --label org.opencontainers.image.url='https://github.com/emqx/emqx' \ + --label org.opencontainers.image.documentation='https://www.emqx.io/docs/en/latest/' \ + --label org.opencontainers.image.description='The most scalable open-source MQTT broker for IoT, IIoT, and connected vehicles' \ + --label org.opencontainers.image.licenses="${LICENSE}" \ + --label org.opencontainers.image.otp.version="${EMQX_BUILDER_OTP}" \ + --tag "${EMQX_IMAGE_TAG}" \ + --tag "${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}${SUFFIX}" \ + --tag "${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}.${VSN_PATCH}${SUFFIX}" \ + --provenance false \ + --pull + ) + if [ "${DOCKER_BUILD_NOCACHE:-false}" = true ]; then + DOCKER_BUILDX_ARGS+=(--no-cache) + fi + if [ "${SUFFIX}" = '-elixir' ]; then + DOCKER_BUILDX_ARGS+=(--label org.opencontainers.image.elixir.version="${EMQX_BUILDER_ELIXIR}") + fi + if [ "${DOCKER_LATEST:-false}" = true ]; then + DOCKER_BUILDX_ARGS+=(--tag "${DOCKER_REGISTRY}/${DOCKER_ORG}/${PROFILE}:latest") + fi + if [ "${DOCKER_PLATFORMS:-default}" != 'default' ]; then + DOCKER_BUILDX_ARGS+=(--platform "${DOCKER_PLATFORMS}") + fi + if [ "${DOCKER_PUSH-false}" = true ]; then + DOCKER_BUILDX_ARGS+=(--push) fi # shellcheck disable=SC2015 [ -f ./.dockerignore ] && mv ./.dockerignore ./.dockerignore.bak || true @@ -393,13 +453,7 @@ make_docker() { echo '/*.lock' } >> ./.dockerignore set -x - docker build --no-cache --pull \ - --build-arg BUILD_FROM="${EMQX_BUILDER}" \ - --build-arg RUN_FROM="${EMQX_RUNNER}" \ - --build-arg EMQX_NAME="${PROFILE}" \ - --build-arg EXTRA_DEPS="${extra_deps}" \ - --tag "${EMQX_IMAGE_TAG}" \ - -f "${EMQX_DOCKERFILE}" . + docker buildx build "${DOCKER_BUILDX_ARGS[@]}" . [[ "${DEBUG:-}" -eq 1 ]] || set +x echo "${EMQX_IMAGE_TAG}" > ./.docker_image_tag } diff --git a/scripts/parse-git-ref.sh b/scripts/parse-git-ref.sh index a486f2589..23d88d2eb 100755 --- a/scripts/parse-git-ref.sh +++ b/scripts/parse-git-ref.sh @@ -18,32 +18,26 @@ is_latest() { if [[ $1 =~ ^refs/tags/v[5-9]+\.[0-9]+\.[0-9]+$ ]]; then PROFILE=emqx - EDITION=Opensource RELEASE=true LATEST=$(is_latest "$1") elif [[ $1 =~ ^refs/tags/v[5-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then PROFILE=emqx - EDITION=Opensource RELEASE=true LATEST=$(is_latest "$1") elif [[ $1 =~ ^refs/tags/e[5-9]+\.[0-9]+\.[0-9]+$ ]]; then PROFILE=emqx-enterprise - EDITION=Enterprise RELEASE=true LATEST=$(is_latest "$1") elif [[ $1 =~ ^refs/tags/e[5-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then PROFILE=emqx-enterprise - EDITION=Enterprise RELEASE=true LATEST=$(is_latest "$1") elif [[ $1 =~ ^refs/tags/v[5-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$ ]]; then PROFILE=emqx - EDITION=Opensource RELEASE=true LATEST=false elif [[ $1 =~ ^refs/tags/e[5-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$ ]]; then PROFILE=emqx-enterprise - EDITION=Enterprise RELEASE=true LATEST=false elif [[ $1 =~ ^refs/tags/.+ ]]; then @@ -51,17 +45,14 @@ elif [[ $1 =~ ^refs/tags/.+ ]]; then exit 1 elif [[ $1 =~ ^refs/heads/master$ ]]; then PROFILE=emqx - EDITION=Opensource RELEASE=false LATEST=false elif [[ $1 =~ ^refs/heads/release-[5-9][0-9]+$ ]]; then PROFILE=emqx-enterprise - EDITION=Enterprise RELEASE=false LATEST=false elif [[ $1 =~ ^refs/heads/ci/.* ]]; then PROFILE=emqx - EDITION=Opensource RELEASE=false LATEST=false else @@ -70,5 +61,5 @@ else fi cat <>> -{"profile": "emqx", "edition": "Opensource", "release": true, "latest": false} +{"profile": "emqx", "release": true, "latest": false} >>>= 0 ./parse-git-ref.sh refs/tags/v5.1.5.1 >>> -{"profile": "emqx", "edition": "Opensource", "release": true, "latest": false} +{"profile": "emqx", "release": true, "latest": false} >>>= 0 ./parse-git-ref.sh refs/tags/v5.2.0-alpha.1 >>> -{"profile": "emqx", "edition": "Opensource", "release": true, "latest": false} +{"profile": "emqx", "release": true, "latest": false} >>>= 0 ./parse-git-ref.sh refs/tags/v5.2.0-alpha-1 @@ -30,62 +30,62 @@ Unrecognized tag: refs/tags/v5.2.0-alpha-1 ./parse-git-ref.sh refs/tags/v5.2.0-beta.1 >>> -{"profile": "emqx", "edition": "Opensource", "release": true, "latest": false} +{"profile": "emqx", "release": true, "latest": false} >>>= 0 ./parse-git-ref.sh refs/tags/v5.2.0-rc.1 >>> -{"profile": "emqx", "edition": "Opensource", "release": true, "latest": false} +{"profile": "emqx", "release": true, "latest": false} >>>= 0 ./parse-git-ref.sh refs/tags/e5.1.0 >>> -{"profile": "emqx-enterprise", "edition": "Enterprise", "release": true, "latest": false} +{"profile": "emqx-enterprise", "release": true, "latest": false} >>>= 0 ./parse-git-ref.sh refs/tags/e5.1.5.1 >>> -{"profile": "emqx-enterprise", "edition": "Enterprise", "release": true, "latest": false} +{"profile": "emqx-enterprise", "release": true, "latest": false} >>>= 0 ./parse-git-ref.sh refs/tags/e5.2.0-alpha.1 >>> -{"profile": "emqx-enterprise", "edition": "Enterprise", "release": true, "latest": false} +{"profile": "emqx-enterprise", "release": true, "latest": false} >>>= 0 ./parse-git-ref.sh refs/tags/e5.2.0-beta.1 >>> -{"profile": "emqx-enterprise", "edition": "Enterprise", "release": true, "latest": false} +{"profile": "emqx-enterprise", "release": true, "latest": false} >>>= 0 ./parse-git-ref.sh refs/tags/e5.2.0-rc.1 >>> -{"profile": "emqx-enterprise", "edition": "Enterprise", "release": true, "latest": false} +{"profile": "emqx-enterprise", "release": true, "latest": false} >>>= 0 ./parse-git-ref.sh refs/tags/e5.1.99 >>> -{"profile": "emqx-enterprise", "edition": "Enterprise", "release": true, "latest": true} +{"profile": "emqx-enterprise", "release": true, "latest": true} >>>= 0 ./parse-git-ref.sh refs/tags/v5.1.99 >>> -{"profile": "emqx", "edition": "Opensource", "release": true, "latest": true} +{"profile": "emqx", "release": true, "latest": true} >>>= 0 ./parse-git-ref.sh refs/heads/master >>> -{"profile": "emqx", "edition": "Opensource", "release": false, "latest": false} +{"profile": "emqx", "release": false, "latest": false} >>>= 0 ./parse-git-ref.sh refs/heads/release-51 >>> -{"profile": "emqx-enterprise", "edition": "Enterprise", "release": false, "latest": false} +{"profile": "emqx-enterprise", "release": false, "latest": false} >>>= 0 ./parse-git-ref.sh refs/heads/ci/foobar >>> -{"profile": "emqx", "edition": "Opensource", "release": false, "latest": false} +{"profile": "emqx", "release": false, "latest": false} >>>= 0 ./parse-git-ref.sh refs/heads/release-44