From 420591d09ab9f2a8885444bd0339c6c917864e44 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Mon, 7 Feb 2022 16:09:09 +0800 Subject: [PATCH 1/3] ci(build_packages): push muilt arch image for aws ecr --- .github/workflows/build_packages.yaml | 83 +++++++++++++++------------ 1 file changed, 47 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 511c0e3c3..ab31ef0a4 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -453,7 +453,14 @@ jobs: build_machine: - aws-arm64 - ubuntu-20.04 + registry: + - 'docker.io' + - 'public.ecr.aws' exclude: + - profile: emqx-edge + registry: 'public.ecr.aws' + - profile: emqx-ee + registry: 'public.ecr.aws' - arch: arm64 build_machine: ubuntu-20.04 - arch: amd64 @@ -494,6 +501,22 @@ jobs: source/deps/quicer/ key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ needs.prepare.outputs.DEP_QUICER_REF }} + - uses: aws-actions/configure-aws-credentials@v1 + if: matrix.repository == 'public.ecr.aws' + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: Docker login for aws ecr + if: matrix.repository == 'public.ecr.aws' + run: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws + + - uses: docker/login-action@v1 + if: matrix.repository == 'docker.io' + with: + username: ${{ secrets.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + - name: prepare for docker-action-parms id: pre-meta run: | @@ -516,7 +539,7 @@ jobs: - uses: docker/metadata-action@v3 id: meta with: - images: ${{ github.repository_owner }}/${{ matrix.profile }} + images: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }} flavor: | latest=${{ github.event_name == 'release' && !github.event.release.prerelease }} suffix=-${{ steps.pre-meta.outputs.img_suffix }} @@ -527,13 +550,6 @@ jobs: type=semver,pattern={{version}} labels: ${{ steps.pre-meta.outputs.img_labels }} - - uses: docker/login-action@v1 - if: > - ${{ (github.event_name == 'release' && !github.event.release.prerelease) - || (github.event.repository.owner != 'emqx' && startsWith(github.ref_name, 'ci/')) }} - with: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - uses: docker/build-push-action@v2 with: @@ -551,28 +567,6 @@ jobs: EMQX_NAME=${{ steps.pre-meta.outputs.emqx_name }} file: source/deploy/docker/Dockerfile context: source - - uses: aws-actions/configure-aws-credentials@v1 - if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - name: Push image to aws ecr - if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' && matrix.build_elixir == 'no_elixir' - run: | - version=${GITHUB_REF##*/} - docker pull emqx/emqx:${version#v} - docker tag emqx/emqx:${version#v} public.ecr.aws/emqx/emqx:${version#v} - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws - docker push public.ecr.aws/emqx/emqx:${version#v} - - name: Push image to aws ecr (elixir) - if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' && matrix.build_elixir == 'with_elixir' - run: | - version=${GITHUB_REF##*/}-elixir - docker pull emqx/emqx:${version#v} - docker tag emqx/emqx:${version#v} public.ecr.aws/emqx/emqx:${version#v} - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws - docker push public.ecr.aws/emqx/emqx:${version#v} docker-push-multi-arch-manifest: # note, we only run on amd64 @@ -604,7 +598,14 @@ jobs: build_machine: - aws-arm64 - ubuntu-20.04 + registry: + - 'docker.io' + - 'public.ecr.aws' exclude: + - profile: emqx-edge + registry: 'public.ecr.aws' + - profile: emqx-ee + registry: 'public.ecr.aws' - arch: arm64 build_machine: ubuntu-20.04 - arch: amd64 @@ -629,11 +630,21 @@ jobs: if: matrix.arch == 'amd64' run: unzip -q source.zip - - uses: docker/login-action@v1 - if: matrix.arch == 'amd64' - with: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} + - uses: aws-actions/configure-aws-credentials@v1 + if: matrix.repository == 'public.ecr.aws' && matrix.arch == 'amd64' + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: Docker login for aws ecr + if: matrix.repository == 'public.ecr.aws' && matrix.arch == 'amd64' + run: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws + + - uses: docker/login-action@v1 + if: matrix.repository == 'docker.io' && matrix.arch == 'amd64' + with: + username: ${{ secrets.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} - name: prepare for docker-action-parms id: pre-meta @@ -658,7 +669,7 @@ jobs: if: matrix.arch == 'amd64' id: meta with: - images: ${{ github.repository_owner }}/${{ matrix.profile }} + images: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }} flavor: | latest=false suffix=-${{ steps.pre-meta.outputs.img_suffix }} From b6e3f666224bde09335331eaa9f3414e7b164c12 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Mon, 7 Feb 2022 17:44:37 +0800 Subject: [PATCH 2/3] ci(build_packages): tag and push aws ecr image when after push docker image --- .github/workflows/build_packages.yaml | 78 +++++++++++++-------------- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index ab31ef0a4..f8986df43 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -453,14 +453,7 @@ jobs: build_machine: - aws-arm64 - ubuntu-20.04 - registry: - - 'docker.io' - - 'public.ecr.aws' exclude: - - profile: emqx-edge - registry: 'public.ecr.aws' - - profile: emqx-ee - registry: 'public.ecr.aws' - arch: arm64 build_machine: ubuntu-20.04 - arch: amd64 @@ -501,18 +494,7 @@ jobs: source/deps/quicer/ key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ needs.prepare.outputs.DEP_QUICER_REF }} - - uses: aws-actions/configure-aws-credentials@v1 - if: matrix.repository == 'public.ecr.aws' - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - name: Docker login for aws ecr - if: matrix.repository == 'public.ecr.aws' - run: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws - - uses: docker/login-action@v1 - if: matrix.repository == 'docker.io' with: username: ${{ secrets.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_HUB_TOKEN }} @@ -539,7 +521,7 @@ jobs: - uses: docker/metadata-action@v3 id: meta with: - images: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }} + images: ${{ github.repository_owner }}/${{ matrix.profile }} flavor: | latest=${{ github.event_name == 'release' && !github.event.release.prerelease }} suffix=-${{ steps.pre-meta.outputs.img_suffix }} @@ -568,6 +550,22 @@ jobs: file: source/deploy/docker/Dockerfile context: source + - uses: aws-actions/configure-aws-credentials@v1 + if: matrix.profile == 'emqx' + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: Docker push to aws ecr + if: matrix.profile == 'emqx' + run: | + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws + for tag in $(echo ${{ steps.meta.outputs.json }} | jq -c '.tags[]'); do + docker pull "$tag" + docker tag "$tag" "public.ecr.aws/$tag" + docker push "public.ecr.aws/$tag" + done + docker-push-multi-arch-manifest: # note, we only run on amd64 if: > @@ -598,14 +596,7 @@ jobs: build_machine: - aws-arm64 - ubuntu-20.04 - registry: - - 'docker.io' - - 'public.ecr.aws' exclude: - - profile: emqx-edge - registry: 'public.ecr.aws' - - profile: emqx-ee - registry: 'public.ecr.aws' - arch: arm64 build_machine: ubuntu-20.04 - arch: amd64 @@ -630,21 +621,11 @@ jobs: if: matrix.arch == 'amd64' run: unzip -q source.zip - - uses: aws-actions/configure-aws-credentials@v1 - if: matrix.repository == 'public.ecr.aws' && matrix.arch == 'amd64' - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - name: Docker login for aws ecr - if: matrix.repository == 'public.ecr.aws' && matrix.arch == 'amd64' - run: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws - - - uses: docker/login-action@v1 - if: matrix.repository == 'docker.io' && matrix.arch == 'amd64' - with: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} + - uses: docker/login-action@v1 + if: matrix.arch == 'amd64' + with: + username: ${{ secrets.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} - name: prepare for docker-action-parms id: pre-meta @@ -669,7 +650,7 @@ jobs: if: matrix.arch == 'amd64' id: meta with: - images: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }} + images: ${{ github.repository_owner }}/${{ matrix.profile }} flavor: | latest=false suffix=-${{ steps.pre-meta.outputs.img_suffix }} @@ -688,6 +669,19 @@ jobs: IsPushLatest=${{ github.event_name == 'release' && !github.event.release.prerelease }}; scripts/docker-create-push-manifests.sh "${{ steps.meta.outputs.tags }}" "$IsPushLatest" + - uses: aws-actions/configure-aws-credentials@v1 + if: matrix.profile == 'emqx' && matrix.arch == 'amd64' + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: Docker push to aws ecr + if: matrix.profile == 'emqx' && matrix.arch == 'amd64' + run: | + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws + IsPushLatest=${{ github.event_name == 'release' && !github.event.release.prerelease }}; + scripts/docker-create-push-manifests.sh "public.ecr.aws/${{ steps.meta.outputs.tags }}" "$IsPushLatest" + delete-artifact: runs-on: ubuntu-20.04 needs: [prepare, mac, linux, docker] From 3875d5c0e217c380952e7b7e10bd61c28917ea0e Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 8 Feb 2022 16:41:16 +0800 Subject: [PATCH 3/3] ci(build_packages): fix the "if" for push aws ecr image --- .github/workflows/build_packages.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index f8986df43..20067af4c 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -557,7 +557,9 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - name: Docker push to aws ecr - if: matrix.profile == 'emqx' + if: > + ${{ (github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx') + || (github.event.repository.owner != 'emqx' && startsWith(github.ref_name, 'ci/')) }} run: | aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws for tag in $(echo ${{ steps.meta.outputs.json }} | jq -c '.tags[]'); do