name: Cross build packages concurrency: group: build-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: true on: push: branches: - 'main-v4.**' tags: - v* - e* schedule: - cron: '0 */6 * * *' workflow_dispatch: jobs: prepare: # avoid building when syncing to ee repo if: endsWith(github.repository, 'emqx') runs-on: ubuntu-20.04 container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04 outputs: profiles: ${{ steps.detect-profiles.outputs.profiles}} steps: - uses: actions/checkout@v3 with: path: source fetch-depth: 0 # clone full git history - name: detect-profiles id: detect-profiles uses: ./source/.github/actions/detect-profiles - name: get_all_deps if: endsWith(github.repository, 'emqx') run: | make -C source deps-all zip -ryq source.zip source/* source/.[^.]* - name: get_all_deps if: endsWith(github.repository, 'enterprise') run: | echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials git config --global credential.helper store make -C source deps-all zip -ryq source.zip source/* source/.[^.]* - uses: actions/upload-artifact@v3 with: name: source path: source.zip windows: runs-on: windows-2019 needs: prepare if: endsWith(github.repository, 'emqx') strategy: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - 23.3.4.17 exclude: - profile: emqx-edge steps: - uses: actions/download-artifact@v3 with: name: source path: . - name: unzip source code run: Expand-Archive -Path source.zip -DestinationPath ./ - uses: ilammy/msvc-dev-cmd@v1.12.0 - uses: erlef/setup-beam@v1 with: otp-version: ${{ matrix.otp }} - name: build env: PYTHON: python DIAGNOSTIC: 1 run: | erl -eval "erlang:display(crypto:info_lib())" -s init stop $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" ) if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") { $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+" $pkg_name = "${{ matrix.profile }}-windows-$([regex]::matches($version, $regex).value).zip" } else { $pkg_name = "${{ matrix.profile }}-windows-$($version -replace '/').zip" } cd source ## We do not build/release bcrypt for windows package Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/ if (Test-Path rebar.lock) { Remove-Item -Force -Path rebar.lock } make ensure-rebar3 make ${{ matrix.profile }} mkdir -p _packages/${{ matrix.profile }} Compress-Archive -Path _build/${{ matrix.profile }}/rel/emqx -DestinationPath _build/${{ matrix.profile }}/rel/$pkg_name mv _build/${{ matrix.profile }}/rel/$pkg_name _packages/${{ matrix.profile }} sha256sum "_packages/${{ matrix.profile }}/$pkg_name" | head -c 64 > "_packages/${{ matrix.profile }}/${pkg_name}.sha256" - name: run emqx timeout-minutes: 1 run: | cd source ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start Start-Sleep -s 5 ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall - uses: actions/upload-artifact@v3 with: name: ${{ matrix.profile }} path: source/_packages/${{ matrix.profile }}/ mac: needs: prepare strategy: matrix: otp: - 23.3.4.18-1 os: - macos-11 runs-on: ${{ matrix.os }} steps: - uses: actions/download-artifact@v3 with: name: source path: . - name: unzip source code run: | ln -s . source unzip -q source.zip rm source source.zip - uses: ./.github/actions/detect-profiles - uses: ./.github/actions/package-macos with: otp: ${{ matrix.otp }} os: ${{ matrix.os }} apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }} apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }} apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} - uses: actions/upload-artifact@v3 with: name: ${{ env.EMQX_NAME }} path: _packages/${{ env.EMQX_NAME }}/ linux: runs-on: ubuntu-20.04 needs: prepare strategy: fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} arch: - amd64 - arm64 os: - ubuntu20.04 - ubuntu18.04 - ubuntu16.04 - debian10 - debian9 # - opensuse - centos8 - centos7 - centos6 # - raspbian10 #armv6l is too slow to emulate # - raspbian9 otp_version: #- 23.2.7.2-emqx-3 - 23.3.4.18-1 exclude: - os: centos6 arch: arm64 - os: raspbian9 arch: amd64 - os: raspbian10 arch: amd64 - os: raspbian9 profile: emqx - os: raspbian10 profile: emqx - os: raspbian9 profile: emqx-ee - os: raspbian10 profile: emqx-ee defaults: run: shell: bash steps: - name: prepare docker run: | mkdir -p $HOME/.docker echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker docker info docker buildx create --use --name mybuild docker run --rm --privileged tonistiigi/binfmt --install all - uses: actions/download-artifact@v3 with: name: source path: . - name: unzip source code run: unzip -q source.zip - name: build emqx packages env: ERL_OTP: erl${{ matrix.otp_version }} PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} run: | set -e -u cd source docker buildx build --no-cache \ --platform=linux/$ARCH \ -t cross_build_emqx_for_$SYSTEM \ -f .ci/build_packages/Dockerfile \ --build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-$SYSTEM \ --build-arg EMQX_NAME=$PROFILE \ --output type=tar,dest=/tmp/cross-build-$PROFILE-for-$SYSTEM.tar . mkdir -p /tmp/packages/$PROFILE tar -xvf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar --wildcards emqx/_packages/$PROFILE/* mv emqx/_packages/$PROFILE/* /tmp/packages/$PROFILE/ rm -rf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar docker rm -f $(docker ps -a -q) docker volume prune -f - name: create sha256 env: PROFILE: ${{ matrix.profile }} ERL_OTP: erl${{ matrix.otp_version }} ARCH: ${{ matrix.arch }} run: | if [ -d /tmp/packages/$PROFILE ]; then cd /tmp/packages/$PROFILE for var in $(ls emqx-* ); do if [[ $ERL_OTP == erl23.2* ]]; then # Keep package with new OTP as default # But move package with old otp to track 2 echo "rename track 2 package" oldfile="$var" var="${var/${ARCH}/otp23.2-${ARCH}}" mv "$oldfile" "$var" fi bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256" done cd - fi - uses: actions/upload-artifact@v3 with: name: ${{ matrix.profile }} path: /tmp/packages/${{ matrix.profile }}/ docker: runs-on: ubuntu-20.04 needs: prepare strategy: fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} registry: - 'docker.io' - 'public.ecr.aws' exclude: # we don't have an aws ecr repo for enterprise and edge yet - profile: emqx-edge registry: 'public.ecr.aws' - profile: emqx-ee registry: 'public.ecr.aws' steps: - uses: actions/download-artifact@v3 with: name: source path: . - name: unzip source code run: unzip -q source.zip - uses: docker/setup-buildx-action@v2 - uses: docker/setup-qemu-action@v2 with: image: tonistiigi/binfmt:latest platforms: all - uses: aws-actions/configure-aws-credentials@v1-node16 if: matrix.registry == 'public.ecr.aws' with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - name: Docker login to aws ecr if: matrix.registry == 'public.ecr.aws' run: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws - uses: docker/login-action@v2 if: matrix.registry == 'docker.io' with: username: ${{ secrets.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - uses: docker/metadata-action@v4 id: meta with: images: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }} ## only 5.0 is latest flavor: | latest=false tags: | type=ref,event=branch type=ref,event=pr type=match,pattern=[v|e](.*),group=1 labels: org.opencontainers.image.otp.version=${{ matrix.otp }} - uses: docker/build-push-action@v3 if: matrix.profile != 'emqx-ee' with: ## only push when stable tag and rc tag push: ${{ contains(github.ref, 'tags') && !contains(github.ref_name, 'beta') && !contains(github.ref_name, 'alpha') }} pull: true no-cache: true platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | BUILD_FROM=emqx/build-env:erl23.3.4.18-1-alpine RUN_FROM=alpine:3.12 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile context: source - uses: docker/build-push-action@v3 if: matrix.profile == 'emqx-ee' with: ## only push when stable tag and rc tag push: ${{ contains(github.ref, 'tags') && !contains(github.ref_name, 'beta') && !contains(github.ref_name, 'alpha') }} pull: true no-cache: true platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | BUILD_FROM=emqx/build-env:erl23.3.4.18-1-alpine RUN_FROM=alpine:3.12 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile.enterprise context: source publish_artifacts: runs-on: ubuntu-20.04 if: startsWith(github.ref, 'refs/tags/') needs: [prepare, mac, linux, docker] strategy: fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} steps: - uses: actions/download-artifact@v3 with: name: ${{ matrix.profile }} path: packages/${{ matrix.profile }} - name: install dos2unix run: sudo apt-get update && sudo apt install -y dos2unix - name: get packages run: | set -e -u cd packages/${{ matrix.profile }} for var in $( ls |grep emqx |grep -v sha256); do dos2unix $var.sha256 echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1 done cd - - uses: aws-actions/configure-aws-credentials@v1-node16 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - name: upload to aws s3 run: | set -e -u if [ "${{ matrix.profile }}" == "emqx" ];then s3dir="emqx-ce" else s3dir=${{ matrix.profile }} fi aws s3 cp --recursive packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/${s3dir}/${{ github.ref_name }} aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/${s3dir}/${{ github.ref_name }}/*"