name: Cross build packages concurrency: group: build-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: true on: schedule: - cron: '0 */6 * * *' release: types: - published workflow_dispatch: jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles }} old_vsns: ${{ steps.set_profile.outputs.old_vsns }} steps: - uses: actions/checkout@v2 with: path: source fetch-depth: 0 - name: set profile id: set_profile shell: bash working-directory: source run: | vsn="$(./pkg-vsn.sh)" pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" if make emqx-ee --dry-run > /dev/null 2>&1; then old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")" echo "::set-output name=old_vsns::$old_vsns" echo "::set-output name=profiles::[\"emqx-ee\"]" else old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")" echo "::set-output name=old_vsns::$old_vsns" echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]" fi - name: get_all_deps if: endsWith(github.repository, 'emqx') run: | make -C source deps-all zip -ryq source.zip source/* source/.[^.]* - name: get_all_deps_ee if: endsWith(github.repository, 'enterprise') run: | echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials git config --global credential.helper store echo "${{ secrets.CI_GIT_TOKEN }}" >> source/scripts/git-token make -C source deps-all zip -ryq source.zip source/* source/.[^.]* - uses: actions/upload-artifact@v2 with: name: source path: source.zip windows: runs-on: windows-2019 needs: prepare if: endsWith(github.repository, 'emqx') strategy: fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - 23.2 exclude: - profile: emqx-edge steps: - uses: actions/download-artifact@v2 with: name: source path: . - name: unzip source code run: Expand-Archive -Path source.zip -DestinationPath ./ - uses: ilammy/msvc-dev-cmd@v1 - uses: gleam-lang/setup-erlang@v1.1.2 id: install_erlang ## gleam-lang/setup-erlang does not yet support the installation of otp24 on windows with: otp-version: ${{ matrix.otp }} - name: build env: PYTHON: python DIAGNOSTIC: 1 working-directory: source run: | $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH" $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" ) if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") { $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+" $pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.zip" } else { $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip" } ## We do not build/release bcrypt for windows package Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/ if (Test-Path rebar.lock) { Remove-Item -Force -Path rebar.lock } make ensure-rebar3 copy rebar3 "${{ steps.install_erlang.outputs.erlpath }}\bin" ls "${{ steps.install_erlang.outputs.erlpath }}\bin" rebar3 --help make ${{ matrix.profile }} mkdir -p _packages/${{ matrix.profile }} Compress-Archive -Path _build/${{ matrix.profile }}/rel/emqx -DestinationPath _build/${{ matrix.profile }}/rel/$pkg_name mv _build/${{ matrix.profile }}/rel/$pkg_name _packages/${{ matrix.profile }} Get-FileHash -Path "_packages/${{ matrix.profile }}/$pkg_name" | Format-List | grep 'Hash' | awk '{print $3}' > _packages/${{ matrix.profile }}/$pkg_name.sha256 - name: run emqx timeout-minutes: 1 working-directory: source run: | ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start Start-Sleep -s 5 ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: name: ${{ matrix.profile }} path: source/_packages/${{ matrix.profile }}/. mac: needs: prepare strategy: fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} erl_otp: - 24.1.5-3 macos: - macos-11 - macos-10.15 exclude: - profile: emqx-edge runs-on: ${{ matrix.macos }} steps: - uses: actions/download-artifact@v2 with: name: source path: . - name: unzip source code run: unzip -q source.zip - name: prepare run: | brew update brew install curl zip unzip gnu-sed kerl unixodbc freetds echo "/usr/local/bin" >> $GITHUB_PATH git config --global credential.helper store - uses: actions/cache@v2 id: cache with: path: ~/.kerl key: erl${{ matrix.erl_otp }}-${{ matrix.macos }} - name: build erlang if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 env: KERL_BUILD_BACKEND: git OTP_GITHUB_URL: https://github.com/emqx/otp run: | kerl update releases kerl build ${{ matrix.erl_otp }} kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }} - name: build working-directory: source run: | . $HOME/.kerl/${{ matrix.erl_otp }}/activate make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 rm -rf _build/${{ matrix.profile }}/lib make ${{ matrix.profile }}-zip - name: test working-directory: source run: | set -x pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip) unzip -q $pkg_name gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' for i in {1..10}; do if curl -fs 127.0.0.1:18083 > /dev/null; then ready='yes' break fi sleep 1 done if [ "$ready" != "yes" ]; then echo "Timed out waiting for emqx to be ready" cat emqx/log/erlang.log.1 exit 1 fi ./emqx/bin/emqx_ctl status ./emqx/bin/emqx stop rm -rf emqx openssl dgst -sha256 $pkg_name | awk '{print $2}' > $pkg_name.sha256 - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: name: ${{ matrix.profile }}-${{ matrix.otp }} path: source/_packages/${{ matrix.profile }}/. linux: runs-on: ubuntu-20.04 needs: prepare strategy: fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} package: - zip - pkg otp: - 23.3.4.9-3 - 24.1.5-3 arch: - amd64 - arm64 os: - ubuntu20.04 - ubuntu18.04 - ubuntu16.04 - debian10 - debian9 # - opensuse - centos8 - centos7 - centos6 - raspbian10 # - raspbian9 exclude: - package: pkg otp: 23.3.4.9-3 - os: centos6 arch: arm64 - os: raspbian9 arch: amd64 - os: raspbian10 arch: amd64 - os: raspbian9 profile: emqx - os: raspbian10 profile: emqx - os: raspbian9 profile: emqx-ee - os: raspbian10 profile: emqx-ee defaults: run: shell: bash steps: - uses: docker/setup-buildx-action@v1 - uses: docker/setup-qemu-action@v1 with: image: tonistiigi/binfmt:latest platforms: all - uses: actions/download-artifact@v2 with: name: source path: . - name: unzip source code run: unzip -q source.zip - name: downloads old emqx zip packages if: matrix.package == 'zip' env: OTP_VSN: ${{ matrix.otp }} PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }} working-directory: source run: | set -e -x -u broker=$PROFILE if [ $PROFILE = "emqx" ];then broker="emqx-ce" fi if [ ! -z "$(echo $SYSTEM | grep -oE 'raspbian')" ]; then export ARCH="arm" fi mkdir -p _upgrade_base cd _upgrade_base old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for tag in ${old_vsns[@]}; do package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip) | grep -oE "^[23]+")" ]; then wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip.sha256 echo "$(cat $package_name.zip.sha256) $package_name.zip" | sha256sum -c || exit 1 fi done - name: build emqx packages env: OTP: ${{ matrix.otp }} PROFILE: ${{ matrix.profile }} PACKAGE: ${{ matrix.package}} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} working-directory: source run: | ./scripts/buildx.sh \ --profile "${PROFILE}" \ --pkgtype "${PACKAGE}" \ --arch "${ARCH}" \ --builder "ghcr.io/emqx/emqx-builder/4.4-4:${OTP}-${SYSTEM}" - name: create sha256 working-directory: source env: PROFILE: ${{ matrix.profile}} run: | if [ -d _packages/$PROFILE ]; then cd _packages/$PROFILE for var in $(ls emqx-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256" done cd - fi - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: name: ${{ matrix.profile }}-${{ matrix.otp }} path: source/_packages/${{ matrix.profile }}/. docker: runs-on: ubuntu-20.04 needs: prepare strategy: fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - 24.1.5-3 steps: - uses: actions/download-artifact@v2 with: name: source path: . - name: unzip source code run: unzip -q source.zip - uses: docker/setup-buildx-action@v1 - uses: docker/setup-qemu-action@v1 with: image: tonistiigi/binfmt:latest platforms: all - uses: docker/metadata-action@v3 id: meta with: images: ${{ github.repository_owner }}/${{ matrix.profile }} flavor: | latest=${{ !github.event.release.prerelease }} tags: | type=ref,event=branch type=ref,event=pr type=match,pattern=[v|e](.*),group=1 labels: org.opencontainers.image.otp.version=${{ matrix.otp }} - uses: docker/login-action@v1 if: github.event_name == 'release' with: username: ${{ secrets.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - uses: docker/build-push-action@v2 with: push: ${{ github.event_name == 'release' && !github.event.release.prerelease }} pull: true no-cache: true platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-4:${{ matrix.otp }}-alpine3.14 RUN_FROM=alpine:3.14 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile context: source - uses: aws-actions/configure-aws-credentials@v1 if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - name: Push image to aws ecr if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' run: | version=${GITHUB_REF##*/} docker pull emqx/emqx:${version#v} docker tag emqx/emqx:${version#v} public.ecr.aws/emqx/emqx:${version#v} aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws docker push public.ecr.aws/emqx/emqx:${version#v} delete-artifact: runs-on: ubuntu-20.04 needs: [prepare, mac, linux, docker] steps: - uses: geekyeggo/delete-artifact@v1 with: name: source upload: runs-on: ubuntu-20.04 if: startsWith(github.ref, 'refs/tags/') needs: [prepare, mac, linux, docker] strategy: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - 24.1.5-3 steps: - uses: actions/checkout@v2 - name: get_version run: | echo 'version<> $GITHUB_ENV echo ${{ github.ref }} | sed -r "s ^refs/heads/|^refs/tags/(.*) \1 g" >> $GITHUB_ENV echo 'EOF' >> $GITHUB_ENV - uses: actions/download-artifact@v2 with: name: ${{ matrix.profile }}-${{ matrix.otp }} path: ./_packages/${{ matrix.profile }} - name: install dos2unix run: sudo apt-get update && sudo apt install -y dos2unix - name: get packages run: | set -e -u cd _packages/${{ matrix.profile }} for var in $( ls |grep emqx |grep -v sha256); do dos2unix $var.sha256 echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1 done cd - - name: upload aws s3 run: | set -e -u if [ "${{ matrix.profile }}" == "emqx" ];then broker="emqx-ce" else broker=${{ matrix.profile }} fi aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws configure set default.region ${{ secrets.AWS_DEFAULT_REGION }} aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$broker/${{ env.version }} aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$broker/${{ env.version }}/*" - uses: Rory-Z/upload-release-asset@v1 if: github.event_name == 'release' && matrix.profile != 'emqx-ee' with: repo: emqx path: "_packages/${{ matrix.profile }}/emqx-*" token: ${{ github.token }} - uses: Rory-Z/upload-release-asset@v1 if: github.event_name == 'release' && matrix.profile == 'emqx-ee' with: repo: emqx-enterprise path: "_packages/${{ matrix.profile }}/emqx-*" token: ${{ github.token }} - name: update to emqx.io if: github.event_name == 'release' run: | set -e -x -u curl -w %{http_code} \ --insecure \ -H "Content-Type: application/json" \ -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \ -X POST \ -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - name: push docker image to docker hub if: github.event_name == 'release' run: | set -e -x -u sudo make docker-prepare cd _packages/${{ matrix.profile }} && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd - echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin sudo TARGET=emqx/${{ matrix.profile }} make docker-push sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list - name: update repo.emqx.io if: github.event_name == 'release' && matrix.profile == 'emqx-ee' run: | curl --silent --show-error \ -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ -X POST \ -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \ "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" - name: update repo.emqx.io if: github.event_name == 'release' && matrix.profile == 'emqx' run: | curl --silent --show-error \ -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ -X POST \ -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \ "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" - name: update homebrew packages if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' run: | if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then curl --silent --show-error \ -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ -X POST \ -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \ "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches" fi - uses: geekyeggo/delete-artifact@v1 with: name: ${{ matrix.profile }}