From bf376afb5abc7c7bd72d7a0a054644869c419919 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 12 Jan 2022 15:33:46 +0100 Subject: [PATCH 1/2] ci: speedup all test jobs with build cache --- .github/workflows/build_packages.yaml | 2 - .github/workflows/run_api_tests.yaml | 29 +++++++++++- .github/workflows/run_emqx_app_tests.yaml | 21 ++++++++- .github/workflows/run_fvt_tests.yaml | 54 ++++++++++++++++++++++- .github/workflows/run_relup_tests.yaml | 30 +++++++++++-- 5 files changed, 125 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 01571105b..465cd1a1b 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -173,7 +173,6 @@ jobs: make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 scripts/get-dep-refs.sh - make clean-all - name: load rocksdb cache uses: actions/cache@v2 with: @@ -286,7 +285,6 @@ jobs: run: | cd source scripts/get-dep-refs.sh - make clean-all - name: load rocksdb cache uses: actions/cache@v2 with: diff --git a/.github/workflows/run_api_tests.yaml b/.github/workflows/run_api_tests.yaml index 76e5867f4..c0c05dff0 100644 --- a/.github/workflows/run_api_tests.yaml +++ b/.github/workflows/run_api_tests.yaml @@ -13,12 +13,37 @@ on: jobs: build: + strategy: + matrix: + otp: + - 24.1.5-3 + os: + - ubuntu20.04 + arch: + - amd64 runs-on: ubuntu-latest - container: "ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04" - + container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}" steps: - uses: actions/checkout@v2 + + - name: Get deps git refs for cache + id: deps-refs + run: | + scripts/get-dep-refs.sh + make clean-all + - name: load rocksdb cache + uses: actions/cache@v2 + with: + path: source/_build/default/lib/rocksdb/ + key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }} + - name: load quicer cache + uses: actions/cache@v2 + with: + path: source/_build/default/lib/quicer/ + key: ${{ matrix.os }}-${{ matrix.otp }}-$${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} + - name: build EMQ X CE and EE tgz package + if: endsWith(github.repository, 'emqx') run: | make emqx-tgz make emqx-enterprise-tgz diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index 1aba6c594..731bd7e0a 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -14,12 +14,31 @@ jobs: otp: - 23.3.4.9-3 - 24.1.5-3 + os: + - ubuntu20.04 + arch: + - amd64 runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}" steps: - uses: actions/checkout@v2 + - name: Get deps git refs for cache + id: deps-refs + run: | + PATH=$PATH:./ scripts/get-dep-refs.sh + make clean-all + - name: load rocksdb cache + uses: actions/cache@v2 + with: + path: _build/default/lib/rocksdb/ + key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }} + - name: load quicer cache + uses: actions/cache@v2 + with: + path: _build/default/lib/quicer/ + key: ${{ matrix.os }}-${{ matrix.otp }}-$${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} - name: run run: | make ensure-rebar3 diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 37683ac74..9b04ffd7b 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -44,6 +44,12 @@ jobs: cluster_db_backend: - mnesia - rlog + os: + - alpine3.14 + otp: + - 24.1.5-3 + arch: + - amd64 exclude: - profile: emqx-edge cluster_db_backend: rlog @@ -54,10 +60,29 @@ jobs: path: . - name: unzip source code run: unzip -q source.zip + + - name: Get deps git refs for cache + id: deps-refs + run: | + cd source + make ensure-rebar3 + sudo cp rebar3 /usr/local/bin/rebar3 + scripts/get-dep-refs.sh + - name: load rocksdb cache + uses: actions/cache@v2 + with: + path: source/_build/default/lib/rocksdb/ + key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }} + - name: load quicer cache + uses: actions/cache@v2 + with: + path: source/_build/default/lib/quicer/ + key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} + - name: make docker image working-directory: source env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14 + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }} run: | make ${{ matrix.profile }}-docker - name: run emqx @@ -90,6 +115,12 @@ jobs: matrix: profile: - emqx + os: + - alpine3.14 + otp: + - 24.1.5-3 + arch: + - amd64 # - emqx-enterprise # TODO test enterprise steps: @@ -99,10 +130,29 @@ jobs: path: . - name: unzip source code run: unzip -q source.zip + + - name: Get deps git refs for cache + id: deps-refs + run: | + cd source + make ensure-rebar3 + sudo cp rebar3 /usr/local/bin/rebar3 + scripts/get-dep-refs.sh + - name: load rocksdb cache + uses: actions/cache@v2 + with: + path: source/_build/default/lib/rocksdb/ + key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }} + - name: load quicer cache + uses: actions/cache@v2 + with: + path: source/_build/default/lib/quicer/ + key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} + - name: make docker image working-directory: source env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14 + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }} run: | make ${{ matrix.profile }}-docker echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 766484e36..630130156 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -18,11 +18,15 @@ jobs: profile: - emqx - emqx-enterprise - otp_vsn: + otp: - 24.1.5-3 + os: + - ubuntu20.04 + arch: + - amd64 runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp_vsn }}-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}" defaults: run: @@ -62,7 +66,7 @@ jobs: set -e -x -u cd emqx export PROFILE=${{ matrix.profile }} - export OTP_VSN=${{ matrix.otp_vsn }} + export OTP_VSN=${{ matrix.otp }} echo "PROFILE=$PROFILE" >> $GITHUB_ENV echo "OTP_VSN=$OTP_VSN" >> $GITHUB_ENV if [ $PROFILE = "emqx" ];then @@ -85,8 +89,26 @@ jobs: cd emqx/_upgrade_base old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for old_vsn in ${old_vsns[@]}; do - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${OTP_VSN}-ubuntu20.04-amd64.tar.gz + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${OTP_VSN}-${{ matrix.os }}-${{ matrix.arch }}.tar.gz done + + - name: Get deps git refs for cache + id: deps-refs + run: | + cd emqx + scripts/get-dep-refs.sh + + - name: load rocksdb cache + uses: actions/cache@v2 + with: + path: emqx/_build/default/lib/rocksdb/ + key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }} + - name: load quicer cache + uses: actions/cache@v2 + with: + path: emqx/_build/default/lib/quicer/ + key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} + - name: build emqx run: make -C emqx ${PROFILE}-tgz - name: build emqtt-bench From 451324300b074b66162c67a1106bacbada9072d6 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 17 Jan 2022 08:33:15 +0100 Subject: [PATCH 2/2] ci: speed up run_test_case jobs with cache --- .github/workflows/run_test_cases.yaml | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 2da627366..57687fa91 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -15,14 +15,35 @@ jobs: run_proper_test: strategy: matrix: - emqx_builder: - - 5.0-3:24.1.5-3 + otp: + - 24.1.5-3 + os: + - ubuntu20.04 + arch: + - amd64 runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/${{ matrix.emqx_builder }}-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}" steps: - uses: actions/checkout@v2 + + - name: Get deps git refs for cache + id: deps-refs + run: | + scripts/get-dep-refs.sh + make clean-all + - name: load rocksdb cache + uses: actions/cache@v2 + with: + path: source/_build/default/lib/rocksdb/ + key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }} + - name: load quicer cache + uses: actions/cache@v2 + with: + path: source/_build/default/lib/quicer/ + key: ${{ matrix.os }}-${{ matrix.otp }}-$${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} + - name: proper run: make proper