ci: speedup all test jobs with build cache

2022-01-12 15:33:46 +01:00 · 2022-01-12 15:33:46 +01:00 · bf376afb5a
parent 03a135c592
commit bf376afb5a
5 changed files with 125 additions and 11 deletions
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@ -173,7 +173,6 @@ jobs:
        make ensure-rebar3
        sudo cp rebar3 /usr/local/bin/rebar3
        scripts/get-dep-refs.sh
-        make clean-all
    - name: load rocksdb cache
      uses: actions/cache@v2
      with:
@ -286,7 +285,6 @@ jobs:
      run: |
        cd source
        scripts/get-dep-refs.sh
-        make clean-all
    - name: load rocksdb cache
      uses: actions/cache@v2
      with:
--- a/.github/workflows/run_api_tests.yaml
+++ b/.github/workflows/run_api_tests.yaml
@ -13,12 +13,37 @@ on:

 jobs:
  build:
+    strategy:
+      matrix:
+        otp:
+          - 24.1.5-3
+        os:
+          - ubuntu20.04
+        arch:
+          - amd64
    runs-on: ubuntu-latest
-    container: "ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04"
-
+    container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}"
    steps:
    - uses: actions/checkout@v2
+
+    - name: Get deps git refs for cache
+      id: deps-refs
+      run: |
+        scripts/get-dep-refs.sh
+        make clean-all
+    - name: load rocksdb cache
+      uses: actions/cache@v2
+      with:
+        path: source/_build/default/lib/rocksdb/
+        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
+    - name: load quicer cache
+      uses: actions/cache@v2
+      with:
+        path: source/_build/default/lib/quicer/
+        key: ${{ matrix.os }}-${{ matrix.otp }}-$${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
+
    - name: build EMQ X CE and EE tgz package
+      if: endsWith(github.repository, 'emqx')
      run: |
        make emqx-tgz
        make emqx-enterprise-tgz
--- a/.github/workflows/run_emqx_app_tests.yaml
+++ b/.github/workflows/run_emqx_app_tests.yaml
@ -14,12 +14,31 @@ jobs:
        otp:
          - 23.3.4.9-3
          - 24.1.5-3
+        os:
+          - ubuntu20.04
+        arch:
+          - amd64

    runs-on: ubuntu-20.04
-    container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-ubuntu20.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}"

    steps:
    - uses: actions/checkout@v2
+    - name: Get deps git refs for cache
+      id: deps-refs
+      run: |
+        PATH=$PATH:./ scripts/get-dep-refs.sh
+        make clean-all
+    - name: load rocksdb cache
+      uses: actions/cache@v2
+      with:
+        path: _build/default/lib/rocksdb/
+        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
+    - name: load quicer cache
+      uses: actions/cache@v2
+      with:
+        path: _build/default/lib/quicer/
+        key: ${{ matrix.os }}-${{ matrix.otp }}-$${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
    - name: run
      run: |
        make ensure-rebar3
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@ -44,6 +44,12 @@ jobs:
        cluster_db_backend:
          - mnesia
          - rlog
+        os:
+          - alpine3.14
+        otp:
+          - 24.1.5-3
+        arch:
+          - amd64
        exclude:
          - profile: emqx-edge
            cluster_db_backend: rlog
@ -54,10 +60,29 @@ jobs:
        path: .
    - name: unzip source code
      run: unzip -q source.zip
+
+    - name: Get deps git refs for cache
+      id: deps-refs
+      run: |
+        cd source
+        make ensure-rebar3
+        sudo cp rebar3 /usr/local/bin/rebar3
+        scripts/get-dep-refs.sh
+    - name: load rocksdb cache
+      uses: actions/cache@v2
+      with:
+        path: source/_build/default/lib/rocksdb/
+        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
+    - name: load quicer cache
+      uses: actions/cache@v2
+      with:
+        path: source/_build/default/lib/quicer/
+        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
+
    - name: make docker image
      working-directory: source
      env:
-        EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14
+        EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}
      run: |
        make ${{ matrix.profile }}-docker
    - name: run emqx
@ -90,6 +115,12 @@ jobs:
      matrix:
        profile:
        - emqx
+        os:
+          - alpine3.14
+        otp:
+          - 24.1.5-3
+        arch:
+          - amd64
      # - emqx-enterprise # TODO test enterprise

    steps:
@ -99,10 +130,29 @@ jobs:
        path: .
    - name: unzip source code
      run: unzip -q source.zip
+
+    - name: Get deps git refs for cache
+      id: deps-refs
+      run: |
+        cd source
+        make ensure-rebar3
+        sudo cp rebar3 /usr/local/bin/rebar3
+        scripts/get-dep-refs.sh
+    - name: load rocksdb cache
+      uses: actions/cache@v2
+      with:
+        path: source/_build/default/lib/rocksdb/
+        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
+    - name: load quicer cache
+      uses: actions/cache@v2
+      with:
+        path: source/_build/default/lib/quicer/
+        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
+
    - name: make docker image
      working-directory: source
      env:
-        EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14
+        EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}
      run: |
        make ${{ matrix.profile }}-docker
        echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@ -18,11 +18,15 @@ jobs:
        profile:
          - emqx
          - emqx-enterprise
-        otp_vsn:
+        otp:
          - 24.1.5-3
+        os:
+          - ubuntu20.04
+        arch:
+          - amd64

    runs-on: ubuntu-20.04
-    container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp_vsn }}-ubuntu20.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}"

    defaults:
      run:
@ -62,7 +66,7 @@ jobs:
        set -e -x -u
        cd emqx
        export PROFILE=${{ matrix.profile }}
-        export OTP_VSN=${{ matrix.otp_vsn }}
+        export OTP_VSN=${{ matrix.otp }}
        echo "PROFILE=$PROFILE" >> $GITHUB_ENV
        echo "OTP_VSN=$OTP_VSN" >> $GITHUB_ENV
        if [ $PROFILE = "emqx" ];then
@ -85,8 +89,26 @@ jobs:
        cd emqx/_upgrade_base
        old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
        for old_vsn in ${old_vsns[@]}; do
-          wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${OTP_VSN}-ubuntu20.04-amd64.tar.gz
+          wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${OTP_VSN}-${{ matrix.os }}-${{ matrix.arch }}.tar.gz
        done
+
+    - name: Get deps git refs for cache
+      id: deps-refs
+      run: |
+        cd emqx
+        scripts/get-dep-refs.sh
+
+    - name: load rocksdb cache
+      uses: actions/cache@v2
+      with:
+        path: emqx/_build/default/lib/rocksdb/
+        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
+    - name: load quicer cache
+      uses: actions/cache@v2
+      with:
+        path: emqx/_build/default/lib/quicer/
+        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
+
    - name: build emqx
      run: make -C emqx ${PROFILE}-tgz
    - name: build emqtt-bench