Merge pull request #6716 from qzhuyan/ci/william/build-cache-for-cross-build-packages

build cache for cross build packages

.github/workflows/build_packages.yaml

@@ -164,6 +164,27 @@ jobs:
         kerl update releases
         kerl build ${{ matrix.otp }}
         kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }}
+
+    - name: Get deps git refs for cache
+      id: deps-refs
+      run: |
+        cd source
+        . $HOME/.kerl/${{ matrix.otp }}/activate
+        make ensure-rebar3
+        sudo cp rebar3 /usr/local/bin/rebar3
+        scripts/get-dep-refs.sh
+        make clean-all
+    - name: load rocksdb cache
+      uses: actions/cache@v2
+      with:
+        path: source/_build/default/lib/rocksdb/
+        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
+    - name: load quicer cache
+      uses: actions/cache@v2
+      with:
+        path: source/_build/default/lib/quicer/
+        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
+
     - name: build
       working-directory: source
       run: |
@@ -260,6 +281,22 @@ jobs:
         path: .
     - name: unzip source code
       run: unzip -q source.zip
+    - name: Get deps git refs for cache
+      id: deps-refs
+      run: |
+        cd source
+        scripts/get-dep-refs.sh
+        make clean-all
+    - name: load rocksdb cache
+      uses: actions/cache@v2
+      with:
+        path: source/_build/default/lib/rocksdb/
+        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
+    - name: load quicer cache
+      uses: actions/cache@v2
+      with:
+        path: source/_build/default/lib/quicer/
+        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
     - name: downloads old emqx zip packages
       env:
         OTP_VSN: ${{ matrix.otp }}
@@ -341,6 +378,9 @@ jobs:
         # NOTE: for docker, only support latest otp version, not a matrix
         otp:
           - 24.1.5-3 # update to latest
+        arch:
+          - amd64
+          - arm64
 
     steps:
     - uses: actions/download-artifact@v2
@@ -372,12 +412,33 @@ jobs:
       with:
         username: ${{ secrets.DOCKER_HUB_USER }}
         password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+    - name: Get deps git refs for cache
+      id: deps-refs
+      run: |
+        cd source
+        make ensure-rebar3
+        sudo cp rebar3 /usr/local/bin/rebar3
+        scripts/get-dep-refs.sh
+        make clean-all
+
+    - name: load rocksdb cache
+      uses: actions/cache@v2
+      with:
+        path: source/_build/default/lib/rocksdb/
+        key: alpine3.14-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
+    - name: load quicer cache
+      uses: actions/cache@v2
+      with:
+        path: source/_build/default/lib/quicer/
+        key: alpine3.14-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
+
     - uses: docker/build-push-action@v2
       with:
         push: ${{ github.event_name == 'release' && !github.event.release.prerelease }}
         pull: true
         no-cache: true
-        platforms: linux/amd64,linux/arm64
+        platforms: linux/${{ matrix.arch }}
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |

.github/workflows/build_slim_packages.yaml

@@ -51,11 +51,7 @@ jobs:
     - name: Get deps git refs for cache
       id: deps-refs
       run: |
-        rebar3 get-deps
+        scripts/get-dep-refs.sh
-        RocksDBRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"rocksdb">>,1, Deps), io:format("~s",[Ref]), init:stop().')
-        echo "::set-output name=DEP_ROCKSDB_REF::${RocksDBRef}"
-        QuicerRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"quicer">>,1, Deps), io:format("~s",[Ref]), init:stop().')
-        echo "::set-output name=DEP_QUICER_REF::${QuicerRef}"
         make clean-all
     - name: load rocksdb cache
       uses: actions/cache@v2
@@ -127,11 +123,7 @@ jobs:
         . $HOME/.kerl/${{ matrix.otp }}/activate
         make ensure-rebar3
         sudo cp rebar3 /usr/local/bin/rebar3
-        rebar3 get-deps
+        scripts/get-dep-refs.sh
-        RocksDBRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"rocksdb">>,1, Deps), io:format("~s",[Ref]), init:stop().')
-        echo "::set-output name=DEP_ROCKSDB_REF::${RocksDBRef}"
-        QuicerRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"quicer">>,1, Deps), io:format("~s",[Ref]), init:stop().')
-        echo "::set-output name=DEP_QUICER_REF::${QuicerRef}"
         make clean-all
     - name: load rocksdb cache
       uses: actions/cache@v2

scripts/get-dep-refs.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+get_ref() {
+    local APP=$1
+    #echo "{ok,Raw}=file:consult(\"rebar.lock\"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<\"${APP}\">>,1, Deps), io:format(\"~s\",[Ref]), init:stop()."
+    erl -noshell -eval "{ok,Raw}=file:consult(\"rebar.lock\"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<\"${APP}\">>,1, Deps), io:format(\"~s\",[Ref]), init:stop()."
+}
+
+rebar3 get-deps
+echo "::set-output name=DEP_ROCKSDB_REF::$(get_ref rocksdb)"
+echo "::set-output name=DEP_QUICER_REF::$(get_ref quicer)"