From 7520620a0822804fd5e60627e30b440aaea67846 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Mon, 10 Jan 2022 12:34:41 +0100
Subject: [PATCH 1/3] ci(cross_build_packages): use cache

---
 .github/workflows/build_packages.yaml | 45 +++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index d72053f49..2a4f8d190 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -164,6 +164,31 @@ jobs:
         kerl update releases
         kerl build ${{ matrix.otp }}
         kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }}
+
+    - name: Get deps git refs for cache
+      id: deps-refs
+      run: |
+        cd source
+        . $HOME/.kerl/${{ matrix.otp }}/activate
+        make ensure-rebar3
+        sudo cp rebar3 /usr/local/bin/rebar3
+        rebar3 get-deps
+        RocksDBRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"rocksdb">>,1, Deps), io:format("~s",[Ref]), init:stop().')
+        echo "::set-output name=DEP_ROCKSDB_REF::${RocksDBRef}"
+        QuicerRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"quicer">>,1, Deps), io:format("~s",[Ref]), init:stop().')
+        echo "::set-output name=DEP_QUICER_REF::${QuicerRef}"
+        make clean-all
+    - name: load rocksdb cache
+      uses: actions/cache@v2
+      with:
+        path: source/_build/default/lib/rocksdb/
+        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
+    - name: load quicer cache
+      uses: actions/cache@v2
+      with:
+        path: source/_build/default/lib/quicer/
+        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
+
     - name: build
       working-directory: source
       run: |
@@ -260,6 +285,26 @@ jobs:
         path: .
     - name: unzip source code
       run: unzip -q source.zip
+    - name: Get deps git refs for cache
+      id: deps-refs
+      run: |
+        cd source
+        rebar3 get-deps
+        RocksDBRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"rocksdb">>,1, Deps), io:format("~s",[Ref]), init:stop().')
+        echo "::set-output name=DEP_ROCKSDB_REF::${RocksDBRef}"
+        QuicerRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"quicer">>,1, Deps), io:format("~s",[Ref]), init:stop().')
+        echo "::set-output name=DEP_QUICER_REF::${QuicerRef}"
+        make clean-all
+    - name: load rocksdb cache
+      uses: actions/cache@v2
+      with:
+        path: source/_build/default/lib/rocksdb/
+        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
+    - name: load quicer cache
+      uses: actions/cache@v2
+      with:
+        path: source/_build/default/lib/quicer/
+        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
     - name: downloads old emqx zip packages
       env:
         OTP_VSN: ${{ matrix.otp }}

From 5312aa4a9303888faf91bb291db6b3ab514bfa31 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Tue, 11 Jan 2022 13:00:43 +0100
Subject: [PATCH 2/3] ci(cross-build-package): docker build with cache

---
 .github/workflows/build_packages.yaml | 30 ++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 2a4f8d190..cf580f015 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -386,6 +386,9 @@ jobs:
         # NOTE: for docker, only support latest otp version, not a matrix
         otp:
           - 24.1.5-3 # update to latest
+        arch:
+          - amd64
+          - arm64
 
     steps:
     - uses: actions/download-artifact@v2
@@ -417,12 +420,37 @@ jobs:
       with:
         username: ${{ secrets.DOCKER_HUB_USER }}
         password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+    - name: Get deps git refs for cache
+      id: deps-refs
+      run: |
+        cd source
+        make ensure-rebar3
+        sudo cp rebar3 /usr/local/bin/rebar3
+        rebar3 get-deps
+        RocksDBRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"rocksdb">>,1, Deps), io:format("~s",[Ref]), init:stop().')
+        echo "::set-output name=DEP_ROCKSDB_REF::${RocksDBRef}"
+        QuicerRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"quicer">>,1, Deps), io:format("~s",[Ref]), init:stop().')
+        echo "::set-output name=DEP_QUICER_REF::${QuicerRef}"
+        make clean-all
+
+    - name: load rocksdb cache
+      uses: actions/cache@v2
+      with:
+        path: source/_build/default/lib/rocksdb/
+        key: alpine3.14-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
+    - name: load quicer cache
+      uses: actions/cache@v2
+      with:
+        path: source/_build/default/lib/quicer/
+        key: alpine3.14-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
+
     - uses: docker/build-push-action@v2
       with:
         push: ${{ github.event_name == 'release' && !github.event.release.prerelease }}
         pull: true
         no-cache: true
-        platforms: linux/amd64,linux/arm64
+        platforms: linux/${{ matrix.arch }}
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |

From e388ad1b2185d126fc4601140868243d072ecce6 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 12 Jan 2022 11:58:50 +0100
Subject: [PATCH 3/3] ci(build-cache): add scripts/get-dep-refs.sh

---
 .github/workflows/build_packages.yaml      | 18 +++---------------
 .github/workflows/build_slim_packages.yaml | 12 ++----------
 scripts/get-dep-refs.sh                    | 11 +++++++++++
 3 files changed, 16 insertions(+), 25 deletions(-)
 create mode 100755 scripts/get-dep-refs.sh

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index cf580f015..c48ceab10 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -172,11 +172,7 @@ jobs:
         . $HOME/.kerl/${{ matrix.otp }}/activate
         make ensure-rebar3
         sudo cp rebar3 /usr/local/bin/rebar3
-        rebar3 get-deps
-        RocksDBRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"rocksdb">>,1, Deps), io:format("~s",[Ref]), init:stop().')
-        echo "::set-output name=DEP_ROCKSDB_REF::${RocksDBRef}"
-        QuicerRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"quicer">>,1, Deps), io:format("~s",[Ref]), init:stop().')
-        echo "::set-output name=DEP_QUICER_REF::${QuicerRef}"
+        scripts/get-dep-refs.sh
         make clean-all
     - name: load rocksdb cache
       uses: actions/cache@v2
@@ -289,11 +285,7 @@ jobs:
       id: deps-refs
       run: |
         cd source
-        rebar3 get-deps
-        RocksDBRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"rocksdb">>,1, Deps), io:format("~s",[Ref]), init:stop().')
-        echo "::set-output name=DEP_ROCKSDB_REF::${RocksDBRef}"
-        QuicerRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"quicer">>,1, Deps), io:format("~s",[Ref]), init:stop().')
-        echo "::set-output name=DEP_QUICER_REF::${QuicerRef}"
+        scripts/get-dep-refs.sh
         make clean-all
     - name: load rocksdb cache
       uses: actions/cache@v2
@@ -427,11 +419,7 @@ jobs:
         cd source
         make ensure-rebar3
         sudo cp rebar3 /usr/local/bin/rebar3
-        rebar3 get-deps
-        RocksDBRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"rocksdb">>,1, Deps), io:format("~s",[Ref]), init:stop().')
-        echo "::set-output name=DEP_ROCKSDB_REF::${RocksDBRef}"
-        QuicerRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"quicer">>,1, Deps), io:format("~s",[Ref]), init:stop().')
-        echo "::set-output name=DEP_QUICER_REF::${QuicerRef}"
+        scripts/get-dep-refs.sh
         make clean-all
 
     - name: load rocksdb cache
diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml
index 80df560a1..aa061e202 100644
--- a/.github/workflows/build_slim_packages.yaml
+++ b/.github/workflows/build_slim_packages.yaml
@@ -51,11 +51,7 @@ jobs:
     - name: Get deps git refs for cache
       id: deps-refs
       run: |
-        rebar3 get-deps
-        RocksDBRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"rocksdb">>,1, Deps), io:format("~s",[Ref]), init:stop().')
-        echo "::set-output name=DEP_ROCKSDB_REF::${RocksDBRef}"
-        QuicerRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"quicer">>,1, Deps), io:format("~s",[Ref]), init:stop().')
-        echo "::set-output name=DEP_QUICER_REF::${QuicerRef}"
+        scripts/get-dep-refs.sh
         make clean-all
     - name: load rocksdb cache
       uses: actions/cache@v2
@@ -123,11 +119,7 @@ jobs:
         . $HOME/.kerl/${{ matrix.otp }}/activate
         make ensure-rebar3
         sudo cp rebar3 /usr/local/bin/rebar3
-        rebar3 get-deps
-        RocksDBRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"rocksdb">>,1, Deps), io:format("~s",[Ref]), init:stop().')
-        echo "::set-output name=DEP_ROCKSDB_REF::${RocksDBRef}"
-        QuicerRef=$(erl -noshell -eval '{ok,Raw}=file:consult("rebar.lock"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<"quicer">>,1, Deps), io:format("~s",[Ref]), init:stop().')
-        echo "::set-output name=DEP_QUICER_REF::${QuicerRef}"
+        scripts/get-dep-refs.sh
         make clean-all
     - name: load rocksdb cache
       uses: actions/cache@v2
diff --git a/scripts/get-dep-refs.sh b/scripts/get-dep-refs.sh
new file mode 100755
index 000000000..5218d4903
--- /dev/null
+++ b/scripts/get-dep-refs.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+get_ref() {
+    local APP=$1
+    #echo "{ok,Raw}=file:consult(\"rebar.lock\"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<\"${APP}\">>,1, Deps), io:format(\"~s\",[Ref]), init:stop()."
+    erl -noshell -eval "{ok,Raw}=file:consult(\"rebar.lock\"), {_Vsn, Deps}=hd(Raw), {_, {git, _Url, {ref, Ref}},_} = lists:keyfind(<<\"${APP}\">>,1, Deps), io:format(\"~s\",[Ref]), init:stop()."
+}
+
+rebar3 get-deps
+echo "::set-output name=DEP_ROCKSDB_REF::$(get_ref rocksdb)"
+echo "::set-output name=DEP_QUICER_REF::$(get_ref quicer)"