Merge pull request #9574 from id/chore-fail-build-if-apple-notarization-failed

chore: fail build if apple notarytool failed

.github/actions/package-macos/action.yaml

@@ -23,27 +23,38 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: prepare
+    - id: prepare
       shell: bash
       run: |
         brew update
-        brew install curl zip unzip gnu-sed coreutils unixodbc freetds openssl@1.1
+        brew install curl zip unzip gnu-sed coreutils autoconf automake cmake unixodbc freetds openssl@1.1
         echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
         echo "/usr/local/bin" >> $GITHUB_PATH
+        OTP_SOURCE_PATH="$HOME/src/otp-${{ inputs.otp }}"
+        OTP_INSTALL_PATH="$HOME/otp/${{ inputs.otp }}"
+        echo "OTP_SOURCE_PATH=$OTP_SOURCE_PATH" >> $GITHUB_OUTPUT
+        echo "OTP_INSTALL_PATH=$OTP_INSTALL_PATH" >> $GITHUB_OUTPUT
+        mkdir -p "$OTP_INSTALL_PATH"
     - uses: actions/cache@v3
       id: cache
       with:
-        path: /opt/erlang/${{ inputs.otp }}
+        path: ${{ steps.prepare.outputs.OTP_INSTALL_PATH }}
         key: otp-install-${{ inputs.otp }}-${{ inputs.os }}-static-ssl-disable-hipe-disable-jit
     - name: build erlang
       if: steps.cache.outputs.cache-hit != 'true'
       shell: bash
       run: |
-        git clone --depth 1 --branch OTP-${{ inputs.otp }} https://github.com/emqx/otp.git $HOME/otp-${{ inputs.otp }}
+        OTP_SOURCE_PATH="${{ steps.prepare.outputs.OTP_SOURCE_PATH }}"
-        cd $HOME/otp-${{ inputs.otp }}
+        OTP_INSTALL_PATH="${{ steps.prepare.outputs.OTP_INSTALL_PATH }}"
-        ./configure --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit --prefix=/opt/erlang/${{ inputs.otp }}
+        if [ -d "$OTP_SOURCE_PATH" ]; then
+          rm -rf "$OTP_SOURCE_PATH"
+        fi
+        git clone --depth 1 --branch OTP-${{ inputs.otp }} https://github.com/emqx/otp.git "$OTP_SOURCE_PATH"
+        cd "$OTP_SOURCE_PATH"
+        ./configure --disable-dynamic-ssl-lib --with-ssl=$(brew --prefix openssl@1.1) --disable-hipe --disable-jit --prefix="$OTP_INSTALL_PATH"
         make -j$(nproc)
-        sudo make install
+        rm -rf "$OTP_INSTALL_PATH"
+        make install
     - name: build
       env:
         AUTO_INSTALL_BUILD_DEPS: 1
@@ -56,13 +67,16 @@ runs:
         APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ inputs.apple_developer_id_bundle_password }}
       shell: bash
       run: |
-        export PATH="/opt/erlang/${{ inputs.otp }}/bin:$PATH"
+        export PATH="${{ steps.prepare.outputs.OTP_INSTALL_PATH }}/bin:$PATH"
         make ensure-rebar3
-        sudo cp rebar3 /usr/local/bin/rebar3
+        mkdir -p $HOME/bin
+        cp rebar3 $HOME/bin/rebar3
+        export PATH="$HOME/bin:$PATH"
         make ${EMQX_NAME}-zip
     - name: test
       shell: bash
       run: |
+        export PATH="${{ steps.prepare.outputs.OTP_INSTALL_PATH }}/bin:$PATH"
         pkg_name=$(basename _packages/${EMQX_NAME}/${EMQX_NAME}-*.zip)
         unzip -q _packages/${EMQX_NAME}/$pkg_name
         # test with a spaces in path

.github/workflows/build_slim_packages.yaml

@@ -124,7 +124,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: emqx/self-hosted-cleanup-action@v1.0.3
-      if: matrix.os == macos-12-arm64
+      if: ${{ matrix.os == 'macos-12-arm64' }}
     - uses: actions/checkout@v3
       with:
           fetch-depth: 0 # clone full git history
@@ -149,5 +149,5 @@ jobs:
         path: ./rebar3.crashdump
     - uses: actions/upload-artifact@v3
       with:
-        name: macos
+        name: ${{ matrix.os }}
         path: _packages/**/*.zip

build

@@ -203,6 +203,16 @@ make_zip() {
                     ;;
             esac
             ;;
+        macos*)
+            case "$PKG_VSN" in
+                4.4.12*)
+                    # this is the first version for macos arm (M1/M2), no relup
+                    if [ "$ARCH" = arm64 ]; then
+                        has_relup='no'
+                    fi
+                    ;;
+            esac
+            ;;
     esac
     # shellcheck disable=SC2207
     bases=($(relup_db base-vsns "$PKG_VSN"))
@@ -236,10 +246,18 @@ make_zip() {
                     #   --apple-id <apple id> \
                     #   --password <apple id password>
                     #   --team-id <apple team id> <submission-id>
-                xcrun notarytool submit \
+                echo 'Submitting the package for notarization to Apple (normally takes about a minute)'
+                notarytool_output="$(xcrun notarytool submit \
                                            --apple-id "${APPLE_ID}" \
                                            --password "${APPLE_ID_PASSWORD}" \
-                      --team-id "${APPLE_TEAM_ID}" "${target_zip}" --wait
+                                           --team-id "${APPLE_TEAM_ID}" "${target_zip}" \
+                                           --no-progress \
+                                           --wait)"
+                echo "$notarytool_output"
+                echo "$notarytool_output" | grep -q 'status: Accepted' || {
+                    echo 'Notarization failed';
+                    exit 1;
+                }
             fi
             # sha256sum may not be available on macos
             openssl dgst -sha256 "${target_zip}" | cut -d ' ' -f 2  > "${target_zip}.sha256"

scripts/macos-sign-binaries.sh

@@ -43,10 +43,24 @@ done
 security -v list-keychains -s "${keychain_names[@]}" "${KEYCHAIN}"
 
 # known runtime executables and binaries
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/erts-*/bin/{beam.smp,dyn_erl,epmd,erl,erl_call,erl_child_setup,erlexec,escript,heart,inet_gethost,run_erl,to_erl}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/runtime_tools-*/priv/lib/{dyntrace.so,trace_ip_drv.so,trace_file_drv.so}
+         "${REL_DIR}"/erts-*/bin/{beam.smp,dyn_erl,epmd,erl,erl_call,erl_child_setup,erlexec,escript,heart,inet_gethost,run_erl,to_erl}
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/os_mon-*/priv/bin/{cpu_sup,memsup}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \
+         "${REL_DIR}"/lib/runtime_tools-*/priv/lib/{dyntrace.so,trace_ip_drv.so,trace_file_drv.so}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \
+         "${REL_DIR}"/lib/os_mon-*/priv/bin/{cpu_sup,memsup}
 # other files from runtime and dependencies
-for f in asn1rt_nif.so bcrypt_nif.so crypto.so otp_test_engine.so crypto_callback.so jiffy.so crc32cer_nif.so sasl_auth.so snappyer.so odbcserver; do
+for f in \
+        asn1rt_nif.so \
+        bcrypt_nif.so \
+        crypto.so \
+        otp_test_engine.so \
+        crypto_callback.so \
+        jiffy.so \
+        crc32cer_nif.so \
+        sasl_auth.so \
+        snappyer.so \
+        odbcserver \
+    ; do
     find "${REL_DIR}"/lib/ -name "$f" -exec codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime {} \;
 done

scripts/relup-base-vsns.escript

@@ -162,21 +162,26 @@ filter_froms(Froms0, AvailableVersionsIndex) ->
     Froms1 =
         case get_system() of
             %% we do not support relup for windows
-            "windows" ->
+            {"windows", _} ->
                 [];
             %% debian11 is introduced since v4.4.2 and e4.4.2
             %% exclude tags before them
-            "debian11" ->
+            {"debian11", _} ->
                 lists:filter(
                   fun(Vsn) ->
                           not lists:member(Vsn, [<<"4.4.0">>, <<"4.4.1">>])
                   end, Froms0);
             %% amzn2 is introduced since v4.4.12 and e4.4.12
             %% exclude tags before them
-            "amzn2" ->
+            {"amzn2", _} ->
                 Excluded = [list_to_binary(["4.4.", integer_to_list(X)]) || X <- lists:seq(0,11)],
                 lists:filter(fun(Vsn) -> not lists:member(Vsn, Excluded) end, Froms0);
-            _ ->
+            %% macos arm64 (M1/M2) packages are introduced since v4.4.12 and e4.4.12
+            %% exclude tags before them
+            {"macos" ++ _, "aarch64" ++ _} ->
+                Excluded = [list_to_binary(["4.4.", integer_to_list(X)]) || X <- lists:seq(0,11)],
+                lists:filter(fun(Vsn) -> not lists:member(Vsn, Excluded) end, Froms0);
+            {_, _} ->
                 Froms0
         end,
     lists:filter(
@@ -184,11 +189,12 @@ filter_froms(Froms0, AvailableVersionsIndex) ->
       Froms1).
 
 get_system() ->
+    Arch = erlang:system_info(system_architecture),
     case os:getenv("SYSTEM") of
         false ->
-            string:trim(os:cmd("./scripts/get-distro.sh"));
+            {string:trim(os:cmd("./scripts/get-distro.sh")), Arch};
         System ->
-            System
+            {System, Arch}
     end.
 
 %% assumes that's X.Y.Z, without pre-releases