From 30b1c5f26d5d9f8e58a36e6b4bfd0a9eb514fa97 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 19 Dec 2022 09:49:28 +0100 Subject: [PATCH 1/2] chore: fail build if apple notarytool failed --- .github/workflows/build_slim_packages.yaml | 2 +- build | 16 ++++++++++++---- scripts/macos-sign-binaries.sh | 22 ++++++++++++++++++---- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 2e7b658cf..2793d9074 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -124,7 +124,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: emqx/self-hosted-cleanup-action@v1.0.3 - if: matrix.os == macos-12-arm64 + if: ${{ matrix.os == 'macos-12-arm64' }} - uses: actions/checkout@v3 with: fetch-depth: 0 # clone full git history diff --git a/build b/build index f0adb79d0..e740ff3b3 100755 --- a/build +++ b/build @@ -236,10 +236,18 @@ make_zip() { # --apple-id \ # --password # --team-id - xcrun notarytool submit \ - --apple-id "${APPLE_ID}" \ - --password "${APPLE_ID_PASSWORD}" \ - --team-id "${APPLE_TEAM_ID}" "${target_zip}" --wait + echo 'Submitting the package for notarization to Apple (normally takes about a minute)' + notarytool_output="$(xcrun notarytool submit \ + --apple-id "${APPLE_ID}" \ + --password "${APPLE_ID_PASSWORD}" \ + --team-id "${APPLE_TEAM_ID}" "${target_zip}" \ + --no-progress \ + --wait)" + echo "$notarytool_output" + echo "$notarytool_output" | grep -q 'status: Accepted' || { + echo 'Notarization failed'; + exit 1; + } fi # sha256sum may not be available on macos openssl dgst -sha256 "${target_zip}" | cut -d ' ' -f 2 > "${target_zip}.sha256" diff --git a/scripts/macos-sign-binaries.sh b/scripts/macos-sign-binaries.sh index 384744b2a..135730694 100755 --- a/scripts/macos-sign-binaries.sh +++ b/scripts/macos-sign-binaries.sh @@ -43,10 +43,24 @@ done security -v list-keychains -s "${keychain_names[@]}" "${KEYCHAIN}" # known runtime executables and binaries -codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/erts-*/bin/{beam.smp,dyn_erl,epmd,erl,erl_call,erl_child_setup,erlexec,escript,heart,inet_gethost,run_erl,to_erl} -codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/runtime_tools-*/priv/lib/{dyntrace.so,trace_ip_drv.so,trace_file_drv.so} -codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/os_mon-*/priv/bin/{cpu_sup,memsup} +codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \ + "${REL_DIR}"/erts-*/bin/{beam.smp,dyn_erl,epmd,erl,erl_call,erl_child_setup,erlexec,escript,heart,inet_gethost,run_erl,to_erl} +codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \ + "${REL_DIR}"/lib/runtime_tools-*/priv/lib/{dyntrace.so,trace_ip_drv.so,trace_file_drv.so} +codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \ + "${REL_DIR}"/lib/os_mon-*/priv/bin/{cpu_sup,memsup} # other files from runtime and dependencies -for f in asn1rt_nif.so bcrypt_nif.so crypto.so otp_test_engine.so crypto_callback.so jiffy.so crc32cer_nif.so sasl_auth.so snappyer.so odbcserver; do +for f in \ + asn1rt_nif.so \ + bcrypt_nif.so \ + crypto.so \ + otp_test_engine.so \ + crypto_callback.so \ + jiffy.so \ + crc32cer_nif.so \ + sasl_auth.so \ + snappyer.so \ + odbcserver \ + ; do find "${REL_DIR}"/lib/ -name "$f" -exec codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime {} \; done From c776afd86a2d876aff5d0bfcc1cd14f34feecaa3 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 19 Dec 2022 09:57:03 +0100 Subject: [PATCH 2/2] ci: backport patch for otp setup on self hosted macos from master --- .github/actions/package-macos/action.yaml | 32 ++++++++++++++++------ .github/workflows/build_slim_packages.yaml | 2 +- build | 10 +++++++ scripts/relup-base-vsns.escript | 18 ++++++++---- 4 files changed, 46 insertions(+), 16 deletions(-) diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml index 0624eb525..18afbd495 100644 --- a/.github/actions/package-macos/action.yaml +++ b/.github/actions/package-macos/action.yaml @@ -23,27 +23,38 @@ inputs: runs: using: composite steps: - - name: prepare + - id: prepare shell: bash run: | brew update - brew install curl zip unzip gnu-sed coreutils unixodbc freetds openssl@1.1 + brew install curl zip unzip gnu-sed coreutils autoconf automake cmake unixodbc freetds openssl@1.1 echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH echo "/usr/local/bin" >> $GITHUB_PATH + OTP_SOURCE_PATH="$HOME/src/otp-${{ inputs.otp }}" + OTP_INSTALL_PATH="$HOME/otp/${{ inputs.otp }}" + echo "OTP_SOURCE_PATH=$OTP_SOURCE_PATH" >> $GITHUB_OUTPUT + echo "OTP_INSTALL_PATH=$OTP_INSTALL_PATH" >> $GITHUB_OUTPUT + mkdir -p "$OTP_INSTALL_PATH" - uses: actions/cache@v3 id: cache with: - path: /opt/erlang/${{ inputs.otp }} + path: ${{ steps.prepare.outputs.OTP_INSTALL_PATH }} key: otp-install-${{ inputs.otp }}-${{ inputs.os }}-static-ssl-disable-hipe-disable-jit - name: build erlang if: steps.cache.outputs.cache-hit != 'true' shell: bash run: | - git clone --depth 1 --branch OTP-${{ inputs.otp }} https://github.com/emqx/otp.git $HOME/otp-${{ inputs.otp }} - cd $HOME/otp-${{ inputs.otp }} - ./configure --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit --prefix=/opt/erlang/${{ inputs.otp }} + OTP_SOURCE_PATH="${{ steps.prepare.outputs.OTP_SOURCE_PATH }}" + OTP_INSTALL_PATH="${{ steps.prepare.outputs.OTP_INSTALL_PATH }}" + if [ -d "$OTP_SOURCE_PATH" ]; then + rm -rf "$OTP_SOURCE_PATH" + fi + git clone --depth 1 --branch OTP-${{ inputs.otp }} https://github.com/emqx/otp.git "$OTP_SOURCE_PATH" + cd "$OTP_SOURCE_PATH" + ./configure --disable-dynamic-ssl-lib --with-ssl=$(brew --prefix openssl@1.1) --disable-hipe --disable-jit --prefix="$OTP_INSTALL_PATH" make -j$(nproc) - sudo make install + rm -rf "$OTP_INSTALL_PATH" + make install - name: build env: AUTO_INSTALL_BUILD_DEPS: 1 @@ -56,13 +67,16 @@ runs: APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ inputs.apple_developer_id_bundle_password }} shell: bash run: | - export PATH="/opt/erlang/${{ inputs.otp }}/bin:$PATH" + export PATH="${{ steps.prepare.outputs.OTP_INSTALL_PATH }}/bin:$PATH" make ensure-rebar3 - sudo cp rebar3 /usr/local/bin/rebar3 + mkdir -p $HOME/bin + cp rebar3 $HOME/bin/rebar3 + export PATH="$HOME/bin:$PATH" make ${EMQX_NAME}-zip - name: test shell: bash run: | + export PATH="${{ steps.prepare.outputs.OTP_INSTALL_PATH }}/bin:$PATH" pkg_name=$(basename _packages/${EMQX_NAME}/${EMQX_NAME}-*.zip) unzip -q _packages/${EMQX_NAME}/$pkg_name # test with a spaces in path diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 2793d9074..205066a07 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -149,5 +149,5 @@ jobs: path: ./rebar3.crashdump - uses: actions/upload-artifact@v3 with: - name: macos + name: ${{ matrix.os }} path: _packages/**/*.zip diff --git a/build b/build index e740ff3b3..d3c9164eb 100755 --- a/build +++ b/build @@ -203,6 +203,16 @@ make_zip() { ;; esac ;; + macos*) + case "$PKG_VSN" in + 4.4.12*) + # this is the first version for macos arm (M1/M2), no relup + if [ "$ARCH" = arm64 ]; then + has_relup='no' + fi + ;; + esac + ;; esac # shellcheck disable=SC2207 bases=($(relup_db base-vsns "$PKG_VSN")) diff --git a/scripts/relup-base-vsns.escript b/scripts/relup-base-vsns.escript index ccbf94168..e99c88a9f 100755 --- a/scripts/relup-base-vsns.escript +++ b/scripts/relup-base-vsns.escript @@ -162,21 +162,26 @@ filter_froms(Froms0, AvailableVersionsIndex) -> Froms1 = case get_system() of %% we do not support relup for windows - "windows" -> + {"windows", _} -> []; %% debian11 is introduced since v4.4.2 and e4.4.2 %% exclude tags before them - "debian11" -> + {"debian11", _} -> lists:filter( fun(Vsn) -> not lists:member(Vsn, [<<"4.4.0">>, <<"4.4.1">>]) end, Froms0); %% amzn2 is introduced since v4.4.12 and e4.4.12 %% exclude tags before them - "amzn2" -> + {"amzn2", _} -> Excluded = [list_to_binary(["4.4.", integer_to_list(X)]) || X <- lists:seq(0,11)], lists:filter(fun(Vsn) -> not lists:member(Vsn, Excluded) end, Froms0); - _ -> + %% macos arm64 (M1/M2) packages are introduced since v4.4.12 and e4.4.12 + %% exclude tags before them + {"macos" ++ _, "aarch64" ++ _} -> + Excluded = [list_to_binary(["4.4.", integer_to_list(X)]) || X <- lists:seq(0,11)], + lists:filter(fun(Vsn) -> not lists:member(Vsn, Excluded) end, Froms0); + {_, _} -> Froms0 end, lists:filter( @@ -184,11 +189,12 @@ filter_froms(Froms0, AvailableVersionsIndex) -> Froms1). get_system() -> + Arch = erlang:system_info(system_architecture), case os:getenv("SYSTEM") of false -> - string:trim(os:cmd("./scripts/get-distro.sh")); + {string:trim(os:cmd("./scripts/get-distro.sh")), Arch}; System -> - System + {System, Arch} end. %% assumes that's X.Y.Z, without pre-releases