From 7ef804059727af8a3698802fe3636f59697cb1e8 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 30 Dec 2021 13:37:10 +0100 Subject: [PATCH] build: ci tests.sh test specific package --- .ci/build_packages/tests.sh | 50 ++++++++++++++++++---- .github/workflows/build_packages.yaml | 12 +++--- .github/workflows/build_slim_packages.yaml | 8 +++- scripts/buildx.sh | 29 ++++++++----- 4 files changed, 70 insertions(+), 29 deletions(-) diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index e7169b8d9..4c82b3d9e 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -1,5 +1,23 @@ -#!/bin/bash +#!/usr/bin/env bash + +## This script tests built package start/stop +## Accept 2 args PACKAGE_NAME and PACKAGE_TYPE + set -x -e -u + +if [ -z "${1:-}" ]; then + echo "Usage $0 zip|pkg" + exit 1 +fi + +if [ "${2:-}" != 'zip' ] && [ "${2:-}" != 'pkg' ]; then + echo "Usage $0 zip|pkg" + exit 1 +fi + +PACKAGE_NAME="${1}" +PACKAGE_TYPE="${2}" + export CODE_PATH=${CODE_PATH:-"/emqx"} export EMQX_NAME=${EMQX_NAME:-"emqx"} export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}" @@ -7,6 +25,26 @@ export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" # export EMQX_NODE_NAME="emqx-on-$(uname -m)@127.0.0.1" # export EMQX_NODE_COOKIE=$(date +%s%N) +if [ "$PACKAGE_TYPE" = 'zip' ]; then + PKG_SUFFIX="zip" +else + SYSTEM="$($CODE_PATH/scripts/get-distro.sh)" + case "${SYSTEM:-}" in + ubuntu*|debian*|raspbian*) + PKG_SUFFIX='deb' + ;; + *) + PKG_SUFFIX='rpm' + ;; + esac +fi +PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}" + +PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}.${PKG_SUFFIX}" +if ! [ -f "$PACKAGE_FILE" ]; then + echo "$PACKAGE_FILE is not a file" +fi + case "$(uname -m)" in x86_64) ARCH='amd64' @@ -22,7 +60,6 @@ export ARCH emqx_prepare(){ mkdir -p "${PACKAGE_PATH}" - if [ ! -d "/paho-mqtt-testing" ]; then git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho-mqtt-testing fi @@ -31,11 +68,10 @@ emqx_prepare(){ emqx_test(){ cd "${PACKAGE_PATH}" + local packagename="${PACKAGE_FILE_NAME}" - for var in "$PACKAGE_PATH"/"${EMQX_NAME}"-*;do - case ${var##*.} in + case ${packagename##*.} in "zip") - packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.zip) unzip -q "${PACKAGE_PATH}/${packagename}" export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \ EMQX_MQTT__MAX_TOPIC_ALIAS=10 @@ -64,7 +100,6 @@ emqx_test(){ rm -rf "${PACKAGE_PATH}"/emqx ;; "deb") - packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.deb) dpkg -i "${PACKAGE_PATH}/${packagename}" if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "ii" ] then @@ -91,8 +126,6 @@ emqx_test(){ fi ;; "rpm") - packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.rpm) - if [[ "${ARCH}" == "amd64" && $(rpm -E '%{rhel}') == 7 ]] ; then # EMQX OTP requires openssl11 to have TLS1.3 support yum install -y openssl11 @@ -116,7 +149,6 @@ emqx_test(){ ;; esac - done } running_test(){ diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index e1f8a43e2..696aedd15 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -318,13 +318,11 @@ jobs: SYSTEM: ${{ matrix.os }} working-directory: source run: | - docker run -i --rm \ - -v $(pwd):/emqx \ - --workdir /emqx \ - --platform linux/$ARCH \ - ghcr.io/emqx/emqx-builder/4.4-4:$OTP-$SYSTEM \ - bash -euc "make ${PROFILE}-${PACKAGE} || cat rebar3.crashdump; \ - EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh" + ./scripts/buildx.sh \ + --profile "${PROFILE}" \ + --pkgtype "${PACKAGE}" \ + --arch "${ARCH}" \ + --builder "ghcr.io/emqx/emqx-builder/4.4-4:${OTP}-${SYSTEM}" - name: create sha256 working-directory: source env: diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 90a3db420..fbac56b5f 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -52,8 +52,12 @@ jobs: path: ./rebar3.crashdump - name: packages test run: | - export CODE_PATH=$GITHUB_WORKSPACE - .ci/build_packages/tests.sh + set -euo pipefail + PKG_VSN="$(./pkg-vsn.sh)" + PKG_NAME="${EMQX_NAME}-${PKG_VSN}-otp{{ matrix.erl_otp }}-{{matrix.os }}-amd64" + export CODE_PATH="$GITHUB_WORKSPACE" + .ci/build_packages/tests.sh "$PKG_NAME" zip + .ci/build_packages/tests.sh "$PKG_NAME" pkg - uses: actions/upload-artifact@v2 with: name: ${{ matrix.os }} diff --git a/scripts/buildx.sh b/scripts/buildx.sh index c0406d5ce..57d3a93d6 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -14,19 +14,19 @@ set -euo pipefail help() { echo - echo "-h|--help: To display this usage information" - echo "--pkgtype zip|pkg: Specify to build zip or deb|rpm package" - echo "--arch amd64|arm64: Target arch to build the EMQ X package for." - echo "--src_dir : EMQ X source ode in this dir, default to PWD" - echo "--profile : EMQ X profile to build, e.g. emqx, emqx-edge" - echo "--builder : Builder image to pull." - echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10" + echo "-h|--help: To display this usage information" + echo "--profile : EMQ X profile to build, e.g. emqx, emqx-edge" + echo "--pkgtype zip|pkg: Specify which package to build, zip for .zip and pkg for .rpm or .deb" + echo "--arch amd64|arm64: Target arch to build the EMQ X package for" + echo "--src_dir : EMQ X source ode in this dir, default to PWD" + echo "--builder : Builder image to pull" + echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10" } while [ "$#" -gt 0 ]; do case $1 in - -h|--help) - help + -h|--help) + help exit 0 ;; --src_dir) @@ -62,14 +62,21 @@ if [ -z "${PROFILE:-}" ] || [ -z "${PKGTYPE:-}" ] || [ -z "${BUILDER:-}" ] || [ exit 1 fi -docker run --rm --privileged tonistiigi/binfmt:latest --install ${ARCH} +if [ "$PKGTYPE" != 'zip' ] && [ "$PKGTYPE" != 'pkg' ]; then + echo "Bad --pkgtype option, should be zip or pkg" + exit 1 +fi cd "${SRC_DIR:-.}" +PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" +OTP_VSN_SYSTEM=$(echo $BUILDER | cut -d ':' -f2) +PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN_SYSTEM}-${ARCH}" + docker run -i --rm \ -v "$(pwd)":/emqx \ --workdir /emqx \ --platform="linux/$ARCH" \ -e EMQX_NAME="$PROFILE" \ "$BUILDER" \ - bash -euc "make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh" + bash -euc "make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PKG_NAME $PKGTYPE"