yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into merge-5.0-beta.3-to-master

This commit is contained in:
Zaiming (Stone) Shi 2022-01-04 21:11:30 +01:00
parent 05ccedd8ec 5df382f054
commit eae2e3ad0c
8 changed files with 161 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
.ci/build_packages/tests.sh
View File

@ -1,11 +1,47 @@
#!/bin/bash
set -x -e -u
#!/usr/bin/env bash
set -euo pipefail
set -x
if [ -z "${1:-}" ]; then
echo "Usage $0 <PACKAGE_NAME> zip|pkg"
exit 1
fi
if [ "${2:-}" != 'zip' ] && [ "${2:-}" != 'pkg' ]; then
echo "Usage $0 <PACKAGE_NAME> zip|pkg"
exit 1
fi
PACKAGE_NAME="${1}"
PACKAGE_TYPE="${2}"
export DEBUG=1
export CODE_PATH=${CODE_PATH:-"/emqx"}
export EMQX_NAME=${EMQX_NAME:-"emqx"}
export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}"
export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base"
if [ "$PACKAGE_TYPE" = 'zip' ]; then
PKG_SUFFIX="zip"
else
SYSTEM="$("$CODE_PATH"/scripts/get-distro.sh)"
case "${SYSTEM:-}" in
ubuntu*|debian*|raspbian*)
PKG_SUFFIX='deb'
;;
*)
PKG_SUFFIX='rpm'
;;
esac
fi
PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}"
PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}.${PKG_SUFFIX}"
if ! [ -f "$PACKAGE_FILE" ]; then
echo "$PACKAGE_FILE is not a file"
fi
case "$(uname -m)" in
x86_64)
ARCH='amd64'
@ -30,11 +66,9 @@ emqx_prepare(){
emqx_test(){
cd "${PACKAGE_PATH}"
for var in "$PACKAGE_PATH"/"${EMQX_NAME}"-*;do
case ${var##*.} in
local packagename="${PACKAGE_FILE_NAME}"
case "$PKG_SUFFIX" in
"zip")
packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.zip)
unzip -q "${PACKAGE_PATH}/${packagename}"
export EMQX_ZONES__DEFAULT__MQTT__SERVER_KEEPALIVE=60
export EMQX_MQTT__MAX_TOPIC_ALIAS=10
@ -42,7 +76,6 @@ emqx_test(){
export EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL=debug
if [[ $(arch) == *arm* || $(arch) == aarch64 ]]; then
export EMQX_LISTENERS__QUIC__DEFAULT__ENABLED=false
export WAIT_FOR_ERLANG_STOP=120
fi
# sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins
@ -72,7 +105,6 @@ emqx_test(){
rm -rf "${PACKAGE_PATH}"/emqx
;;
"deb")
packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.deb)
dpkg -i "${PACKAGE_PATH}/${packagename}"
if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "ii" ]
then
@ -99,15 +131,8 @@ emqx_test(){
fi
;;
"rpm")
packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.rpm)
if [[ "${ARCH}" == "amd64" && $(rpm -E '%{rhel}') == 7 ]] ;
then
# EMQX OTP requires openssl11 to have TLS1.3 support
yum install -y openssl11;
fi
rpm -ivh "${PACKAGE_PATH}/${packagename}"
if ! rpm -q emqx | grep -q emqx; then
yum install -y "${PACKAGE_PATH}/${packagename}"
if ! rpm -q "${EMQX_NAME}" | grep -q "${EMQX_NAME}"; then
echo "package install error"
exit 1
fi
@ -124,7 +149,6 @@ emqx_test(){
;;
esac
done
}
run_test(){
@ -142,7 +166,6 @@ EOF
## for ARM, due to CI env issue, skip start of quic listener for the moment
[[ $(arch) == *arm* || $(arch) == aarch64 ]] && tee -a "$emqx_env_vars" <<EOF
export EMQX_LISTENERS__QUIC__DEFAULT__ENABLED=false
export WAIT_FOR_ERLANG_STOP=120
EOF
else
echo "Error: cannot locate emqx_vars"

20
.github/workflows/build_packages.yaml vendored
View File

@ -297,14 +297,18 @@ jobs:
SYSTEM: ${{ matrix.os }}
working-directory: source
run: |
docker run -i --rm \
-v $(pwd):/emqx \
--workdir /emqx \
--platform linux/$ARCH \
ghcr.io/emqx/emqx-builder/5.0-3:$OTP-$SYSTEM \
bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \
make $PROFILE-pkg || cat rebar3.crashdump; \
EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh"
./scripts/buildx.sh \
--profile "${PROFILE}" \
--pkgtype "zip" \
--arch "${ARCH}" \
--builder "ghcr.io/emqx/emqx-builder/5.0-3:${OTP}-${SYSTEM}"
## the pkg build is incremental on the zip build
./scripts/buildx.sh \
--profile "${PROFILE}" \
--pkgtype "pkg" \
--arch "${ARCH}" \
--builder "ghcr.io/emqx/emqx-builder/5.0-3:${OTP}-${SYSTEM}"
- name: create sha256
env:
PROFILE: ${{ matrix.profile}}

20
.github/workflows/build_slim_packages.yaml vendored
View File

@ -43,19 +43,23 @@ jobs:
steps:
- uses: actions/checkout@v1
- name: build zip package
run: make ${{ matrix.profile }}-zip
- name: build deb/rpm packages
run: make ${{ matrix.profile }}-pkg
- name: packages test
- name: prepare
run: |
export CODE_PATH=$GITHUB_WORKSPACE
EMQX_NAME=${{ matrix.profile }} .ci/build_packages/tests.sh
echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
echo "EMQX_PKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh)-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV
- name: build and test zip package
run: |
make ${EMQX_NAME}-zip
.ci/build_packages/tests.sh "$EMQX_PKG_NAME" zip
- name: build and test deb/rpm packages
run: |
make ${EMQX_NAME}-pkg
.ci/build_packages/tests.sh "$EMQX_PKG_NAME" pkg
- uses: actions/upload-artifact@v2
with:
name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }}
path: _packages/**/*.zip
mac:
if: contains(github.event.pull_request.labels.*.name, 'Mac')
strategy:

5
.github/workflows/elixir_deps_check.yaml vendored
View File

@ -16,6 +16,11 @@ jobs:
uses: actions/checkout@v2.4.0
- name: ensure rebar
run: ./scripts/ensure-rebar3.sh 3.16.1-emqx-1
- name: setup mix
run: |
mix local.hex --force
mix local.rebar --force
mix deps.get
- name: check elixir deps
run: ./scripts/check-elixir-deps-discrepancies.exs

13
bin/emqx
View File

@ -601,16 +601,15 @@ case "${COMMAND}" in
"$BINDIR/run_erl" -daemon "$PIPE_DIR" "$RUNNER_LOG_DIR" \
"$(relx_start_command)"
WAIT_TIME=${WAIT_FOR_ERLANG:-15}
WAIT_TIME=${EMQX_WAIT_FOR_START:-120}
if wait_until_return_val "true" "$WAIT_TIME" 'relx_nodetool' \
'eval' 'emqx:is_running()'; then
echo "$EMQX_DESCRIPTION $REL_VSN is started successfully!"
exit 0
else
echo "$EMQX_DESCRIPTION $REL_VSN failed to start within ${WAIT_TIME} seconds,"
echo "see the output of '$0 console' for more information."
echo "If you want to wait longer, set the environment variable"
echo "WAIT_FOR_ERLANG to the number of seconds to wait."
echo "$EMQX_DESCRIPTION $REL_VSN failed to start in ${WAIT_TIME} seconds."
echo "Please find more information in erlang.log.N"
echo "Or run 'DEBUG=1 $0 console' to have logs printed to console."
exit 1
fi
;;
@ -622,13 +621,13 @@ case "${COMMAND}" in
echoerr "Graceful shutdown failed PID=[$PID]"
exit 1
fi
WAIT_TIME="${WAIT_FOR_ERLANG_STOP:-60}"
WAIT_TIME="${EMQX_WAIT_FOR_STOP:-120}"
if ! wait_for "$WAIT_TIME" 'is_down' "$PID"; then
msg="dangling after ${WAIT_TIME} seconds"
# also log to syslog
logger -t "${REL_NAME}[${PID}]" "STOP: $msg"
# log to user console
echoerr "stop failed, $msg"
echoerr "Stop failed, $msg"
echo "ERROR: $PID is still around"
ps -p "$PID"
exit 1

84
scripts/buildx.sh Executable file
View File

@ -0,0 +1,84 @@
#!/usr/bin/env bash
## This script helps to run docker buildx to build cross-arch/platform packages (linux only)
## It mounts (not copy) host directory to a cross-arch/platform builder container
## Make sure the source dir (specified by --src_dir option) is clean before running this script
## NOTE: it requires $USER in docker group
## i.e. will not work if docker command has to be executed with sudo
## example:
## ./scripts/buildx.sh --profile emqx --pkgtype zip --arch arm64 --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10
set -euo pipefail
help() {
echo
echo "-h|--help: To display this usage information"
echo "--profile <PROFILE>: EMQ X profile to build, e.g. emqx, emqx-edge"
echo "--pkgtype zip|pkg: Specify which package to build, zip for .zip and pkg for .rpm or .deb"
echo "--arch amd64|arm64: Target arch to build the EMQ X package for"
echo "--src_dir <SRC_DIR>: EMQ X source ode in this dir, default to PWD"
echo "--builder <BUILDER>: Builder image to pull"
echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10"
}
while [ "$#" -gt 0 ]; do
case $1 in
-h|--help)
help
exit 0
;;
--src_dir)
SRC_DIR="$2"
shift 2
;;
--profile)
PROFILE="$2"
shift 2
;;
--pkgtype)
PKGTYPE="$2"
shift 2
;;
--builder)
BUILDER="$2"
shift 2
;;
--arch)
ARCH="$2"
shift 2
;;
*)
echo "WARN: Unknown arg (ignored): $1"
shift
continue
;;
esac
done
if [ -z "${PROFILE:-}" ] || [ -z "${PKGTYPE:-}" ] || [ -z "${BUILDER:-}" ] || [ -z "${ARCH:-}" ]; then
help
exit 1
fi
if [ "$PKGTYPE" != 'zip' ] && [ "$PKGTYPE" != 'pkg' ]; then
echo "Bad --pkgtype option, should be zip or pkg"
exit 1
fi
cd "${SRC_DIR:-.}"
PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
OTP_VSN_SYSTEM=$(echo "$BUILDER" | cut -d ':' -f2)
PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN_SYSTEM}-${ARCH}"
docker info
docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}"
docker run -i --rm \
-v "$(pwd)":/emqx \
--workdir /emqx \
--platform="linux/$ARCH" \
-e EMQX_NAME="$PROFILE" \
"$BUILDER" \
bash -euc "make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PKG_NAME $PKGTYPE"

3
scripts/check-elixir-deps-discrepancies.exs
View File

@ -1,8 +1,5 @@
#!/usr/bin/env elixir
# ensure we have a mix.lock
{_, 0} = System.cmd("mix", ["deps.get"], into: IO.stream())
# ensure we have a fresh rebar.lock
case File.stat("rebar.lock") do

2
scripts/start-two-nodes-in-docker.sh
View File

@ -37,7 +37,6 @@ docker run -d -t --restart=always --name "$NODE1" \
--net "$NET" \
-e EMQX_NODE_NAME="emqx@$NODE1" \
-e EMQX_NODE_COOKIE="$COOKIE" \
-e WAIT_FOR_ERLANG=60 \
-p 18083:18083 \
-v "$PROJ_DIR"/_build/emqx/rel/emqx:/built \
"$IMAGE" sh -c 'cp -r /built /emqx && /emqx/bin/emqx console'
@ -46,7 +45,6 @@ docker run -d -t --restart=always --name "$NODE2" \
--net "$NET" \
-e EMQX_NODE_NAME="emqx@$NODE2" \
-e EMQX_NODE_COOKIE="$COOKIE" \
-e WAIT_FOR_ERLANG=60 \
-p 18084:18083 \
-v "$PROJ_DIR"/_build/emqx/rel/emqx:/built \
"$IMAGE" sh -c 'cp -r /built /emqx && /emqx/bin/emqx console'