Merge pull request #8684 from thalesmg/upgrade-builder-45

ci: upgrade builder image -> 4.5-1 (OTP 24.3.4.2-1)
This commit is contained in:
Thales Macedo Garitezi 2022-08-22 09:01:41 -03:00 committed by GitHub
commit 88cb641f7a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 110 additions and 53 deletions

View File

@ -3,7 +3,7 @@ version: '3.9'
services:
erlang:
container_name: erlang
image: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
image: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04
env_file:
- conf.env
environment:

View File

@ -15,8 +15,8 @@ PROFILE="$1"
VSN="$2"
OLD_VSN="$3"
PACKAGE_PATH="$4"
FROM_OTP_VSN="${5:-24.1.5-3}"
TO_OTP_VSN="${6:-24.1.5-3}"
FROM_OTP_VSN="${5:-24.3.4.2-1}"
TO_OTP_VSN="${6:-24.3.4.2-1}"
TEMPDIR=$(mktemp -d)
trap '{ rm -rf -- "$TEMPDIR"; }' EXIT

View File

@ -9,11 +9,11 @@ jobs:
strategy:
matrix:
erl_otp:
- 24.1.5-3
- 24.3.4.2-1
os:
- ubuntu20.04
container: ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.erl_otp }}-${{ matrix.os }}
container: ghcr.io/emqx/emqx-builder/4.5-1:${{ matrix.erl_otp }}-${{ matrix.os }}
steps:
- uses: actions/checkout@v2

View File

@ -19,7 +19,7 @@ jobs:
prepare:
runs-on: ubuntu-20.04
# prepare source with any OTP version, no need for a matrix
container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04
outputs:
profiles: ${{ steps.set_profile.outputs.profiles}}
@ -114,7 +114,7 @@ jobs:
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
otp:
- 24.1.5-3
- 24.3.4.2-1
macos:
- macos-11
exclude:
@ -203,7 +203,7 @@ jobs:
- zip
- pkg
otp:
- 24.1.5-3
- 24.3.4.2-1
arch:
- amd64
- arm64
@ -260,7 +260,7 @@ jobs:
--profile "${PROFILE}" \
--pkgtype "${PACKAGE}" \
--arch "${ARCH}" \
--builder "ghcr.io/emqx/emqx-builder/4.4-19:${OTP}-${SYSTEM}"
--builder "ghcr.io/emqx/emqx-builder/4.5-1:${OTP}-${SYSTEM}"
- uses: actions/upload-artifact@v1
with:
name: ${{ matrix.profile }}-${{ matrix.otp }}
@ -275,7 +275,7 @@ jobs:
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
otp:
- 24.1.5-3
- 24.3.4.2-1
registry:
- 'docker.io'
- 'public.ecr.aws'
@ -336,7 +336,7 @@ jobs:
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.otp }}-alpine3.15.1
BUILD_FROM=ghcr.io/emqx/emqx-builder/4.5-1:${{ matrix.otp }}-alpine3.15.1
RUN_FROM=alpine:3.15.1
EMQX_NAME=${{ matrix.profile }}
file: source/deploy/docker/Dockerfile
@ -352,7 +352,7 @@ jobs:
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.otp }}-alpine3.15.1
BUILD_FROM=ghcr.io/emqx/emqx-builder/4.5-1:${{ matrix.otp }}-alpine3.15.1
RUN_FROM=alpine:3.15.1
EMQX_NAME=${{ matrix.profile }}
file: source/deploy/docker/Dockerfile.enterprise
@ -370,7 +370,7 @@ jobs:
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
otp:
- 24.1.5-3
- 24.3.4.2-1
include:
- profile: emqx
otp: windows # otp version on windows is rather fixed

View File

@ -17,13 +17,12 @@ jobs:
fail-fast: false
matrix:
erl_otp:
- 23.3.4.9-3
- 24.1.5-3
- 24.3.4.2-1
os:
- ubuntu20.04
- el8
container: ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.erl_otp }}-${{ matrix.os }}
container: ghcr.io/emqx/emqx-builder/4.5-1:${{ matrix.erl_otp }}-${{ matrix.os }}
steps:
- uses: actions/checkout@v1
@ -69,7 +68,7 @@ jobs:
profile:
- emqx
otp:
- 24.2.1
- 24.3.4.2
steps:
- uses: actions/checkout@v2
- uses: ilammy/msvc-dev-cmd@v1
@ -101,7 +100,7 @@ jobs:
fail-fast: false
matrix:
otp:
- 24.1.5-3
- 24.3.4.2-1
macos:
- macos-11

View File

@ -5,7 +5,7 @@ on: [pull_request]
jobs:
check_deps_integrity:
runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04
steps:
- uses: actions/checkout@v2

View File

@ -7,7 +7,7 @@ on:
jobs:
prepare:
runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04
outputs:
profiles: ${{ steps.set_profile.outputs.profiles}}

View File

@ -5,7 +5,7 @@ on: workflow_dispatch
jobs:
test:
runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04
strategy:
fail-fast: true
env:

View File

@ -29,7 +29,7 @@ jobs:
- uses: actions/checkout@v2
- uses: erlef/setup-beam@v1
with:
otp-version: "24.1.5"
otp-version: "24.3.4.2"
- name: prepare
id: prepare
run: |

View File

@ -15,7 +15,7 @@ jobs:
- uses: actions/checkout@v1
- uses: erlef/setup-beam@v1
with:
otp-version: "24.1.5"
otp-version: "24.3.4.2"
- name: prepare
run: |
if make emqx-ee --dry-run > /dev/null 2>&1; then
@ -78,7 +78,7 @@ jobs:
- uses: actions/checkout@v1
- uses: erlef/setup-beam@v1
with:
otp-version: "24.1.5"
otp-version: "24.3.4.2"
- name: prepare
run: |
if make emqx-ee --dry-run > /dev/null 2>&1; then
@ -226,7 +226,7 @@ jobs:
relup_test_plan:
runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04
outputs:
profile: ${{ steps.profile-and-versions.outputs.profile }}
vsn: ${{ steps.profile-and-versions.outputs.vsn }}
@ -275,9 +275,9 @@ jobs:
fail-fast: false
matrix:
otp:
- 24.1.5-3
- 24.3.4.2-1
runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04
defaults:
run:
shell: bash
@ -312,13 +312,15 @@ jobs:
- relup_test_plan
- relup_test_build
runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04
strategy:
fail-fast: false
matrix:
old_vsn: ${{ fromJson(needs.relup_test_plan.outputs.matrix) }}
old_otp:
- 24.1.5-3
otp:
- 24.1.5-3
- 24.3.4.2-1
env:
OLD_VSN: "${{ matrix.old_vsn }}"
PROFILE: "${{ needs.relup_test_plan.outputs.profile }}"
@ -345,7 +347,7 @@ jobs:
mkdir -p packages
cp emqx_built/_packages/*/*.zip packages
cd packages
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-${OLD_VSN#[e|v]}-otp${{ matrix.otp }}-ubuntu20.04-amd64.zip
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-${OLD_VSN#[e|v]}-otp${{ matrix.old_otp }}-ubuntu20.04-amd64.zip
- name: Run relup test scenario
timeout-minutes: 5
run: |
@ -357,8 +359,8 @@ jobs:
--var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \
--var VSN="$VSN" \
--var OLD_VSN="$OLD_VSN" \
--var FROM_OTP_VSN="24.1.5-3" \
--var TO_OTP_VSN="24.1.5-3" \
--var FROM_OTP_VSN="${{ matrix.old_otp }}" \
--var TO_OTP_VSN="${{ matrix.otp }}" \
emqx_built/.ci/fvt_tests/relup.lux
- uses: actions/upload-artifact@v2
name: Save debug data

View File

@ -10,7 +10,7 @@ on:
jobs:
run_proper_test:
runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04
steps:
- uses: actions/checkout@v2

View File

@ -1 +1 @@
erlang 24.1.5-3
erlang 24.3.4.2-1

View File

@ -3,7 +3,7 @@ REBAR = $(CURDIR)/rebar3
BUILD = $(CURDIR)/build
SCRIPTS = $(CURDIR)/scripts
export EMQX_RELUP ?= true
export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-alpine3.15.1
export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-alpine3.15.1
export EMQX_DEFAULT_RUNNER = alpine:3.15.1
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)

View File

@ -19,7 +19,7 @@
-include("emqx_auth_ldap.hrl").
-include_lib("emqx/include/emqx.hrl").
-include_lib("eldap/include/eldap.hrl").
-include_lib("eldap2/include/eldap.hrl").
-include_lib("emqx/include/logger.hrl").
-export([ check_acl/5

View File

@ -1,6 +1,6 @@
{application, emqx_auth_ldap,
[{description, "EMQ X Authentication/ACL with LDAP"},
{vsn, "4.3.5"}, % strict semver, bump manually!
{vsn, "4.3.6"}, % strict semver, bump manually!
{modules, []},
{registered, [emqx_auth_ldap_sup]},
{applications, [kernel,stdlib,eldap2,ecpool]},

View File

@ -1,8 +1,12 @@
%% -*- mode: erlang -*-
%% Unless you know what you are doing, DO NOT edit manually!!
{VSN,
[{<<"4\\.3\\.[3-4]">>,
[{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
[{"4.3.5",
[{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]},
{<<"4\\.3\\.[3-4]">>,
[{load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]},
{"4.3.2",
[{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
@ -14,8 +18,12 @@
{load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}],
[{<<"4\\.3\\.[3-4]">>,
[{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
[{"4.3.5",
[{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]},
{<<"4\\.3\\.[3-4]">>,
[{load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]},
{"4.3.2",
[{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},

View File

@ -19,7 +19,7 @@
-include("emqx_auth_ldap.hrl").
-include_lib("emqx/include/emqx.hrl").
-include_lib("eldap/include/eldap.hrl").
-include_lib("eldap2/include/eldap.hrl").
-include_lib("emqx/include/logger.hrl").
-import(proplists, [get_value/2]).

23
build
View File

@ -71,7 +71,14 @@ make_relup() {
local lib_dir="_build/$PROFILE/rel/emqx/lib"
local releases_dir="_build/$PROFILE/rel/emqx/releases"
local name_pattern
name_pattern="${PROFILE}-$(./scripts/pkg-full-vsn.sh 'vsn_matcher')"
# 4.5.0 upgraded OTP from 24.1.5-3 to 24.3.4.2-1, so we need to
# fool the matcher to make it pick up the previous 4.4.X releases
# with older OTP.
if [[ "${PKG_VSN}" == 4.5.0* ]]; then
name_pattern="${PROFILE}-$(env OTP_VSN=24.1.5-3 ./scripts/pkg-full-vsn.sh 'vsn_matcher')"
else
name_pattern="${PROFILE}-$(./scripts/pkg-full-vsn.sh 'vsn_matcher')"
fi
mkdir -p "$lib_dir" "$releases_dir" '_upgrade_base'
local releases=()
if [ -d "$releases_dir" ]; then
@ -96,8 +103,8 @@ make_relup() {
fi
RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")"
export RELX_BASE_VERSIONS
if [[ ${PKG_VSN} == 4.3* ]]; then
echo "EMQX 4.3 specific, overwrite OTP app versions"
if [[ ${PKG_VSN} == 4.3* || ${PKG_VSN} == 4.5.* ]]; then
echo "EMQX 4.3/4.5 specific, overwrite OTP app versions"
local emqx_rel_file="${releases_dir}/${PKG_VSN}/emqx.rel"
if [ ! -f "${emqx_rel_file}" ]; then
./rebar3 as "${PROFILE}" release
@ -115,7 +122,7 @@ make_relup() {
# rollback rel file per releases
#
if [[ ${PKG_VSN} == 4.3* ]]; then
if [[ ${PKG_VSN} == 4.3* || ${PKG_VSN} == 4.5.* ]]; then
echo "restore upgrade base rel files... "
for rel in ${releases[*]};
do
@ -181,6 +188,12 @@ make_zip() {
esac
;;
esac
## one can only upgrade to 4.5.X from 4.4.8 or later, when that's
## released. Until that's released, there's no relup.
v44_bases="$(git tag -l "*4.4*" | grep -c -E '4\.4\.([8-9]|1[0-9])' || true)"
if [[ "${v44_bases}" -eq 0 ]]; then
has_relup='no'
fi
if [ "$has_relup" = 'yes' ]; then
./scripts/inject-relup.escript "${tard}/emqx/releases/${PKG_VSN}/relup"
fi
@ -232,7 +245,7 @@ make_docker() {
## Name Default Example
## ---------------------------------------------------------------------
## EMQX_BASE_IMAGE current os centos:7
## EMQX_ZIP_PACKAGE _packages/<current-zip-target> /tmp/emqx-4.4.0-otp24.1.5-3-el7-amd64.zip
## EMQX_ZIP_PACKAGE _packages/<current-zip-target> /tmp/emqx-4.4.0-otp24.3.4.2-1-el7-amd64.zip
## EMQX_IMAGE_TAG emqx/emqx:<current-vns-rel> emqx/emqx:testing-tag
##
make_docker_testing() {

View File

@ -1,4 +1,4 @@
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-alpine3.15.1
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-alpine3.15.1
ARG RUN_FROM=alpine:3.15.1
FROM ${BUILD_FROM} AS builder

View File

@ -29,7 +29,7 @@
-ifndef(EMQX_ENTERPRISE).
-define(EMQX_RELEASE, {opensource, "4.4.8"}).
-define(EMQX_RELEASE, {opensource, "4.5.0-alpha.1"}).
-else.

View File

@ -8,7 +8,7 @@
## i.e. will not work if docker command has to be executed with sudo
## example:
## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-debian10 --arch arm64
## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-debian10 --arch arm64
set -euo pipefail
@ -20,7 +20,7 @@ help() {
echo "--arch amd64|arm64: Target arch to build the EMQ X package for"
echo "--src_dir <SRC_DIR>: EMQ X source ode in this dir, default to PWD"
echo "--builder <BUILDER>: Builder image to pull"
echo " E.g. ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-debian10"
echo " E.g. ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-debian10"
}
while [ "$#" -gt 0 ]; do

View File

@ -16,6 +16,11 @@ case "${PKG_VSN}" in
EMQX_CE_DASHBOARD_VERSION='v4.4.4'
EMQX_EE_DASHBOARD_VERSION='v4.4.14'
;;
4.5*)
# keep the above 4.3 untouched, otherwise conflicts!
EMQX_CE_DASHBOARD_VERSION='v4.4.4'
EMQX_EE_DASHBOARD_VERSION='v4.4.14'
;;
*)
echo "Unsupported version $PKG_VSN" >&2
exit 1

View File

@ -18,7 +18,7 @@ case "${VSN_MATCH}" in
PKG_VSN='*'
;;
*)
echo "$0 ERROR: second arg must "
echo "$0 ERROR: arg must be either 'vsn_exact' or 'vsn_matcher'"
exit 1
;;
esac

View File

@ -63,8 +63,21 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
mkdir -p _upgrade_base
pushd _upgrade_base
# For 4.5+, we upgrade from OTP 24.1.5-3 to 24.3.4.2-1, so we must manually
# check the old OTP releases.
otp_vsn_for() {
case "${1#[e|v]}" in
4.4.*)
echo "24.1.5-3"
;;
4.5.*)
echo "$OTP_VSN"
;;
esac
}
for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do
filename="$PROFILE-${tag#[e|v]}-otp$OTP_VSN-$SYSTEM-$ARCH.zip"
filename="$PROFILE-${tag#[e|v]}-otp$(otp_vsn_for "$tag")-$SYSTEM-$ARCH.zip"
url="https://packages.emqx.io/$DIR/$tag/$filename"
if [ ! -f "$filename" ] && curl -L -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then
echo "downloading base package from ${url} ..."

View File

@ -61,6 +61,7 @@ esac
# must not be empty for MacOS (bash 3.x)
TAGS=( 'dummy' )
TAGS_EXCLUDE=( 'dummy' )
while read -r git_tag; do
# shellcheck disable=SC2207
semver=($(parse_semver "$git_tag"))
@ -88,6 +89,22 @@ for tag_to_del in "${TAGS_EXCLUDE[@]}"; do
TAGS=( "${TAGS[@]/$tag_to_del}" )
done
# 4.5.X versions uses the previous 4.4.Y as a base we emulate
# that we are the last 4.4.Y version that allows upgrading to 4.5.X
# We add that version, if available.
maybe_add_tag() {
local tag="$1"
if [[ $(git tag -l "$tag") ]]; then
TAGS+=( "$tag" )
fi
}
if [[ "${CUR_SEMVER[0]}" = 4 && "${CUR_SEMVER[1]}" = 5 ]]; then
for tag in "v4.4.8" "e4.4.8"; do
maybe_add_tag "$tag"
done
fi
for tag in "${TAGS[@]}"; do
if [ "$tag" != '' ]; then
echo "$tag"