From 39e5b31360541bc53951f7bd5129b8b5dfb42ea2 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 10 Aug 2022 10:23:58 -0300 Subject: [PATCH 01/10] ci: upgrade builder image -> 4.5-1 (OTP 24.3.4.2-1) --- .ci/docker-compose-file/docker-compose.yaml | 2 +- .ci/fvt_tests/local_relup_test_run.sh | 4 ++-- .github/workflows/apps_version_check.yaml | 4 ++-- .github/workflows/build_packages.yaml | 16 ++++++++-------- .github/workflows/build_slim_packages.yaml | 9 ++++----- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/release.yaml | 2 +- .github/workflows/run_acl_migration_tests.yaml | 2 +- .github/workflows/run_automate_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 18 +++++++++--------- .github/workflows/run_test_cases.yaml | 2 +- .tool-versions | 2 +- Makefile | 2 +- build | 2 +- deploy/docker/Dockerfile | 2 +- scripts/buildx.sh | 4 ++-- 16 files changed, 37 insertions(+), 38 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index ec505ddf4..1b981593a 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04 env_file: - conf.env environment: diff --git a/.ci/fvt_tests/local_relup_test_run.sh b/.ci/fvt_tests/local_relup_test_run.sh index 649af9587..0344823c9 100755 --- a/.ci/fvt_tests/local_relup_test_run.sh +++ b/.ci/fvt_tests/local_relup_test_run.sh @@ -15,8 +15,8 @@ PROFILE="$1" VSN="$2" OLD_VSN="$3" PACKAGE_PATH="$4" -FROM_OTP_VSN="${5:-24.1.5-3}" -TO_OTP_VSN="${6:-24.1.5-3}" +FROM_OTP_VSN="${5:-24.3.4.2-1}" +TO_OTP_VSN="${6:-24.3.4.2-1}" TEMPDIR=$(mktemp -d) trap '{ rm -rf -- "$TEMPDIR"; }' EXIT diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index d27c2b3d8..9cd508415 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -9,11 +9,11 @@ jobs: strategy: matrix: erl_otp: - - 24.1.5-3 + - 24.3.4.2-1 os: - ubuntu20.04 - container: ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.5-1:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 42e7a2a16..0af1fcf1b 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -19,7 +19,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} @@ -114,7 +114,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 24.1.5-3 + - 24.3.4.2-1 macos: - macos-11 exclude: @@ -203,7 +203,7 @@ jobs: - zip - pkg otp: - - 24.1.5-3 + - 24.3.4.2-1 arch: - amd64 - arm64 @@ -260,7 +260,7 @@ jobs: --profile "${PROFILE}" \ --pkgtype "${PACKAGE}" \ --arch "${ARCH}" \ - --builder "ghcr.io/emqx/emqx-builder/4.4-19:${OTP}-${SYSTEM}" + --builder "ghcr.io/emqx/emqx-builder/4.5-1:${OTP}-${SYSTEM}" - uses: actions/upload-artifact@v1 with: name: ${{ matrix.profile }}-${{ matrix.otp }} @@ -275,7 +275,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 24.1.5-3 + - 24.3.4.2-1 registry: - 'docker.io' - 'public.ecr.aws' @@ -336,7 +336,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.otp }}-alpine3.15.1 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.5-1:${{ matrix.otp }}-alpine3.15.1 RUN_FROM=alpine:3.15.1 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile @@ -352,7 +352,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.otp }}-alpine3.15.1 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.5-1:${{ matrix.otp }}-alpine3.15.1 RUN_FROM=alpine:3.15.1 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile.enterprise @@ -370,7 +370,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 24.1.5-3 + - 24.3.4.2-1 include: - profile: emqx otp: windows # otp version on windows is rather fixed diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 970fc981f..28ed8d65a 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -17,13 +17,12 @@ jobs: fail-fast: false matrix: erl_otp: - - 23.3.4.9-3 - - 24.1.5-3 + - 24.3.4.2-1 os: - ubuntu20.04 - el8 - container: ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.5-1:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 @@ -69,7 +68,7 @@ jobs: profile: - emqx otp: - - 24.2.1 + - 24.3.4.2 steps: - uses: actions/checkout@v2 - uses: ilammy/msvc-dev-cmd@v1 @@ -101,7 +100,7 @@ jobs: fail-fast: false matrix: otp: - - 24.1.5-3 + - 24.3.4.2-1 macos: - macos-11 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 433b00d71..4ff4665d8 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 560ce0ed1..1a3eacae8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,7 +7,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml index 6ebe37f98..ecb047f8b 100644 --- a/.github/workflows/run_acl_migration_tests.yaml +++ b/.github/workflows/run_acl_migration_tests.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04 strategy: fail-fast: true env: diff --git a/.github/workflows/run_automate_tests.yaml b/.github/workflows/run_automate_tests.yaml index a4473f62b..7bae9ff60 100644 --- a/.github/workflows/run_automate_tests.yaml +++ b/.github/workflows/run_automate_tests.yaml @@ -29,7 +29,7 @@ jobs: - uses: actions/checkout@v2 - uses: erlef/setup-beam@v1 with: - otp-version: "24.1.5" + otp-version: "24.3.4.2" - name: prepare id: prepare run: | diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 5779c5ca3..88eac8d45 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v1 - uses: erlef/setup-beam@v1 with: - otp-version: "24.1.5" + otp-version: "24.3.4.2" - name: prepare run: | if make emqx-ee --dry-run > /dev/null 2>&1; then @@ -78,7 +78,7 @@ jobs: - uses: actions/checkout@v1 - uses: erlef/setup-beam@v1 with: - otp-version: "24.1.5" + otp-version: "24.3.4.2" - name: prepare run: | if make emqx-ee --dry-run > /dev/null 2>&1; then @@ -226,7 +226,7 @@ jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04 outputs: profile: ${{ steps.profile-and-versions.outputs.profile }} vsn: ${{ steps.profile-and-versions.outputs.vsn }} @@ -275,9 +275,9 @@ jobs: fail-fast: false matrix: otp: - - 24.1.5-3 + - 24.3.4.2-1 runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04 defaults: run: shell: bash @@ -312,13 +312,13 @@ jobs: - relup_test_plan - relup_test_build runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04 strategy: fail-fast: false matrix: old_vsn: ${{ fromJson(needs.relup_test_plan.outputs.matrix) }} otp: - - 24.1.5-3 + - 24.3.4.2-1 env: OLD_VSN: "${{ matrix.old_vsn }}" PROFILE: "${{ needs.relup_test_plan.outputs.profile }}" @@ -357,8 +357,8 @@ jobs: --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \ --var VSN="$VSN" \ --var OLD_VSN="$OLD_VSN" \ - --var FROM_OTP_VSN="24.1.5-3" \ - --var TO_OTP_VSN="24.1.5-3" \ + --var FROM_OTP_VSN="24.3.4.2-1" \ + --var TO_OTP_VSN="24.3.4.2-1" \ emqx_built/.ci/fvt_tests/relup.lux - uses: actions/upload-artifact@v2 name: Save debug data diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 138b07993..1bd14c73d 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -10,7 +10,7 @@ on: jobs: run_proper_test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.tool-versions b/.tool-versions index a6568713b..9ff4a119c 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -erlang 24.1.5-3 +erlang 24.3.4.2-1 diff --git a/Makefile b/Makefile index 755ee8e88..d605ba180 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export EMQX_RELUP ?= true -export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-alpine3.15.1 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-alpine3.15.1 export EMQX_DEFAULT_RUNNER = alpine:3.15.1 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) diff --git a/build b/build index 979af4510..024b9267d 100755 --- a/build +++ b/build @@ -232,7 +232,7 @@ make_docker() { ## Name Default Example ## --------------------------------------------------------------------- ## EMQX_BASE_IMAGE current os centos:7 -## EMQX_ZIP_PACKAGE _packages/ /tmp/emqx-4.4.0-otp24.1.5-3-el7-amd64.zip +## EMQX_ZIP_PACKAGE _packages/ /tmp/emqx-4.4.0-otp24.3.4.2-1-el7-amd64.zip ## EMQX_IMAGE_TAG emqx/emqx: emqx/emqx:testing-tag ## make_docker_testing() { diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 8695b9d78..6eab6adc4 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-alpine3.15.1 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-alpine3.15.1 ARG RUN_FROM=alpine:3.15.1 FROM ${BUILD_FROM} AS builder diff --git a/scripts/buildx.sh b/scripts/buildx.sh index be1725976..80bc54b60 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -8,7 +8,7 @@ ## i.e. will not work if docker command has to be executed with sudo ## example: -## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-debian10 --arch arm64 +## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-debian10 --arch arm64 set -euo pipefail @@ -20,7 +20,7 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQ X package for" echo "--src_dir : EMQ X source ode in this dir, default to PWD" echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-debian10" + echo " E.g. ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-debian10" } while [ "$#" -gt 0 ]; do From 710887bde3c71974cd224aa2de890b33f32a10a0 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 10 Aug 2022 16:01:13 -0300 Subject: [PATCH 02/10] chore: bump release version -> 4.5.0 --- include/emqx_release.hrl | 2 +- scripts/get-dashboard.sh | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 261b38af0..439a6ac14 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.8"}). +-define(EMQX_RELEASE, {opensource, "4.5.0"}). -else. diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index b4ddc5679..846ff8e51 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -16,6 +16,11 @@ case "${PKG_VSN}" in EMQX_CE_DASHBOARD_VERSION='v4.4.4' EMQX_EE_DASHBOARD_VERSION='v4.4.14' ;; + 4.5*) + # keep the above 4.3 untouched, otherwise conflicts! + EMQX_CE_DASHBOARD_VERSION='v4.4.4' + EMQX_EE_DASHBOARD_VERSION='v4.4.14' + ;; *) echo "Unsupported version $PKG_VSN" >&2 exit 1 From 5d431fdd6476949bb171b9fd96353fe7f02b1322 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 10 Aug 2022 17:48:28 -0300 Subject: [PATCH 03/10] ci: adapt relup script for 4.5.0 --- build | 15 +++++++++++---- scripts/pkg-full-vsn.sh | 2 +- scripts/relup-base-packages.sh | 7 +++++++ scripts/relup-base-vsns.sh | 8 ++++++++ 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/build b/build index 024b9267d..056394bac 100755 --- a/build +++ b/build @@ -71,7 +71,14 @@ make_relup() { local lib_dir="_build/$PROFILE/rel/emqx/lib" local releases_dir="_build/$PROFILE/rel/emqx/releases" local name_pattern - name_pattern="${PROFILE}-$(./scripts/pkg-full-vsn.sh 'vsn_matcher')" + # 4.5.0 upgraded OTP from 24.1.5-3 to 24.3.4.2-1, so we need to + # fool the matcher to make it pick up the previous 4.4.X releases + # with older OTP. + if [[ "${PKG_VSN}" == 4.5.0* ]]; then + name_pattern="${PROFILE}-$(env OTP_VSN=24.1.5-3 ./scripts/pkg-full-vsn.sh 'vsn_matcher')" + else + name_pattern="${PROFILE}-$(./scripts/pkg-full-vsn.sh 'vsn_matcher')" + fi mkdir -p "$lib_dir" "$releases_dir" '_upgrade_base' local releases=() if [ -d "$releases_dir" ]; then @@ -96,8 +103,8 @@ make_relup() { fi RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")" export RELX_BASE_VERSIONS - if [[ ${PKG_VSN} == 4.3* ]]; then - echo "EMQX 4.3 specific, overwrite OTP app versions" + if [[ ${PKG_VSN} == 4.3* || ${PKG_VSN} == 4.5.0* ]]; then + echo "EMQX 4.3/4.5 specific, overwrite OTP app versions" local emqx_rel_file="${releases_dir}/${PKG_VSN}/emqx.rel" if [ ! -f "${emqx_rel_file}" ]; then ./rebar3 as "${PROFILE}" release @@ -115,7 +122,7 @@ make_relup() { # rollback rel file per releases # - if [[ ${PKG_VSN} == 4.3* ]]; then + if [[ ${PKG_VSN} == 4.3* || ${PKG_VSN} == 4.5.0* ]]; then echo "restore upgrade base rel files... " for rel in ${releases[*]}; do diff --git a/scripts/pkg-full-vsn.sh b/scripts/pkg-full-vsn.sh index e118643c9..752bb62e2 100755 --- a/scripts/pkg-full-vsn.sh +++ b/scripts/pkg-full-vsn.sh @@ -18,7 +18,7 @@ case "${VSN_MATCH}" in PKG_VSN='*' ;; *) - echo "$0 ERROR: second arg must " + echo "$0 ERROR: arg must be either 'vsn_exact' or 'vsn_matcher'" exit 1 ;; esac diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 9505a68f0..5ac10ede2 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -63,6 +63,13 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." mkdir -p _upgrade_base pushd _upgrade_base +# For 4.5, we upgrade from OTP 24.1.5-3 to 24.3.4.2-1, so we must manually +# check the old OTP releases. +CURRENT_VERSION="$(../pkg-vsn.sh)" +if [[ "${CURRENT_VERSION}" = 4.5.0* ]]; then + OTP_VSN=24.1.5-3 +fi + for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do filename="$PROFILE-${tag#[e|v]}-otp$OTP_VSN-$SYSTEM-$ARCH.zip" url="https://packages.emqx.io/$DIR/$tag/$filename" diff --git a/scripts/relup-base-vsns.sh b/scripts/relup-base-vsns.sh index 91c26823e..6b35fb196 100755 --- a/scripts/relup-base-vsns.sh +++ b/scripts/relup-base-vsns.sh @@ -61,6 +61,14 @@ esac # must not be empty for MacOS (bash 3.x) TAGS=( 'dummy' ) TAGS_EXCLUDE=( 'dummy' ) + +# first 4.5.0 version uses the previous 4.4.X as a base +# we emulate that we are the last 4.4.X version. +if [[ "${CUR_SEMVER[0]}" = 4 && "${CUR_SEMVER[1]}" = 5 && "${CUR_SEMVER[2]}" = 0 ]]; then + CUR_SEMVER[1]=4 + CUR_SEMVER[2]=9999 +fi + while read -r git_tag; do # shellcheck disable=SC2207 semver=($(parse_semver "$git_tag")) From 7a20d3216436d7abfb6846cf5b4cbdd0ef4a606a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 11 Aug 2022 13:09:24 -0300 Subject: [PATCH 04/10] fix: eldap lib is eldap2 --- apps/emqx_auth_ldap/src/emqx_acl_ldap.erl | 2 +- apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src | 2 +- apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src | 16 ++++++++++++---- apps/emqx_auth_ldap/src/emqx_auth_ldap.erl | 2 +- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/apps/emqx_auth_ldap/src/emqx_acl_ldap.erl b/apps/emqx_auth_ldap/src/emqx_acl_ldap.erl index 5a249c860..a8f08f2b4 100644 --- a/apps/emqx_auth_ldap/src/emqx_acl_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_acl_ldap.erl @@ -19,7 +19,7 @@ -include("emqx_auth_ldap.hrl"). -include_lib("emqx/include/emqx.hrl"). --include_lib("eldap/include/eldap.hrl"). +-include_lib("eldap2/include/eldap.hrl"). -include_lib("emqx/include/logger.hrl"). -export([ check_acl/5 diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src index 6875fca30..48dc39bc9 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_ldap, [{description, "EMQ X Authentication/ACL with LDAP"}, - {vsn, "4.3.5"}, % strict semver, bump manually! + {vsn, "4.3.6"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_ldap_sup]}, {applications, [kernel,stdlib,eldap2,ecpool]}, diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src index fbb59a176..d7d3b6fb2 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src @@ -1,8 +1,12 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{<<"4\\.3\\.[3-4]">>, - [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + [{"4.3.5", + [{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[3-4]">>, + [{load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]}, {"4.3.2", [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, @@ -14,8 +18,12 @@ {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{<<"4\\.3\\.[3-4]">>, - [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + [{"4.3.5", + [{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[3-4]">>, + [{load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]}, {"4.3.2", [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl b/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl index da932c2fe..dbc0e8732 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl @@ -19,7 +19,7 @@ -include("emqx_auth_ldap.hrl"). -include_lib("emqx/include/emqx.hrl"). --include_lib("eldap/include/eldap.hrl"). +-include_lib("eldap2/include/eldap.hrl"). -include_lib("emqx/include/logger.hrl"). -import(proplists, [get_value/2]). From 71bb0436e65136656129518e45f9b2175e4d20aa Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 11 Aug 2022 14:15:19 -0300 Subject: [PATCH 05/10] ci: fix old otp vsn for relup test --- .github/workflows/run_fvt_tests.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 88eac8d45..1ef6f1188 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -317,8 +317,10 @@ jobs: fail-fast: false matrix: old_vsn: ${{ fromJson(needs.relup_test_plan.outputs.matrix) }} + old_otp: + - 24.1.5-3 otp: - - 24.3.4.2-1 + - 24.3.4.2-1 env: OLD_VSN: "${{ matrix.old_vsn }}" PROFILE: "${{ needs.relup_test_plan.outputs.profile }}" @@ -345,7 +347,7 @@ jobs: mkdir -p packages cp emqx_built/_packages/*/*.zip packages cd packages - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-${OLD_VSN#[e|v]}-otp${{ matrix.otp }}-ubuntu20.04-amd64.zip + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-${OLD_VSN#[e|v]}-otp${{ matrix.old_otp }}-ubuntu20.04-amd64.zip - name: Run relup test scenario timeout-minutes: 5 run: | @@ -357,8 +359,8 @@ jobs: --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \ --var VSN="$VSN" \ --var OLD_VSN="$OLD_VSN" \ - --var FROM_OTP_VSN="24.3.4.2-1" \ - --var TO_OTP_VSN="24.3.4.2-1" \ + --var FROM_OTP_VSN="${{ matrix.old_otp }}" \ + --var TO_OTP_VSN="${{ matrix.otp }}" \ emqx_built/.ci/fvt_tests/relup.lux - uses: actions/upload-artifact@v2 name: Save debug data From 396e44ce299881d133414218da2e41604e2e3b7e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 15 Aug 2022 09:35:21 -0300 Subject: [PATCH 06/10] ci: fix relup base vsns to account for future 4.5.X versions --- scripts/relup-base-vsns.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/relup-base-vsns.sh b/scripts/relup-base-vsns.sh index 6b35fb196..8919f9cdf 100755 --- a/scripts/relup-base-vsns.sh +++ b/scripts/relup-base-vsns.sh @@ -62,13 +62,6 @@ esac TAGS=( 'dummy' ) TAGS_EXCLUDE=( 'dummy' ) -# first 4.5.0 version uses the previous 4.4.X as a base -# we emulate that we are the last 4.4.X version. -if [[ "${CUR_SEMVER[0]}" = 4 && "${CUR_SEMVER[1]}" = 5 && "${CUR_SEMVER[2]}" = 0 ]]; then - CUR_SEMVER[1]=4 - CUR_SEMVER[2]=9999 -fi - while read -r git_tag; do # shellcheck disable=SC2207 semver=($(parse_semver "$git_tag")) @@ -96,6 +89,13 @@ for tag_to_del in "${TAGS_EXCLUDE[@]}"; do TAGS=( "${TAGS[@]/$tag_to_del}" ) done +# first 4.5.0 version uses the previous 4.4.X as a base we emulate +# that we are the last 4.4.X version that allows upgrading to 4.4.5. +if [[ "${CUR_SEMVER[0]}" = 4 && "${CUR_SEMVER[1]}" = 5 && "${CUR_SEMVER[2]}" = 0 ]]; then + TAGS+=( "v4.4.8" ) + TAGS+=( "e4.4.8" ) +fi + for tag in "${TAGS[@]}"; do if [ "$tag" != '' ]; then echo "$tag" From 5c720a01a3858792ecfbc5b207d0a170b19041e6 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 16 Aug 2022 16:27:16 -0300 Subject: [PATCH 07/10] chore: skip relup while {e,v}4.4.8 does not exist --- build | 5 +++++ scripts/relup-base-vsns.sh | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/build b/build index 056394bac..890a9a1ff 100755 --- a/build +++ b/build @@ -188,6 +188,11 @@ make_zip() { esac ;; esac + ## one can only upgrade to 4.5.X from 4.4.8, when that's released. + ## Until that's released, there's no relup. + if [[ -z "$(ls -A _upgrade_base 2>/dev/null || true)" && "${PKG_VSN}" == 4.5.0* ]]; then + has_relup='no' + fi if [ "$has_relup" = 'yes' ]; then ./scripts/inject-relup.escript "${tard}/emqx/releases/${PKG_VSN}/relup" fi diff --git a/scripts/relup-base-vsns.sh b/scripts/relup-base-vsns.sh index 8919f9cdf..724f17d1a 100755 --- a/scripts/relup-base-vsns.sh +++ b/scripts/relup-base-vsns.sh @@ -91,9 +91,10 @@ done # first 4.5.0 version uses the previous 4.4.X as a base we emulate # that we are the last 4.4.X version that allows upgrading to 4.4.5. +# We add that version, if available. if [[ "${CUR_SEMVER[0]}" = 4 && "${CUR_SEMVER[1]}" = 5 && "${CUR_SEMVER[2]}" = 0 ]]; then - TAGS+=( "v4.4.8" ) - TAGS+=( "e4.4.8" ) + [[ $(git tag -l "v4.4.8") ]] && TAGS+=( "v4.4.8" ) + [[ $(git tag -l "e4.4.8") ]] && TAGS+=( "e4.4.8" ) fi for tag in "${TAGS[@]}"; do From 257a86a57667f160f7463b4b5635023ed093a6f9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 17 Aug 2022 09:17:44 -0300 Subject: [PATCH 08/10] chore: prepare for 4.5+ and consider 4.4.9+ --- build | 4 ++-- include/emqx_release.hrl | 2 +- scripts/relup-base-packages.sh | 18 ++++++++++++------ scripts/relup-base-vsns.sh | 18 +++++++++++++----- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/build b/build index 890a9a1ff..e2c68bce1 100755 --- a/build +++ b/build @@ -188,8 +188,8 @@ make_zip() { esac ;; esac - ## one can only upgrade to 4.5.X from 4.4.8, when that's released. - ## Until that's released, there's no relup. + ## one can only upgrade to 4.5.X from 4.4.8 or later, when that's + ## released. Until that's released, there's no relup. if [[ -z "$(ls -A _upgrade_base 2>/dev/null || true)" && "${PKG_VSN}" == 4.5.0* ]]; then has_relup='no' fi diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 439a6ac14..eed5fc0b2 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.5.0"}). +-define(EMQX_RELEASE, {opensource, "4.5.0-alpha.1"}). -else. diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 5ac10ede2..f201a2dba 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -63,15 +63,21 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." mkdir -p _upgrade_base pushd _upgrade_base -# For 4.5, we upgrade from OTP 24.1.5-3 to 24.3.4.2-1, so we must manually +# For 4.5+, we upgrade from OTP 24.1.5-3 to 24.3.4.2-1, so we must manually # check the old OTP releases. -CURRENT_VERSION="$(../pkg-vsn.sh)" -if [[ "${CURRENT_VERSION}" = 4.5.0* ]]; then - OTP_VSN=24.1.5-3 -fi +otp_vsn_for() { + case "${1#[e|v]}" in + 4.4.*) + echo "24.1.5-3" + ;; + 4.5.*) + echo "$OTP_VSN" + ;; + esac +} for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do - filename="$PROFILE-${tag#[e|v]}-otp$OTP_VSN-$SYSTEM-$ARCH.zip" + filename="$PROFILE-${tag#[e|v]}-otp$(otp_vsn_for "$tag")-$SYSTEM-$ARCH.zip" url="https://packages.emqx.io/$DIR/$tag/$filename" if [ ! -f "$filename" ] && curl -L -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then echo "downloading base package from ${url} ..." diff --git a/scripts/relup-base-vsns.sh b/scripts/relup-base-vsns.sh index 724f17d1a..d8e9bf380 100755 --- a/scripts/relup-base-vsns.sh +++ b/scripts/relup-base-vsns.sh @@ -89,12 +89,20 @@ for tag_to_del in "${TAGS_EXCLUDE[@]}"; do TAGS=( "${TAGS[@]/$tag_to_del}" ) done -# first 4.5.0 version uses the previous 4.4.X as a base we emulate -# that we are the last 4.4.X version that allows upgrading to 4.4.5. +# 4.5.X versions uses the previous 4.4.Y as a base we emulate +# that we are the last 4.4.Y version that allows upgrading to 4.5.X # We add that version, if available. -if [[ "${CUR_SEMVER[0]}" = 4 && "${CUR_SEMVER[1]}" = 5 && "${CUR_SEMVER[2]}" = 0 ]]; then - [[ $(git tag -l "v4.4.8") ]] && TAGS+=( "v4.4.8" ) - [[ $(git tag -l "e4.4.8") ]] && TAGS+=( "e4.4.8" ) +maybe_add_tag() { + local tag="$1" + if [[ $(git tag -l "$tag") ]]; then + TAGS+=( "$tag" ) + fi +} + +if [[ "${CUR_SEMVER[0]}" = 4 && "${CUR_SEMVER[1]}" = 5 ]]; then + for tag in "v4.4.8" "e4.4.8"; do + maybe_add_tag "$tag" + done fi for tag in "${TAGS[@]}"; do From ca9628342ad55b37d86508987975f312204ace9e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 18 Aug 2022 10:22:58 -0300 Subject: [PATCH 09/10] refactor: check existing tags for relup Co-authored-by: Zaiming (Stone) Shi --- build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build b/build index e2c68bce1..c24bed785 100755 --- a/build +++ b/build @@ -190,7 +190,8 @@ make_zip() { esac ## one can only upgrade to 4.5.X from 4.4.8 or later, when that's ## released. Until that's released, there's no relup. - if [[ -z "$(ls -A _upgrade_base 2>/dev/null || true)" && "${PKG_VSN}" == 4.5.0* ]]; then + v44_bases="$(git tag -l "*4.4*" | grep -c -E '4\.4\.([8-9]|1[0-9])' || true)" + if [[ "${v44_bases}" -eq 0 ]]; then has_relup='no' fi if [ "$has_relup" = 'yes' ]; then From 2bfb61385f35e4fc1144f87d03094f842f915c42 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 17 Aug 2022 09:17:44 -0300 Subject: [PATCH 10/10] chore: prepare for 4.5+ and consider 4.4.9+ --- build | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build b/build index c24bed785..c48a66fdc 100755 --- a/build +++ b/build @@ -103,7 +103,7 @@ make_relup() { fi RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")" export RELX_BASE_VERSIONS - if [[ ${PKG_VSN} == 4.3* || ${PKG_VSN} == 4.5.0* ]]; then + if [[ ${PKG_VSN} == 4.3* || ${PKG_VSN} == 4.5.* ]]; then echo "EMQX 4.3/4.5 specific, overwrite OTP app versions" local emqx_rel_file="${releases_dir}/${PKG_VSN}/emqx.rel" if [ ! -f "${emqx_rel_file}" ]; then @@ -122,7 +122,7 @@ make_relup() { # rollback rel file per releases # - if [[ ${PKG_VSN} == 4.3* || ${PKG_VSN} == 4.5.0* ]]; then + if [[ ${PKG_VSN} == 4.3* || ${PKG_VSN} == 4.5.* ]]; then echo "restore upgrade base rel files... " for rel in ${releases[*]}; do