Merge pull request #8684 from thalesmg/upgrade-builder-45

ci: upgrade builder image -> 4.5-1 (OTP 24.3.4.2-1)

.ci/docker-compose-file/docker-compose.yaml

@@ -3,7 +3,7 @@ version: '3.9'
 services:
   erlang:
     container_name: erlang
-    image: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
+    image: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04
     env_file:
       - conf.env
     environment:

.ci/fvt_tests/local_relup_test_run.sh

@@ -15,8 +15,8 @@ PROFILE="$1"
 VSN="$2"
 OLD_VSN="$3"
 PACKAGE_PATH="$4"
-FROM_OTP_VSN="${5:-24.1.5-3}"
+FROM_OTP_VSN="${5:-24.3.4.2-1}"
-TO_OTP_VSN="${6:-24.1.5-3}"
+TO_OTP_VSN="${6:-24.3.4.2-1}"
 
 TEMPDIR=$(mktemp -d)
 trap '{ rm -rf -- "$TEMPDIR"; }' EXIT

.github/workflows/apps_version_check.yaml

@@ -9,11 +9,11 @@ jobs:
     strategy:
       matrix:
         erl_otp:
-        - 24.1.5-3
+        - 24.3.4.2-1
         os:
         - ubuntu20.04
 
-    container: ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.erl_otp }}-${{ matrix.os }}
+    container: ghcr.io/emqx/emqx-builder/4.5-1:${{ matrix.erl_otp }}-${{ matrix.os }}
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/build_packages.yaml

@@ -19,7 +19,7 @@ jobs:
   prepare:
     runs-on: ubuntu-20.04
     # prepare source with any OTP version, no need for a matrix
-    container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04
 
     outputs:
       profiles: ${{ steps.set_profile.outputs.profiles}}
@@ -114,7 +114,7 @@ jobs:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         otp:
-          - 24.1.5-3
+          - 24.3.4.2-1
         macos:
           - macos-11
         exclude:
@@ -203,7 +203,7 @@ jobs:
           - zip
           - pkg
         otp:
-          - 24.1.5-3
+          - 24.3.4.2-1
         arch:
           - amd64
           - arm64
@@ -260,7 +260,7 @@ jobs:
           --profile "${PROFILE}" \
           --pkgtype "${PACKAGE}" \
           --arch "${ARCH}" \
-          --builder "ghcr.io/emqx/emqx-builder/4.4-19:${OTP}-${SYSTEM}"
+          --builder "ghcr.io/emqx/emqx-builder/4.5-1:${OTP}-${SYSTEM}"
     - uses: actions/upload-artifact@v1
       with:
         name: ${{ matrix.profile }}-${{ matrix.otp }}
@@ -275,7 +275,7 @@ jobs:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         otp:
-          - 24.1.5-3
+          - 24.3.4.2-1
         registry:
           - 'docker.io'
           - 'public.ecr.aws'
@@ -336,7 +336,7 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
-          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.otp }}-alpine3.15.1
+          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.5-1:${{ matrix.otp }}-alpine3.15.1
           RUN_FROM=alpine:3.15.1
           EMQX_NAME=${{ matrix.profile }}
         file: source/deploy/docker/Dockerfile
@@ -352,7 +352,7 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
-          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.otp }}-alpine3.15.1
+          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.5-1:${{ matrix.otp }}-alpine3.15.1
           RUN_FROM=alpine:3.15.1
           EMQX_NAME=${{ matrix.profile }}
         file: source/deploy/docker/Dockerfile.enterprise
@@ -370,7 +370,7 @@ jobs:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         otp:
-          - 24.1.5-3
+          - 24.3.4.2-1
         include:
           - profile: emqx
             otp: windows # otp version on windows is rather fixed

.github/workflows/build_slim_packages.yaml

@@ -17,13 +17,12 @@ jobs:
       fail-fast: false
       matrix:
         erl_otp:
-        - 23.3.4.9-3
+        - 24.3.4.2-1
-        - 24.1.5-3
         os:
         - ubuntu20.04
         - el8
 
-    container: ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.erl_otp }}-${{ matrix.os }}
+    container: ghcr.io/emqx/emqx-builder/4.5-1:${{ matrix.erl_otp }}-${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v1
@@ -69,7 +68,7 @@ jobs:
         profile:
           - emqx
         otp:
-          - 24.2.1
+          - 24.3.4.2
     steps:
     - uses: actions/checkout@v2
     - uses: ilammy/msvc-dev-cmd@v1
@@ -101,7 +100,7 @@ jobs:
       fail-fast: false
       matrix:
         otp:
-        - 24.1.5-3
+        - 24.3.4.2-1
         macos:
         - macos-11
 

.github/workflows/check_deps_integrity.yaml

@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   check_deps_integrity:
     runs-on: ubuntu-20.04
-    container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/release.yaml

@@ -7,7 +7,7 @@ on:
 jobs:
   prepare:
     runs-on: ubuntu-20.04
-    container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04
 
     outputs:
       profiles: ${{ steps.set_profile.outputs.profiles}}

.github/workflows/run_acl_migration_tests.yaml

@@ -5,7 +5,7 @@ on: workflow_dispatch
 jobs:
     test:
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04
         strategy:
             fail-fast: true
         env:

.github/workflows/run_automate_tests.yaml

@@ -29,7 +29,7 @@ jobs:
     - uses: actions/checkout@v2
     - uses: erlef/setup-beam@v1
       with:
-        otp-version: "24.1.5"
+        otp-version: "24.3.4.2"
     - name: prepare
       id: prepare
       run: |

.github/workflows/run_fvt_tests.yaml

@@ -15,7 +15,7 @@ jobs:
         - uses: actions/checkout@v1
         - uses: erlef/setup-beam@v1
           with:
-            otp-version: "24.1.5"
+            otp-version: "24.3.4.2"
         - name: prepare
           run: |
             if make emqx-ee --dry-run > /dev/null 2>&1; then
@@ -78,7 +78,7 @@ jobs:
         - uses: actions/checkout@v1
         - uses: erlef/setup-beam@v1
           with:
-            otp-version: "24.1.5"
+            otp-version: "24.3.4.2"
         - name: prepare
           run: |
             if make emqx-ee --dry-run > /dev/null 2>&1; then
@@ -226,7 +226,7 @@ jobs:
 
     relup_test_plan:
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04
         outputs:
           profile: ${{ steps.profile-and-versions.outputs.profile }}
           vsn: ${{ steps.profile-and-versions.outputs.vsn }}
@@ -275,9 +275,9 @@ jobs:
            fail-fast: false
            matrix:
              otp:
-             - 24.1.5-3
+             - 24.3.4.2-1
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04
         defaults:
           run:
             shell: bash
@@ -312,13 +312,15 @@ jobs:
           - relup_test_plan
           - relup_test_build
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04
         strategy:
           fail-fast: false
           matrix:
             old_vsn: ${{ fromJson(needs.relup_test_plan.outputs.matrix) }}
+            old_otp:
+              - 24.1.5-3
             otp:
-            - 24.1.5-3
+              - 24.3.4.2-1
         env:
           OLD_VSN: "${{ matrix.old_vsn }}"
           PROFILE: "${{ needs.relup_test_plan.outputs.profile }}"
@@ -345,7 +347,7 @@ jobs:
             mkdir -p packages
             cp emqx_built/_packages/*/*.zip packages
             cd packages
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-${OLD_VSN#[e|v]}-otp${{ matrix.otp }}-ubuntu20.04-amd64.zip
+            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-${OLD_VSN#[e|v]}-otp${{ matrix.old_otp }}-ubuntu20.04-amd64.zip
         - name: Run relup test scenario
           timeout-minutes: 5
           run: |
@@ -357,8 +359,8 @@ jobs:
             --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \
             --var VSN="$VSN" \
             --var OLD_VSN="$OLD_VSN" \
-            --var FROM_OTP_VSN="24.1.5-3" \
+            --var FROM_OTP_VSN="${{ matrix.old_otp }}" \
-            --var TO_OTP_VSN="24.1.5-3" \
+            --var TO_OTP_VSN="${{ matrix.otp }}" \
             emqx_built/.ci/fvt_tests/relup.lux
         - uses: actions/upload-artifact@v2
           name: Save debug data

.github/workflows/run_test_cases.yaml

@@ -10,7 +10,7 @@ on:
 jobs:
     run_proper_test:
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-ubuntu20.04
 
         steps:
         - uses: actions/checkout@v2

.tool-versions

@@ -1 +1 @@
-erlang 24.1.5-3
+erlang 24.3.4.2-1

Makefile

@@ -3,7 +3,7 @@ REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
 export EMQX_RELUP ?= true
-export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-alpine3.15.1
+export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-alpine3.15.1
 export EMQX_DEFAULT_RUNNER = alpine:3.15.1
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)

apps/emqx_auth_ldap/src/emqx_acl_ldap.erl

@@ -19,7 +19,7 @@
 -include("emqx_auth_ldap.hrl").
 
 -include_lib("emqx/include/emqx.hrl").
--include_lib("eldap/include/eldap.hrl").
+-include_lib("eldap2/include/eldap.hrl").
 -include_lib("emqx/include/logger.hrl").
 
 -export([ check_acl/5

apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_ldap,
  [{description, "EMQ X Authentication/ACL with LDAP"},
-  {vsn, "4.3.5"}, % strict semver, bump manually!
+  {vsn, "4.3.6"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_ldap_sup]},
   {applications, [kernel,stdlib,eldap2,ecpool]},

apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src

@@ -1,8 +1,12 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{<<"4\\.3\\.[3-4]">>,
+  [{"4.3.5",
-    [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[3-4]">>,
+    [{load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]},
    {"4.3.2",
     [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
@@ -14,8 +18,12 @@
      {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{<<"4\\.3\\.[3-4]">>,
+  [{"4.3.5",
-    [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[3-4]">>,
+    [{load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]},
    {"4.3.2",
     [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},

apps/emqx_auth_ldap/src/emqx_auth_ldap.erl

@@ -19,7 +19,7 @@
 -include("emqx_auth_ldap.hrl").
 
 -include_lib("emqx/include/emqx.hrl").
--include_lib("eldap/include/eldap.hrl").
+-include_lib("eldap2/include/eldap.hrl").
 -include_lib("emqx/include/logger.hrl").
 
 -import(proplists, [get_value/2]).

build

@@ -71,7 +71,14 @@ make_relup() {
     local lib_dir="_build/$PROFILE/rel/emqx/lib"
     local releases_dir="_build/$PROFILE/rel/emqx/releases"
     local name_pattern
-    name_pattern="${PROFILE}-$(./scripts/pkg-full-vsn.sh 'vsn_matcher')"
+    # 4.5.0 upgraded OTP from 24.1.5-3 to 24.3.4.2-1, so we need to
+    # fool the matcher to make it pick up the previous 4.4.X releases
+    # with older OTP.
+    if [[ "${PKG_VSN}" == 4.5.0* ]]; then
+      name_pattern="${PROFILE}-$(env OTP_VSN=24.1.5-3 ./scripts/pkg-full-vsn.sh 'vsn_matcher')"
+    else
+      name_pattern="${PROFILE}-$(./scripts/pkg-full-vsn.sh 'vsn_matcher')"
+    fi
     mkdir -p "$lib_dir" "$releases_dir" '_upgrade_base'
     local releases=()
     if [ -d "$releases_dir" ]; then
@@ -96,8 +103,8 @@ make_relup() {
     fi
     RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")"
     export RELX_BASE_VERSIONS
-    if [[ ${PKG_VSN} == 4.3* ]]; then
+    if [[ ${PKG_VSN} == 4.3* || ${PKG_VSN} == 4.5.* ]]; then
-        echo "EMQX 4.3 specific, overwrite OTP app versions"
+        echo "EMQX 4.3/4.5 specific, overwrite OTP app versions"
         local emqx_rel_file="${releases_dir}/${PKG_VSN}/emqx.rel"
         if [ ! -f "${emqx_rel_file}" ]; then
             ./rebar3 as "${PROFILE}" release
@@ -115,7 +122,7 @@ make_relup() {
 
     # rollback rel file per releases
     #
-    if [[ ${PKG_VSN} == 4.3* ]]; then
+    if [[ ${PKG_VSN} == 4.3* || ${PKG_VSN} == 4.5.* ]]; then
         echo "restore upgrade base rel files... "
         for rel in ${releases[*]};
         do
@@ -181,6 +188,12 @@ make_zip() {
             esac
             ;;
     esac
+    ## one can only upgrade to 4.5.X from 4.4.8 or later, when that's
+    ## released.  Until that's released, there's no relup.
+    v44_bases="$(git tag -l "*4.4*" | grep -c -E '4\.4\.([8-9]|1[0-9])' || true)"
+    if [[ "${v44_bases}" -eq 0 ]]; then
+      has_relup='no'
+    fi
     if [ "$has_relup" = 'yes' ]; then
         ./scripts/inject-relup.escript "${tard}/emqx/releases/${PKG_VSN}/relup"
     fi
@@ -232,7 +245,7 @@ make_docker() {
 ## Name               Default                         Example
 ## ---------------------------------------------------------------------
 ## EMQX_BASE_IMAGE    current os                      centos:7
-## EMQX_ZIP_PACKAGE   _packages/<current-zip-target>  /tmp/emqx-4.4.0-otp24.1.5-3-el7-amd64.zip
+## EMQX_ZIP_PACKAGE   _packages/<current-zip-target>  /tmp/emqx-4.4.0-otp24.3.4.2-1-el7-amd64.zip
 ## EMQX_IMAGE_TAG     emqx/emqx:<current-vns-rel>     emqx/emqx:testing-tag
 ##
 make_docker_testing() {

deploy/docker/Dockerfile

@@ -1,4 +1,4 @@
-ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-alpine3.15.1
+ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-alpine3.15.1
 ARG RUN_FROM=alpine:3.15.1
 FROM ${BUILD_FROM} AS builder
 

include/emqx_release.hrl

@@ -29,7 +29,7 @@
 
 -ifndef(EMQX_ENTERPRISE).
 
--define(EMQX_RELEASE, {opensource, "4.4.8"}).
+-define(EMQX_RELEASE, {opensource, "4.5.0-alpha.1"}).
 
 -else.
 

scripts/buildx.sh

@@ -8,7 +8,7 @@
 ## i.e. will not work if docker command has to be executed with sudo
 
 ## example:
-## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-debian10 --arch arm64
+## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-debian10 --arch arm64
 
 set -euo pipefail
 
@@ -20,7 +20,7 @@ help() {
     echo "--arch amd64|arm64:  Target arch to build the EMQ X package for"
     echo "--src_dir <SRC_DIR>: EMQ X source ode in this dir, default to PWD"
     echo "--builder <BUILDER>: Builder image to pull"
-    echo "                     E.g. ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-debian10"
+    echo "                     E.g. ghcr.io/emqx/emqx-builder/4.5-1:24.3.4.2-1-debian10"
 }
 
 while [ "$#" -gt 0 ]; do

scripts/get-dashboard.sh

@@ -16,6 +16,11 @@ case "${PKG_VSN}" in
         EMQX_CE_DASHBOARD_VERSION='v4.4.4'
         EMQX_EE_DASHBOARD_VERSION='v4.4.14'
         ;;
+    4.5*)
+        # keep the above 4.3 untouched, otherwise conflicts!
+        EMQX_CE_DASHBOARD_VERSION='v4.4.4'
+        EMQX_EE_DASHBOARD_VERSION='v4.4.14'
+        ;;
     *)
         echo "Unsupported version $PKG_VSN" >&2
         exit 1

scripts/pkg-full-vsn.sh

@@ -18,7 +18,7 @@ case "${VSN_MATCH}" in
         PKG_VSN='*'
         ;;
     *)
-        echo "$0 ERROR: second arg must "
+        echo "$0 ERROR: arg must be either 'vsn_exact' or 'vsn_matcher'"
         exit 1
         ;;
 esac

scripts/relup-base-packages.sh

@@ -63,8 +63,21 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
 mkdir -p _upgrade_base
 pushd _upgrade_base
 
+# For 4.5+, we upgrade from OTP 24.1.5-3 to 24.3.4.2-1, so we must manually
+# check the old OTP releases.
+otp_vsn_for() {
+  case "${1#[e|v]}" in
+    4.4.*)
+      echo "24.1.5-3"
+      ;;
+    4.5.*)
+      echo "$OTP_VSN"
+      ;;
+  esac
+}
+
 for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do
-    filename="$PROFILE-${tag#[e|v]}-otp$OTP_VSN-$SYSTEM-$ARCH.zip"
+    filename="$PROFILE-${tag#[e|v]}-otp$(otp_vsn_for "$tag")-$SYSTEM-$ARCH.zip"
     url="https://packages.emqx.io/$DIR/$tag/$filename"
     if [ ! -f "$filename" ] && curl -L -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then
         echo "downloading base package from ${url} ..."

scripts/relup-base-vsns.sh

@@ -61,6 +61,7 @@ esac
 # must not be empty for MacOS (bash 3.x)
 TAGS=( 'dummy' )
 TAGS_EXCLUDE=( 'dummy' )
+
 while read -r git_tag; do
     # shellcheck disable=SC2207
     semver=($(parse_semver "$git_tag"))
@@ -88,6 +89,22 @@ for tag_to_del in "${TAGS_EXCLUDE[@]}"; do
     TAGS=( "${TAGS[@]/$tag_to_del}" )
 done
 
+# 4.5.X versions uses the previous 4.4.Y as a base we emulate
+# that we are the last 4.4.Y version that allows upgrading to 4.5.X
+# We add that version, if available.
+maybe_add_tag() {
+  local tag="$1"
+  if [[ $(git tag -l "$tag") ]]; then
+    TAGS+=( "$tag" )
+  fi
+}
+
+if [[ "${CUR_SEMVER[0]}" = 4 && "${CUR_SEMVER[1]}" = 5 ]]; then
+  for tag in "v4.4.8" "e4.4.8"; do
+    maybe_add_tag "$tag"
+  done
+fi
+
 for tag in "${TAGS[@]}"; do
     if [ "$tag" != '' ]; then
         echo "$tag"