Merge pull request #11289 from id/0713-build-debian12-packages

.github/workflows/build_and_push_docker_images.yaml

@@ -120,7 +120,7 @@ jobs:
         # NOTE: 'otp' and 'elixir' are to configure emqx-builder image
         #       only support latest otp and elixir, not a matrix
         builder:
-          - 5.1-1 # update to latest
+          - 5.1-3 # update to latest
         otp:
           - 25.3.2-1
         elixir:

.github/workflows/build_packages.yaml

@@ -181,6 +181,7 @@ jobs:
           - ubuntu22.04
           - ubuntu20.04
           - ubuntu18.04
+          - debian12
           - debian11
           - debian10
           - el9
@@ -190,16 +191,16 @@ jobs:
           - amzn2023
         build_machine:
           - aws-arm64
-          - ubuntu-22.04
+          - aws-amd64
         builder:
-          - 5.1-1
+          - 5.1-3
         elixir:
           - 1.14.5
         with_elixir:
           - 'no'
         exclude:
           - arch: arm64
-            build_machine: ubuntu-22.04
+            build_machine: aws-amd64
           - arch: amd64
             build_machine: aws-arm64
         include:
@@ -207,16 +208,8 @@ jobs:
             otp: 25.3.2-1
             arch: amd64
             os: ubuntu22.04
-            build_machine: ubuntu-22.04
+            build_machine: aws-amd64
-            builder: 5.1-1
+            builder: 5.1-3
-            elixir: 1.14.5
-            with_elixir: 'yes'
-          - profile: emqx
-            otp: 25.3.2-1
-            arch: amd64
-            os: amzn2
-            build_machine: ubuntu-22.04
-            builder: 5.1-1
             elixir: 1.14.5
             with_elixir: 'yes'
 
@@ -226,18 +219,13 @@ jobs:
 
     steps:
     - uses: AutoModality/action-clean@v1
-      if: matrix.build_machine == 'aws-arm64'
 
     - uses: actions/checkout@v3
       with:
         ref: ${{ github.event.inputs.branch_or_tag }}
         fetch-depth: 0
 
-    - name: build emqx packages
+    - name: fix workdir
-      env:
-        ELIXIR: ${{ matrix.elixir }}
-        PROFILE: ${{ matrix.profile }}
-        ARCH: ${{ matrix.arch }}
       run: |
         set -eu
         git config --global --add safe.directory "$GITHUB_WORKSPACE"
@@ -247,22 +235,33 @@ jobs:
           cd /emqx
         fi
         echo "pwd is $PWD"
-        PKGTYPES="tgz pkg"
+
-        IS_ELIXIR=${{ matrix.with_elixir }}
+    - name: build emqx packages
+      env:
+        PROFILE: ${{ matrix.profile }}
+        IS_ELIXIR: ${{ matrix.with_elixir }}
+        ACLOCAL_PATH: "/usr/share/aclocal:/usr/local/share/aclocal"
+      run: |
+        set -eu
         if [ "${IS_ELIXIR:-}" == 'yes' ]; then
-          PKGTYPES="tgz"
+          make "${PROFILE}-elixir-tgz"
+        else
+          make "${PROFILE}-tgz"
+          make "${PROFILE}-pkg"
+        fi
+    - name: test emqx packages
+      env:
+        PROFILE: ${{ matrix.profile }}
+        IS_ELIXIR: ${{ matrix.with_elixir }}
+      run: |
+        set -eu
+        if [ "${IS_ELIXIR:-}" == 'yes' ]; then
+          ./scripts/pkg-tests.sh "${PROFILE}-elixir-tgz"
+        else
+          ./scripts/pkg-tests.sh "${PROFILE}-tgz"
+          ./scripts/pkg-tests.sh "${PROFILE}-pkg"
         fi
-        for PKGTYPE in ${PKGTYPES};
-        do
-          ./scripts/buildx.sh \
-            --profile "${PROFILE}" \
-            --pkgtype "${PKGTYPE}" \
-            --arch "${ARCH}" \
-            --elixir "${IS_ELIXIR}" \
-            --builder "force_host"
-        done
     - uses: actions/upload-artifact@v3
-      if: success()
       with:
         name: ${{ matrix.profile }}
         path: _packages/${{ matrix.profile }}/

.github/workflows/build_packages_cron.yaml

@@ -32,7 +32,7 @@ jobs:
           - debian10
           - amzn2023
         builder:
-          - 5.1-1
+          - 5.1-3
         elixir:
           - 1.14.5
 

.github/workflows/build_slim_packages.yaml

@@ -35,7 +35,7 @@ jobs:
           - ["emqx-enterprise", "25.3.2-1", "amzn2023", "erlang"]
           - ["emqx-enterprise", "25.3.2-1", "ubuntu20.04", "erlang"]
         builder:
-          - 5.1-1
+          - 5.1-3
         elixir:
           - '1.14.5'
 

.github/workflows/release.yaml

@@ -87,6 +87,8 @@ jobs:
           push "debian/buster" "packages/$PROFILE-$VERSION-debian10-arm64.deb"
           push "debian/bullseye" "packages/$PROFILE-$VERSION-debian11-amd64.deb"
           push "debian/bullseye" "packages/$PROFILE-$VERSION-debian11-arm64.deb"
+          push "debian/bookworm" "packages/$PROFILE-$VERSION-debian12-amd64.deb"
+          push "debian/bookworm" "packages/$PROFILE-$VERSION-debian12-arm64.deb"
           push "ubuntu/bionic" "packages/$PROFILE-$VERSION-ubuntu18.04-amd64.deb"
           push "ubuntu/bionic" "packages/$PROFILE-$VERSION-ubuntu18.04-arm64.deb"
           push "ubuntu/focal" "packages/$PROFILE-$VERSION-ubuntu20.04-amd64.deb"

.github/workflows/run_emqx_app_tests.yaml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       matrix:
         builder:
-          - 5.1-1
+          - 5.1-3
         otp:
           - 25.3.2-1
         # no need to use more than 1 version of Elixir, since tests

.github/workflows/run_fvt_tests.yaml

@@ -50,7 +50,7 @@ jobs:
         os:
           - ["debian11", "debian:11-slim"]
         builder:
-          - 5.1-1
+          - 5.1-3
         otp:
           - 25.3.2-1
         elixir:
@@ -123,7 +123,7 @@ jobs:
         os:
         - ["debian11", "debian:11-slim"]
         builder:
-        - 5.1-1
+        - 5.1-3
         otp:
         - 25.3.2-1
         elixir:

.github/workflows/run_test_cases.yaml

@@ -34,12 +34,12 @@ jobs:
           MATRIX="$(echo "${APPS}" | jq -c '
             [
               (.[] | select(.profile == "emqx") | . + {
-                builder: "5.1-1",
+                builder: "5.1-3",
                 otp: "25.3.2-1",
                 elixir: "1.14.5"
               }),
               (.[] | select(.profile == "emqx-enterprise") | . + {
-                builder: "5.1-1",
+                builder: "5.1-3",
                 otp: ["25.3.2-1"][],
                 elixir: "1.14.5"
               })

bin/emqx

@@ -811,8 +811,8 @@ is_down() {
     PID="$1"
     if ps -p "$PID" >/dev/null; then
         # still around
-        # shellcheck disable=SC2009 # this grep pattern is not a part of the progra names
+        # shellcheck disable=SC2009 # this grep pattern is not a part of the program names
-        if ps -p "$PID" | $GREP -q 'defunct'; then
+        if ps -efp "$PID" | $GREP -q 'defunct'; then
             # zombie state, print parent pid
             parent="$(ps -o ppid= -p "$PID" | tr -d ' ')"
             logwarn "$PID is marked <defunct>, parent: $(ps -p "$parent")"

changes/ce/feat-11289.en.md

@@ -0,0 +1 @@
+Release packages for Debian 12.

scripts/pkg-tests.sh

@@ -46,6 +46,7 @@ export SCRIPTS="${CODE_PATH}/scripts"
 export EMQX_NAME
 export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}"
 export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base"
+export PAHO_MQTT_TESTING_PATH="${PAHO_MQTT_TESTING_PATH:-/paho-mqtt-testing}"
 
 SYSTEM="$("$SCRIPTS"/get-distro.sh)"
 
@@ -64,7 +65,7 @@ fi
 PACKAGE_VERSION="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}")"
 PACKAGE_VERSION_LONG="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}" --long --elixir "${IS_ELIXIR}")"
 PACKAGE_NAME="${EMQX_NAME}-${PACKAGE_VERSION_LONG}"
-PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}"
+PACKAGE_FILE_NAME="${PACKAGE_FILE_NAME:-${PACKAGE_NAME}.${PKG_SUFFIX}}"
 
 PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}"
 if ! [ -f "$PACKAGE_FILE" ]; then
@@ -75,9 +76,21 @@ fi
 emqx_prepare(){
     mkdir -p "${PACKAGE_PATH}"
 
-    if [ ! -d "/paho-mqtt-testing" ]; then
+    if [ ! -d "${PAHO_MQTT_TESTING_PATH}" ]; then
-        git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho-mqtt-testing
+        git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git "${PAHO_MQTT_TESTING_PATH}"
     fi
+    # Debian 12 complains if we don't use venv
+    case "${SYSTEM:-}" in
+        debian12)
+            apt-get update -y && apt-get install -y virtualenv
+            virtualenv venv
+            # https://www.shellcheck.net/wiki/SC1091
+            # shellcheck source=/dev/null
+            source ./venv/bin/activate
+            ;;
+        *)
+            ;;
+    esac
     pip3 install pytest
 }
 
@@ -97,36 +110,22 @@ emqx_test(){
             # fi
             # sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins
 
-            echo "running ${packagename} start"
+            run_test "${PACKAGE_PATH}/emqx/bin" "${PACKAGE_PATH}/emqx/log" "${PACKAGE_PATH}/emqx/releases/emqx_vars"
-            if ! "${PACKAGE_PATH}"/emqx/bin/emqx start; then
+
-                cat "${PACKAGE_PATH}"/emqx/log/erlang.log.1 || true
-                cat "${PACKAGE_PATH}"/emqx/log/emqx.log.1 || true
-                exit 1
-            fi
-            "$SCRIPTS/test/emqx-smoke-test.sh" 127.0.0.1 18083
-            pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic
-            if ! "${PACKAGE_PATH}"/emqx/bin/emqx stop; then
-                cat "${PACKAGE_PATH}"/emqx/log/erlang.log.1 || true
-                cat "${PACKAGE_PATH}"/emqx/log/emqx.log.1 || true
-                exit 1
-            fi
-            echo "running ${packagename} stop"
             rm -rf "${PACKAGE_PATH}"/emqx
         ;;
         "deb")
             dpkg -i "${PACKAGE_PATH}/${packagename}"
-            if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "ii" ]
+            if [ "$(dpkg -l | grep ${EMQX_NAME} | awk '{print $1}')" != "ii" ]
             then
                 echo "package install error"
                 exit 1
             fi
 
-            echo "running ${packagename} start"
+            run_test "/usr/bin" "/var/log/emqx" "$(dpkg -L ${EMQX_NAME} | grep emqx_vars)"
-            run_test
-            echo "running ${packagename} stop"
 
             dpkg -r "${EMQX_NAME}"
-            if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "rc" ]
+            if [ "$(dpkg -l | grep ${EMQX_NAME} | awk '{print $1}')" != "rc" ]
             then
                 echo "package remove error"
                 exit 1
@@ -146,6 +145,10 @@ emqx_test(){
                     # el8 is fine with python3
                     true
                     ;;
+                "el9")
+                    # el9 is fine with python3
+                    true
+                    ;;
                 *)
                     alternatives --list | grep python && alternatives --set python /usr/bin/python2
                     ;;
@@ -161,12 +164,10 @@ emqx_test(){
                 exit 1
             fi
 
-            echo "running ${packagename} start"
+            run_test "/usr/bin" "/var/log/emqx" "$(rpm -ql ${EMQX_NAME} | grep emqx_vars)"
-            run_test
-            echo "running ${packagename} stop"
 
             rpm -e "${EMQX_NAME}"
-            if [ "$(rpm -q emqx)" != "package emqx is not installed" ];then
+            if [ "$(rpm -q ${EMQX_NAME})" != "package ${EMQX_NAME} is not installed" ];then
                 echo "package uninstall error"
                 exit 1
             fi
@@ -175,8 +176,10 @@ emqx_test(){
 }
 
 run_test(){
+    local bin_dir="$1"
+    local log_dir="$2"
+    local emqx_env_vars="$3"
     # sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins
-    emqx_env_vars=$(dirname "$(readlink "$(command -v emqx)")")/../releases/emqx_vars
 
     if [ -f "$emqx_env_vars" ];
     then
@@ -194,21 +197,21 @@ EOF
         echo "Error: cannot locate emqx_vars"
         exit 1
     fi
-    if ! emqx 'start'; then
+    echo "running ${packagename} start"
-        cat /var/log/emqx/erlang.log.1 || true
+    if ! "${bin_dir}/emqx" 'start'; then
-        cat /var/log/emqx/emqx.log.1 || true
+        echo "ERROR: failed_to_start_emqx"
+        cat "${log_dir}/erlang.log.1" || true
+        cat "${log_dir}/emqx.log.1" || true
         exit 1
     fi
     "$SCRIPTS/test/emqx-smoke-test.sh" 127.0.0.1 18083
-    pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic
+    pytest -v "${PAHO_MQTT_TESTING_PATH}"/interoperability/test_client/V5/test_connect.py::test_basic
-    # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions
+    "${bin_dir}/emqx" ping
-    ps -ef | grep -E '\-progname\s.+emqx\s'
+    echo "running ${packagename} stop"
-    if ! emqx 'stop'; then
+    if ! "${bin_dir}/emqx" 'stop'; then
-        # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions
-        ps -ef | grep -E '\-progname\s.+emqx\s'
         echo "ERROR: failed_to_stop_emqx_with_the_stop_command"
-        cat /var/log/emqx/erlang.log.1 || true
+        cat "${log_dir}/erlang.log.1" || true
-        cat /var/log/emqx/emqx.log.1 || true
+        cat "${log_dir}/emqx.log.1" || true
         exit 1
     fi
 }