emqx/.github/workflows/run_helm_tests.yaml

name: Helm tests

concurrency:
  group: helm-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: true

on:
  workflow_call:

permissions:
  contents: read

jobs:
  helm_test:
    runs-on: ${{ github.repository_owner == 'emqx' && fromJSON('["self-hosted","ephemeral","linux","x64"]') || 'ubuntu-22.04' }}
    defaults:
      run:
        shell: bash
    env:
      EMQX_NAME: ${{ matrix.profile }}
      REPOSITORY: "emqx/${{ matrix.profile }}"

    strategy:
      fail-fast: false
      matrix:
        discovery:
        - k8s
        - dns
        profile:
        - emqx
        - emqx-enterprise
        rpc:
        - tcp
        - ssl1.3
        - ssl1.2
    steps:
    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
      with:
        path: source
    - name: Set up environment
      id: env
      run: |
        cd source
        source env.sh
        PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh "$EMQX_NAME")
        echo "EMQX_TAG=$PKG_VSN" >> "$GITHUB_ENV"        
    - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
      with:
        name: "${{ env.EMQX_NAME }}-docker"
        path: /tmp
    - run: minikube start
    - run: |
        img="/tmp/${EMQX_NAME}-docker-${EMQX_TAG}.tar.gz"
        if stderr=$(minikube image load "${img}" 2>&1 >/dev/null) && test -n "$stderr"; then
          echo "${stderr}";
          exit 1;
        fi        
    - name: Prepare emqxConfig.EMQX_RPC using TCP
      working-directory: source
      if: matrix.rpc == 'tcp'
      run: |
        cat > rpc-overrides.yaml <<EOL
        emqxConfig:
          EMQX_RPC__PROTOCOL: tcp
        EOL        
    - name: Prepare emqxConfig.EMQX_RPC using ssl1.3
      working-directory: source
      if: matrix.rpc == 'ssl1.3'
      run: |
        cat > rpc-overrides.yaml <<EOL
        emqxConfig:
          EMQX_RPC__PROTOCOL: ssl
          EMQX_RPC__CERTFILE: /opt/emqx/etc/certs/cert.pem
          EMQX_RPC__KEYFILE: /opt/emqx/etc/certs/key.pem
          EMQX_RPC__CACERTFILE: /opt/emqx/etc/certs/cacert.pem
          EMQX_RPC__CIPHERS: TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256
          EMQX_RPC__TLS_VERSIONS: "[tlsv1.3]"
        EOL        
    - name: Prepare emqxConfig.EMQX_RPC using ssl1.2
      working-directory: source
      if: matrix.rpc == 'ssl1.2'
      run: |
        cat > rpc-overrides.yaml <<EOL
        emqxConfig:
          EMQX_RPC__PROTOCOL: ssl
          EMQX_RPC__CERTFILE: /opt/emqx/etc/certs/cert.pem
          EMQX_RPC__KEYFILE: /opt/emqx/etc/certs/key.pem
          EMQX_RPC__CACERTFILE: /opt/emqx/etc/certs/cacert.pem
          EMQX_RPC__CIPHERS: TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256
          EMQX_RPC__TLS_VERSIONS: "[tlsv1.2]"
        EOL        
    - name: run emqx on chart (k8s)
      if: matrix.discovery == 'k8s'
      working-directory: source
      run: |
        helm install ${EMQX_NAME} \
            --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="k8s" \
            --set emqxConfig.EMQX_CLUSTER__K8S__APISERVER="https://kubernetes.default.svc:443" \
            --set emqxConfig.EMQX_CLUSTER__K8S__SERVICE_NAME="${EMQX_NAME}-headless" \
            --set emqxConfig.EMQX_CLUSTER__K8S__NAMESPACE="default" \
            --set image.repository=$REPOSITORY \
            --set image.pullPolicy=Never \
            --set image.tag=$EMQX_TAG \
            --set emqxAclConfig="" \
            --set emqxConfig.EMQX_MQTT__RETRY_INTERVAL=2s \
            --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \
            --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \
            --set emqxConfig.EMQX_LOG__CONSOLE__LEVEL=debug \
            --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \
            --values rpc-overrides.yaml \
            deploy/charts/${EMQX_NAME} \
            --debug        
    - name: run emqx on chart (dns)
      if: matrix.discovery == 'dns'
      working-directory: source
      run: |
        helm install ${EMQX_NAME} \
            --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="dns" \
            --set emqxConfig.EMQX_CLUSTER__DNS__RECORD_TYPE="srv" \
            --set emqxConfig.EMQX_CLUSTER__DNS__NAME="${EMQX_NAME}-headless.default.svc.cluster.local" \
            --set image.repository=$REPOSITORY \
            --set image.pullPolicy=Never \
            --set image.tag=$EMQX_TAG \
            --set emqxAclConfig="" \
            --set emqxConfig.EMQX_MQTT__RETRY_INTERVAL=2s \
            --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \
            --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \
            --set emqxConfig.EMQX_LOG__CONSOLE__LEVEL=debug \
            --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \
            --values rpc-overrides.yaml \
            deploy/charts/${EMQX_NAME} \
            --wait \
            --debug        
    - name: waiting emqx started
      timeout-minutes: 5
      run: |
        while [ "$(kubectl get StatefulSet -l app.kubernetes.io/instance=${EMQX_NAME} -o jsonpath='{.items[0].status.replicas}')" \
          != "$(kubectl get StatefulSet -l app.kubernetes.io/instance=${EMQX_NAME} -o jsonpath='{.items[0].status.readyReplicas}')" ]; do
          echo "==============================";
          kubectl get pods;
          echo "==============================";
          echo "waiting emqx started";
          sleep 10;
        done        
    - name: Setup 18083 port forwarding
      run: |
        nohup kubectl port-forward service/${EMQX_NAME} 18083:18083 > /dev/null &        
    - name: Get auth token
      run: |
        curl --head -X GET --retry 10 --retry-connrefused --retry-delay 6 http://localhost:18083/status
        echo "TOKEN=$(curl --silent -X 'POST' 'http://127.0.0.1:18083/api/v5/login' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"username": "admin","password": "public"}' | jq -r ".token")" >> $GITHUB_ENV        
    - name: Check cluster
      timeout-minutes: 1
      run: |
        while
          nodes_length="$(curl --silent -H "Authorization: Bearer $TOKEN" -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')"
          [ $nodes_length != "3" ]
        do
          if [ $nodes_length -eq 0 ]; then
            echo "node len must >= 1, refresh Token... "
            TOKEN=$(curl --silent -X 'POST' 'http://127.0.0.1:18083/api/v5/login' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"username": "admin","password": "public"}' | jq -r ".token")
          else
            echo "waiting ${EMQX_NAME} cluster scale. Current live nodes: $nodes_length."
          fi
          sleep 1;
        done        
    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
      with:
        repository: emqx/paho.mqtt.testing
        ref: develop-5.0
        path: paho.mqtt.testing
    - name: install pytest
      run: |
        pip install --require-hashes -r source/.ci/docker-compose-file/python/requirements.txt
        echo "$HOME/.local/bin" >> $GITHUB_PATH        
    - name: run paho test
      timeout-minutes: 10
      run: |
        port_connected () {
          local server="$1"
          local port="$2"
          echo > /dev/tcp/${server}/${port} 2>/dev/null
        }

        kubectl port-forward service/${EMQX_NAME} 1883:1883 > /dev/null &

        while ! port_connected localhost 1883; do
          echo server not listening yet...
          sleep 10
        done

        pytest --retries 3 -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "127.0.0.1"        
    - if: failure()
      run: kubectl logs -l "app.kubernetes.io/instance=${EMQX_NAME}" -c emqx --tail=1000