emqx/.github/workflows/run_fvt_tests.yaml

name: Functional Verification Tests

concurrency:
  group: fvt-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: true

on:
  push:
    branches:
      - master
      - 'ci/**'
    tags:
      - v*
  pull_request:

jobs:
  prepare:
    runs-on: ubuntu-22.04
    # prepare source with any OTP version, no need for a matrix
    container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-debian11

    steps:
      - uses: actions/checkout@v3
        with:
          path: source
          fetch-depth: 0
      - name: get deps
        run: |
          make -C source deps-all
          zip -ryq source.zip source/* source/.[^.]*          
      - uses: actions/upload-artifact@v3
        with:
          name: source
          path: source.zip

  docker_test:
    runs-on: ubuntu-22.04
    needs: prepare

    strategy:
      fail-fast: false
      matrix:
        profile:
          - emqx
          - emqx-enterprise
          - emqx-elixir
        cluster_db_backend:
          - mnesia
          - rlog
        os:
          - ["debian11", "debian:11-slim"]
        builder:
          - 5.0-32
        otp:
          - 24.3.4.2-2
        elixir:
          - 1.13.4
        arch:
          - amd64
    steps:
    - uses: erlef/setup-beam@v1.15.2
      with:
        otp-version: 24.3.4.6
    - uses: actions/download-artifact@v3
      with:
        name: source
        path: .
    - name: unzip source code
      run: unzip -q source.zip

    - name: make docker image
      working-directory: source
      env:
        EMQX_BUILDER: ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
        EMQX_RUNNER: ${{ matrix.os[1] }}
      run: |
        make ${{ matrix.profile }}-docker        
    - name: run emqx
      timeout-minutes: 5
      working-directory: source
      run: |
        set -x
        if [[ "${{ matrix.profile }}" = *-elixir ]]
        then
          export IS_ELIXIR=yes
          PROFILE=$(echo ${{ matrix.profile }} | sed -e "s/-elixir//g")
          IMAGE=emqx/$PROFILE:$(./pkg-vsn.sh ${{ matrix.profile }})-elixir
        else
          IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh ${{ matrix.profile }})
        fi
        ./.ci/docker-compose-file/scripts/run-emqx.sh $IMAGE ${{ matrix.cluster_db_backend }}        
    - name: make paho tests
      run: |
        if ! docker exec -i python /scripts/pytest.sh "${{ matrix.cluster_db_backend }}"; then
          echo "DUMP_CONTAINER_LOGS_BGN"
          echo "============== haproxy =============="
          docker logs haproxy
          echo "==============  node1  =============="
          docker logs node1.emqx.io
          echo "==============  node2  =============="
          docker logs node2.emqx.io
          echo "DUMP_CONTAINER_LOGS_END"
          exit 1
        fi        
    # simple smoke test for node_dump
    - name: test node_dump
      run: |
        docker exec node1.emqx.io node_dump        

  helm_test:
    runs-on: ubuntu-22.04
    needs: prepare

    strategy:
      fail-fast: false
      matrix:
        discovery:
        - k8s
        - dns
        profile:
        - emqx
        - emqx-enterprise
        os:
        - ["debian11", "debian:11-slim"]
        builder:
        - 5.0-32
        otp:
        - 24.3.4.2-2
        elixir:
        - 1.13.4
        arch:
        - amd64
      # - emqx-enterprise # TODO test enterprise

    steps:
    - uses: erlef/setup-beam@v1.15.2
      with:
        otp-version: 24.3.4.6
    - uses: actions/download-artifact@v3
      with:
        name: source
        path: .
    - name: unzip source code
      run: unzip -q source.zip

    - name: make docker image
      working-directory: source
      env:
        EMQX_BUILDER: ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
        EMQX_RUNNER: ${{ matrix.os[1] }}
      run: |
        make ${{ matrix.profile }}-docker
        echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV
        echo "EMQX_TAG=$(./pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV        
    - run: minikube start
    - run: minikube image load $TARGET:$EMQX_TAG
    - name: run emqx on chart
      working-directory: source
      if: matrix.discovery == 'k8s'
      run: |
        helm install ${{ matrix.profile }} \
            --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="k8s" \
            --set emqxConfig.EMQX_CLUSTER__K8S__APISERVER="https://kubernetes.default.svc:443" \
            --set emqxConfig.EMQX_CLUSTER__K8S__SERVICE_NAME="${{ matrix.profile }}-headless" \
            --set emqxConfig.EMQX_CLUSTER__K8S__NAMESPACE="default" \
            --set image.repository=$TARGET \
            --set image.pullPolicy=Never \
            --set image.tag=$EMQX_TAG \
            --set emqxAclConfig="" \
            --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s \
            --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \
            --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \
            --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \
            deploy/charts/${{ matrix.profile }} \
            --debug        
    - name: run emqx on chart
      working-directory: source
      if: matrix.discovery == 'dns'
      run: |
        helm install ${{ matrix.profile }} \
            --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="dns" \
            --set emqxConfig.EMQX_CLUSTER__DNS__RECORD_TYPE="srv" \
            --set emqxConfig.EMQX_CLUSTER__DNS__NAME="${{ matrix.profile }}-headless.default.svc.cluster.local" \
            --set image.repository=$TARGET \
            --set image.pullPolicy=Never \
            --set image.tag=$EMQX_TAG \
            --set emqxAclConfig="" \
            --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s \
            --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \
            --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \
            --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \
            deploy/charts/${{ matrix.profile }} \
            --debug        
    - name: waiting emqx started
      timeout-minutes: 10
      run: |
        while [ "$(kubectl get StatefulSet -l app.kubernetes.io/instance=${{ matrix.profile }} -o jsonpath='{.items[0].status.replicas}')" \
          != "$(kubectl get StatefulSet -l app.kubernetes.io/instance=${{ matrix.profile }} -o jsonpath='{.items[0].status.readyReplicas}')" ]; do
          echo "==============================";
          kubectl get pods;
          echo "==============================";
          echo "waiting emqx started";
          sleep 10;
        done        
    - name: Get Token
      timeout-minutes: 1
      run: |
        kubectl port-forward service/${{ matrix.profile }} 18083:18083 > /dev/null &

        while
          [ "$(curl --silent -X 'GET' 'http://127.0.0.1:18083/api/v5/status' | tail -n1)" != "emqx is running" ]
        do
          echo "waiting emqx"
          sleep 1
        done

        echo "TOKEN=$(curl --silent -X 'POST' 'http://127.0.0.1:18083/api/v5/login' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"username": "admin","password": "public"}' | jq -r ".token")" >> $GITHUB_ENV        

    - name: Check cluster
      timeout-minutes: 10
      run: |
        while
          [ "$(curl --silent -H "Authorization: Bearer $TOKEN" -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" != "3" ];
        do
          echo "waiting ${{ matrix.profile }} cluster scale"
          sleep 1
        done        
    - uses: actions/checkout@v3
      with:
        repository: emqx/paho.mqtt.testing
        ref: develop-4.0
        path: paho.mqtt.testing
    - name: install pytest
      run: |
        pip install pytest
        echo "$HOME/.local/bin" >> $GITHUB_PATH        
    - name: run paho test
      timeout-minutes: 10
      run: |
        port_connected () {
          local server="$1"
          local port="$2"
          echo > /dev/tcp/${server}/${port} 2>/dev/null
        }

        kubectl port-forward service/${{ matrix.profile }} 1883:1883 > /dev/null &

        while ! port_connected localhost 1883; do
          echo server not listening yet...
          sleep 10
        done

        pytest -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "127.0.0.1"        
    - if: failure()
      run: kubectl logs -l "app.kubernetes.io/instance=${{ matrix.profile }}" -c emqx --tail=1000