emqx/.github/workflows/run_fvt_tests.yaml

name: Functional Verification Tests

concurrency:
  group: fvt-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: true

on:
  push:
    branches:
      - '**'
    tags:
      - v*
  pull_request:

jobs:
  prepare:
    runs-on: ubuntu-20.04
    # prepare source with any OTP version, no need for a matrix
    container: ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-alpine3.14

    steps:
      - uses: actions/checkout@v2
        with:
          path: source
          fetch-depth: 0
      - name: get deps
        run: |
          make -C source deps-all
          zip -ryq source.zip source/* source/.[^.]*          
      - uses: actions/upload-artifact@v2
        with:
          name: source
          path: source.zip

  docker_test:
    runs-on: ubuntu-20.04
    needs: prepare

    strategy:
      fail-fast: false
      matrix:
        profile:
          - emqx
          - emqx-edge
          - emqx-enterprise
          - emqx-elixir
        cluster_db_backend:
          - mnesia
          - rlog
        os:
          - alpine3.14
        otp:
          - 24.1.5-4
        elixir:
          - 1.13.2
        arch:
          - amd64
        exclude:
          - profile: emqx-edge
            cluster_db_backend: rlog
    steps:
    - uses: actions/download-artifact@v2
      with:
        name: source
        path: .
    - name: unzip source code
      run: unzip -q source.zip

    - name: Get deps git refs for cache
      id: deps-refs
      run: |
        cd source
        make ensure-rebar3
        sudo cp rebar3 /usr/local/bin/rebar3
        scripts/get-dep-refs.sh        
    - name: load rocksdb cache
      uses: actions/cache@v2
      with:
        path: |
          source/_build/default/lib/rocksdb/
          source/deps/rocksdb/          
        key: ${{ matrix.os }}-${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
    - name: load quicer cache
      uses: actions/cache@v2
      with:
        path: |
          source/_build/default/lib/quicer/
          source/deps/quicer/          
        key: ${{ matrix.os }}-${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}

    - name: make docker image
      working-directory: source
      env:
        EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
      run: |
        make ${{ matrix.profile }}-docker        
    - name: run emqx
      timeout-minutes: 5
      working-directory: source
      run: |
        set -x
        if [[ "${{ matrix.profile }}" = *-elixir ]]
        then
          export IS_ELIXIR=yes
          PROFILE=$(echo ${{ matrix.profile }} | sed -e "s/-elixir//g")
          IMAGE=emqx/$PROFILE:$(./pkg-vsn.sh ${{ matrix.profile }})-elixir
        else
          IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh ${{ matrix.profile }})
        fi
        ./.ci/docker-compose-file/scripts/run-emqx.sh $IMAGE ${{ matrix.cluster_db_backend }}        
    - name: make paho tests
      run: |
        if ! docker exec -i python /scripts/pytest.sh "${{ matrix.cluster_db_backend }}"; then
          echo "DUMP_CONTAINER_LOGS_BGN"
          echo "============== haproxy =============="
          docker logs haproxy
          echo "==============  node1  =============="
          docker logs node1.emqx.io
          echo "==============  node2  =============="
          docker logs node2.emqx.io
          echo "DUMP_CONTAINER_LOGS_END"
          exit 1
        fi        

  helm_test:
    runs-on: ubuntu-20.04
    needs: prepare

    strategy:
      fail-fast: false
      matrix:
        profile:
        - emqx
        os:
          - alpine3.14
        otp:
          - 24.1.5-4
        elixir:
          - 1.13.2
        arch:
          - amd64
      # - emqx-enterprise # TODO test enterprise

    steps:
    - uses: actions/download-artifact@v2
      with:
        name: source
        path: .
    - name: unzip source code
      run: unzip -q source.zip

    - name: Get deps git refs for cache
      id: deps-refs
      run: |
        cd source
        make ensure-rebar3
        sudo cp rebar3 /usr/local/bin/rebar3
        scripts/get-dep-refs.sh        
    - name: load rocksdb cache
      uses: actions/cache@v2
      with:
        path: source/_build/default/lib/rocksdb/
        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
    - name: load quicer cache
      uses: actions/cache@v2
      with:
        path: source/_build/default/lib/quicer/
        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}

    - name: make docker image
      working-directory: source
      env:
        EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
      run: |
        make ${{ matrix.profile }}-docker
        echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV
        echo "EMQX_TAG=$(./pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV        
    - run: minikube start
    - name: run emqx on chart
      timeout-minutes: 5
      working-directory: source
      run: |
        minikube image load $TARGET:$EMQX_TAG

        sed -i -r "s/^appVersion: .*$/appVersion: \"$EMQX_TAG\"/g" deploy/charts/emqx/Chart.yaml

        helm install emqx \
            --set image.repository=$TARGET \
            --set image.pullPolicy=Never \
            --set emqxAclConfig="" \
            --set image.pullPolicy=Never \
            --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s \
            --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \
            deploy/charts/emqx \
            --debug

        while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.replicas}')" \
          != "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do
          echo "==============================";
          kubectl get pods;
          echo "==============================";
          echo "waiting emqx started";
          sleep 10;
        done        
    - name: get emqx-0 pods log
      if: failure()
      run: |
        kubectl describe pods emqx-0
        kubectl logs emqx-0        
    - name: get emqx-1 pods log
      if: failure()
      run: |
        kubectl describe pods emqx-1
        kubectl logs emqx-1        
    - name: get emqx-2 pods log
      if: failure()
      run: |
        kubectl describe pods emqx-2
        kubectl logs emqx-2        
    - uses: actions/checkout@v2
      with:
        repository: emqx/paho.mqtt.testing
        ref: develop-4.0
        path: paho.mqtt.testing
    - name: install pytest
      run: |
        pip install pytest
        echo "$HOME/.local/bin" >> $GITHUB_PATH        
    - name: run paho test
      run: |
        port_connected () {
          local server="$1"
          local port="$2"
          echo > /dev/tcp/${server}/${port} 2>/dev/null
        }

        kubectl port-forward service/emqx 1883:1883 > /dev/null &

        while ! port_connected localhost 1883; do
          echo server not listening yet...
          sleep 10
        done

        pytest -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "127.0.0.1"