name: Functional Verification Tests

on:
  push:
    tags:
      - v*
      - e*
  pull_request:

jobs:
    docker_test:
        runs-on: ubuntu-20.04

        steps:
        - uses: actions/checkout@v3
        - uses: emqx/setup-beam@v1.16.1-emqx
          with:
            otp-version: 24.3.4.6
        - name: prepare
          run: |
            if make emqx-ee --dry-run > /dev/null 2>&1; then
              echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
              git config --global credential.helper store
              make deps-emqx-ee
              make clean
              echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
              echo "PROFILE=emqx-ee" >> $GITHUB_ENV
              echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
              make emqx-ee-docker
            else
              echo "TARGET=emqx/emqx" >> $GITHUB_ENV
              echo "PROFILE=emqx" >> $GITHUB_ENV
              echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
              make emqx-docker
            fi
        - name: make emqx image
          run: make ${PROFILE}-docker
        - name: run emqx
          timeout-minutes: 5
          run: |
            set -e -u -x
            echo "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" >> .ci/docker-compose-file/conf.cluster.env
            echo "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s" >> .ci/docker-compose-file/conf.cluster.env
            echo "EMQX_MQTT__MAX_TOPIC_ALIAS=10" >> .ci/docker-compose-file/conf.cluster.env
            docker-compose \
                -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
                -f .ci/docker-compose-file/docker-compose-python.yaml \
                up -d
            while ! docker exec -i node1.emqx.io bash -c "emqx eval \"['emqx@node1.emqx.io','emqx@node2.emqx.io'] = maps:get(running_nodes, ekka_cluster:info()).\"" > /dev/null 2>&1; do
                echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:waiting emqx";
                sleep 5;
            done
        - name: verify EMQX_LOADED_PLUGINS override working
          run: |
            expected="{emqx_sn, true}."
            output=$(docker exec -i node1.emqx.io bash -c "cat data/loaded_plugins" | tail -n1)
            if [ "$expected" != "$output" ]; then
                exit 1
            fi
        - name: make paho tests
          run: |
            if ! docker exec -i python /scripts/pytest.sh; then
              docker logs node1.emqx.io
              docker logs node2.emqx.io
              exit 1
            fi

    helm_test:
        runs-on: ubuntu-20.04
        strategy:
          fail-fast: false
          matrix:
            discovery:
              - k8s
              - dns

        steps:
        - uses: actions/checkout@v3
        - uses: emqx/setup-beam@v1.16.1-emqx
          with:
            otp-version: 24.3.4.6
        - name: prepare
          run: |
            if make emqx-ee --dry-run > /dev/null 2>&1; then
              echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
              git config --global credential.helper store
              make deps-emqx-ee
              make clean
              echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
              echo "PROFILE=emqx-ee" >> $GITHUB_ENV
              echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
            else
              echo "TARGET=emqx/emqx" >> $GITHUB_ENV
              echo "PROFILE=emqx" >> $GITHUB_ENV
              echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
            fi
        - name: make emqx image
          run: make ${PROFILE}-docker
        - run: minikube start
        - name: install helm
          run: |
            curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
            sudo chmod 700 get_helm.sh
            sudo ./get_helm.sh
            helm version
        - name: run emqx on chart
          timeout-minutes: 5
          run: |
            version=$(./pkg-vsn.sh)
            minikube image load ${TARGET}:$version

            sed -i -r "s/^appVersion: .*$/appVersion: \"${version}\"/g" deploy/charts/emqx/Chart.yaml
            sed -i '/emqx_telemetry/d' deploy/charts/emqx/values.yaml
        - name: run emqx on chart
          if: matrix.discovery == 'k8s'
          run: |
            helm install emqx \
                --set image.repository=${TARGET} \
                --set image.pullPolicy=Never \
                --set emqxAclConfig="" \
                --set image.pullPolicy=Never \
                --set emqxConfig.EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s \
                --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \
                deploy/charts/emqx \
                --debug
        - name: run emqx on chart
          if: matrix.discovery == 'dns'
          run: |
            helm install emqx \
                --set emqxConfig.EMQX_CLUSTER__DISCOVERY="dns" \
                --set emqxConfig.EMQX_CLUSTER__DNS__NAME="emqx-headless.default.svc.cluster.local" \
                --set emqxConfig.EMQX_CLUSTER__DNS__APP="emqx" \
                --set emqxConfig.EMQX_CLUSTER__DNS__TYPE="srv" \
                --set image.repository=${TARGET} \
                --set image.pullPolicy=Never \
                --set emqxAclConfig="" \
                --set image.pullPolicy=Never \
                --set emqxConfig.EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s \
                --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \
                deploy/charts/emqx \
                --debug
        - name: waiting emqx started
          timeout-minutes: 5
          run: |
            while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.replicas}')" \
              != "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do
              echo "==============================";
              kubectl get pods;
              echo "==============================";
              echo "waiting emqx started";
              sleep 10;
            done
        - name: Check ${{ matrix.kind[0]}} cluster
          timeout-minutes: 10
          run: |
            while
              nodes="$(kubectl exec -i emqx-0 -- curl --silent --basic -u admin:public -X GET http://localhost:8081/api/v4/brokers | jq '.data|length')";
              [ "$nodes" != "3" ];
            do
              echo "waiting emqx cluster scale"
              sleep 1
            done
        - name: get emqx-0 pods log
          if: failure()
          run: |
            kubectl describe pods emqx-0
            kubectl logs emqx-0
        - name: get emqx-1 pods log
          if: failure()
          run: |
            kubectl describe pods emqx-1
            kubectl logs emqx-1
        - name: get emqx-2 pods log
          if: failure()
          run: |
            kubectl describe pods emqx-2
            kubectl logs emqx-2
        - uses: actions/checkout@v3
          with:
            repository: emqx/paho.mqtt.testing
            ref: develop-4.0
            path: paho.mqtt.testing
        - name: install pytest
          run: |
            pip install pytest
            echo "$HOME/.local/bin" >> $GITHUB_PATH
        - name: run paho test
          run: |
            nohup kubectl port-forward svc/emqx 1883:1883 &

            pytest -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic
            RESULT=$?
            pytest -v paho.mqtt.testing/interoperability/test_cluster
            RESULT=$((RESULT + $?))
            if [ 0 -ne $RESULT ]; then
                kubectl logs emqx-1
                kubectl logs emqx-2
            fi
            exit $RESULT

    relup_test_plan:
        runs-on: ubuntu-20.04
        container: ghcr.io/emqx/emqx-builder/4.4-24:24.3.4.2-1-ubuntu20.04
        outputs:
          profile: ${{ steps.profile-and-versions.outputs.profile }}
          vsn: ${{ steps.profile-and-versions.outputs.vsn }}
          old_vsns: ${{ steps.profile-and-versions.outputs.old_vsns }}
          broker: ${{ steps.profile-and-versions.outputs.broker }}
          matrix: ${{ steps.generate-matrix.outputs.matrix }}
        defaults:
          run:
            shell: bash
        steps:
        - uses: actions/checkout@v3
          name: Checkout
          with:
            path: emqx
            fetch-depth: 0
        - name: Get profile and version list
          id: profile-and-versions
          run: |
            cd emqx
            vsn="$(./pkg-vsn.sh)"
            if make emqx-ee --dry-run > /dev/null 2>&1; then
              profile="emqx-ee"
              old_vsns="$(./scripts/relup-base-vsns.sh enterprise | xargs)"
              broker="emqx-ee"
            else
              profile="emqx"
              old_vsns="$(./scripts/relup-base-vsns.sh community | xargs)"
              broker="emqx-ce"
            fi

            echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV

            echo "::set-output name=vsn::$vsn"
            echo "::set-output name=profile::$profile"
            echo "::set-output name=broker::$broker"
            echo "::set-output name=old_vsns::$old_vsns"
        - name: Generate matrix
          id: generate-matrix
          run: |
            matrix=$(echo -n "$OLD_VSNS" | sed 's/ $//g' | jq -R -s -c 'split(" ")')
            echo "::set-output name=matrix::$matrix"

    relup_test_build:
        needs: relup_test_plan
        strategy:
           fail-fast: false
           matrix:
             otp:
             - 24.3.4.2-1
        runs-on: ubuntu-20.04
        container: ghcr.io/emqx/emqx-builder/4.4-24:24.3.4.2-1-ubuntu20.04
        defaults:
          run:
            shell: bash
        env:
          OLD_VSNS: "${{ needs.relup_test_plan.outputs.old_vsns }}"
          PROFILE: "${{ needs.relup_test_plan.outputs.profile }}"
          BROKER: "${{ needs.relup_test_plan.outputs.broker }}"
        steps:
        - uses: actions/checkout@v3
          name: Checkout
          with:
            path: emqx
            fetch-depth: 0
        - name: Prepare credentials
          run: |
            if [ "$PROFILE" = "emqx-ee" ]; then
              echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
              git config --global credential.helper store
            fi
        - name: Build emqx
          run: make -C emqx ${PROFILE}-zip
        - uses: actions/upload-artifact@v3
          name: Upload built emqx and test scenario
          with:
            name: emqx_built
            path: |
              emqx/_packages/*/*.zip
              emqx/.ci/fvt_tests
              emqx/data/relup-paths.eterm
              emqx/scripts/relup-base-vsns.escript

    relup_test_run:
        needs:
          - relup_test_plan
          - relup_test_build
        runs-on: ubuntu-20.04
        container: ghcr.io/emqx/emqx-builder/4.4-24:24.3.4.2-1-ubuntu20.04
        strategy:
          fail-fast: false
          matrix:
            old_vsn: ${{ fromJson(needs.relup_test_plan.outputs.matrix) }}
        env:
          OLD_VSN: "${{ matrix.old_vsn }}"
          PROFILE: "${{ needs.relup_test_plan.outputs.profile }}"
          VSN: "${{ needs.relup_test_plan.outputs.vsn }}"
          BROKER: "${{ needs.relup_test_plan.outputs.broker }}"
        defaults:
          run:
            shell: bash
        steps:
        - uses: actions/download-artifact@v3
          name: Download built emqx and test scenario
          with:
            name: emqx_built
            path: emqx_built
        - uses: actions/checkout@v3
          name: Checkout one_more_emqx
          with:
            repository: terry-xiaoyu/one_more_emqx
            ref: master
            path: one_more_emqx
        - name: Run relup test scenario
          timeout-minutes: 6
          run: |
            set -e -x -u
            old_emqx_vsn=${OLD_VSN#[e|v]}
            old_otp_vsn=$(escript emqx_built/scripts/relup-base-vsns.escript otp-vsn-for "${old_emqx_vsn}" emqx_built/data/relup-paths.eterm)
            wget --no-verbose -P packages https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-${old_emqx_vsn}-otp${old_otp_vsn}-ubuntu20.04-amd64.zip
            cp emqx_built/_packages/*/*.zip packages
            lux \
            --progress verbose \
            --case_timeout infinity \
            --var PROFILE=$PROFILE \
            --var PACKAGE_PATH=$(pwd)/packages \
            --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \
            --var VSN="$VSN" \
            --var OLD_VSN="$OLD_VSN" \
            --var FROM_OTP_VSN="${old_otp_vsn}" \
            --var TO_OTP_VSN="24.3.4.2-1" \
            emqx_built/.ci/fvt_tests/relup.lux
        - uses: actions/upload-artifact@v3
          name: Save debug data
          if: failure()
          with:
            name: debug_data
            path: |
              packages/emqx/log/emqx.log.1
              packages/emqx2/log/emqx.log.1
              packages/*.zip
              lux_logs