name: Test cloud compatibility

on:
  workflow_dispatch:
    inputs:
      target:
        description: Select target
        type: choice
        options:
        - '["ovirt-44-rocky-8"]'
        - '["ovirt-45-rocky-8"]'
        - '["ovirt-45-rocky-9"]'
        - '["kolla-ansible-master-debian-12"]'
        default: '["kolla-ansible-master-debian-12"]'
        required: true
      retention:
        description: 'How long to retain the environment after the workflow run in minutes'
        required: true
        default: '0'
  pull_request:
    branches:
      - develop

jobs:
  sanity_checks:
    runs-on: [self-hosted, vm, debian-12]
    concurrency:
      group: ${{ github.workflow }}-${{ github.ref }}-lint
      cancel-in-progress: true

    steps:
      - name: Checkout code with two commits
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Install additional dependencies not tracked by pip
        timeout-minutes: 5
        run: |
          sudo apt-get install -y python3-venv
          python3 -mvenv /tmp/bindep-venv
          /tmp/bindep-venv/bin/pip3 install bindep
          echo
          /tmp/bindep-venv/bin/bindep || true
          echo
          sudo apt-get install -y $(/tmp/bindep-venv/bin/bindep --brief || true)

      - name: Upgrade base test tools
        timeout-minutes: 5
        run: |
          python3 -mvenv /tmp/venv-test
          . /tmp/venv-test/bin/activate
          pip3 install uv
          uv pip install tox tox-uv

      - name: Lint with flake8
        timeout-minutes: 5
        run: |
          . /tmp/venv-test/bin/activate
          tox -eflake8

      - name: Attempt to install requirements
        timeout-minutes: 5
        run: |
          python3 -mvenv /tmp/venv-reqs
          . /tmp/venv-reqs/bin/activate
          pip3 install uv
          uv pip install -r pyproject.toml

      - name: Run python3 unit tests
        timeout-minutes: 10
        run: |
          . /tmp/venv-test/bin/activate
          tox -epy3

      - name: List failing tests
        if: failure()
        id: failures
        run: |
          echo "Gathering list of failed tests..."
          touch /srv/ci/failed
          . /tmp/venv-test/bin/activate
          tox -efailing || true > /srv/ci/failed
          echo

          failed=$(cat /srv/ci/failed | egrep "^FAIL: " | \
              sed -z 's/FAIL: shakenfist_ci\.//g;s/\n/, /g;s/, $/\n/')

          echo "Failed tests:"
          cat /srv/ci/failed
          echo

          echo "failures<<EOF" >> ${GITHUB_OUTPUT}
          echo "${failed}" >> ${GITHUB_OUTPUT}
          echo "EOF" >> ${GITHUB_OUTPUT}
          echo "GITHUB_OUTPUT is:"
          cat ${GITHUB_OUTPUT}

      - name: Generate coverage report
        timeout-minutes: 10
        run: |
          . /tmp/venv-test/bin/activate
          tox -ecover
          zip -r cover.zip cover

      - uses: actions/upload-artifact@v4
        with:
          name: coverage
          retention-days: 90
          if-no-files-found: error
          path: cover.zip

  ovirt_matrix:
    name: "${{ matrix.test.description }}"
    strategy:
      fail-fast: false
      matrix:
        test: [
          {
            'name': 'ovirt-44-rocky-8',
            'description': 'oVirt 4.4 on Rocky 8',
            'environment_name': 'ovirt',
            'ovirt_rpm': 'https://resources.ovirt.org/pub/yum-repo/ovirt-release44.rpm',
            'base': 'rocky:8',
            'baseuser': 'cloud-user',
            'hostname': 'ovirt'
          },
          {
            'name': 'ovirt-45-rocky-8',
            'description': 'oVirt 4.5 on Rocky 8',
            'environment_name': 'ovirt',
            'ovirt_rpm': 'centos-release-ovirt45',
            'base': 'rocky:8',
            'baseuser': 'cloud-user',
            'hostname': 'ovirt'
          },
          {
            'name': 'ovirt-45-rocky-9',
            'description': 'oVirt 4.5 on Rocky 9',
            'environment_name': 'ovirt',
            'ovirt_rpm': 'centos-release-ovirt45',
            'base': 'rocky:9',
            'baseuser': 'cloud-user',
            'hostname': 'ovirt'
          },
        ]
    runs-on: [self-hosted, vm]
    needs: sanity_checks
    timeout-minutes: 120
    concurrency:
      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.test.name }}
      cancel-in-progress: true

    # NOTE(mikal): git repos are checked out to /srv/github/_work/{org}/{repo}
    # which is available as GITHUB_WORKSPACE. You can find other environment
    # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables

    steps:
      - name: Filter workflow_dispatch runs
        if: github.event_name == 'workflow_dispatch' && ! contains(inputs.target, matrix.test.name)
        uses: actions/github-script@v6
        with:
          script: |
              core.setFailed('Target skipped')

      - name: Set environment variables
        shell: bash
        run: |
          echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV

      - name: Checkout the actions repository
        uses: actions/checkout@v4
        with:
          repository: shakenfist/actions
          path: actions
          fetch-depth: 0

      - name: Checkout kerbside
        uses: actions/checkout@v4
        with:
          path: kerbside
          fetch-depth: 0

      - name: Build infrastructure
        shell: bash
        run: |
          . $GITHUB_ENV
          ansible-playbook -i /home/debian/ansible-hosts \
              --extra-vars "identifier=${SHAKENFIST_NAMESPACE} base=${{ matrix.test.base }} environment_name=${{ matrix.test.environment_name }} hostname=${{ matrix.test.hostname }}" \
              --tags all ${GITHUB_WORKSPACE}/actions/ansible/kerbside-single-node.yml

      - name: Install base packages
        run: |
          . $GITHUB_ENV

          script="export PS4='=======================\n+ '; set -x;"
          script="${script} sudo dnf update -y;"
          script="${script} sudo dnf install -y epel-release;"
          script="${script} sudo crb enable;"
          script="${script} sudo dnf update -y;"
          script="${script} sudo dnf install -y vim patch yum-utils;"
          script="${script} sudo chown ${{ matrix.test.baseuser }}:${{ matrix.test.baseuser }} /srv;"

          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.test.baseuser }}@10.0.2.2 \
              "${script}"

      - name: Install upstream oVirt release package
        run: |
          . $GITHUB_ENV

          script="export PS4='=======================\n+ '; set -x;"
          script="${script} sudo dnf install -y ${{ matrix.test.ovirt_rpm }};"

          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.test.baseuser }}@10.0.2.2 \
              "${script}"

      - name: Copy oVirt yum repos patch to target node
        run: |
          . $GITHUB_ENV

          scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
            -o UserKnownHostsFile=/dev/null \
            ${GITHUB_WORKSPACE}/kerbside/etc/${{ matrix.test.name }}-repos.patch \
            ${{ matrix.test.baseuser }}@10.0.2.2:/srv/

      - name: Apply repos patch
        run: |
          . $GITHUB_ENV

          script="export PS4='=======================\n+ '; set -x;"
          script="${script} cd /etc/yum.repos.d;"
          script="${script} sudo patch -p1 < /srv/${{ matrix.test.name }}-repos.patch;"

          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.test.baseuser }}@10.0.2.2 \
              "${script}"

      - name: Further package configuration
        run: |
          . $GITHUB_ENV

          script="export PS4='=======================\n+ '; set -x;"
          script="${script} sudo dnf update -y;"
          script="${script} sudo dnf module -y enable javapackages-tools;"
          script="${script} sudo dnf module -y enable postgresql:12;"
          script="${script} sudo dnf module -y enable mod_auth_openidc:2.3;"
          script="${script} sudo dnf module -y enable nodejs:14;"
          script="${script} sudo dnf module -y enable pki-deps;"
          script="${script} sudo dnf distro-sync --nobest -y;"

          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.test.baseuser }}@10.0.2.2 \
              "${script}"

      - name: Install ovirt-engine
        run: |
          . $GITHUB_ENV

          script="export PS4='=======================\n+ '; set -x;"
          script="${script} sudo dnf install -y ovirt-engine;"

          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.test.baseuser }}@10.0.2.2 \
              "${script}"

      - name: Setup ovirt-engine
        run: |
          . $GITHUB_ENV

          scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
            -o UserKnownHostsFile=/dev/null \
            ${GITHUB_WORKSPACE}/kerbside/etc/${{ matrix.test.name }}-answers.conf \
            ${{ matrix.test.baseuser }}@10.0.2.2:/srv/

          script="export PS4='=======================\n+ '; set -x;"
          script="${script} sudo engine-setup"
              script="${script} --config-append=/srv/${{ matrix.test.name }}-answers.conf;"

          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.test.baseuser }}@10.0.2.2 \
              "${script}"

      - name: Gather artifacts
        run: |
          . $GITHUB_ENV

          script="export PS4='=======================\n+ '; set -x;"

          script="${script} sudo rpm -qa --qf "
              script="${script} \"%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}\n\" "
              script="${script} > /tmp/rpms.list;"

          script="${script} for rpm in \$(cat /tmp/rpms.list); do"
              script="${script} echo \"Caching URLs for \${rpm}\";"
              script="${script} sudo yumdownloader --resolve --urls \${rpm} |"
              script="${script} egrep -v '(metadata expiration|Waiting for)' "
              script="${script} >> /tmp/rpms.urls;"
              script="${script} done;"

          script="${script} sudo zip -r /tmp/bundle.zip "
              script="${script} /etc/yum.repos.d "
              script="${script} /tmp/rpms.list "
              script="${script} /tmp/rpms.urls "
              script="${script} /var/lib/ovirt-engine/setup "
              script="${script} || true;"
          script="${script} sudo chmod ugo+r /tmp/bundle.zip;"
          script="${script} ls -lrth /tmp/bundle.zip;"

          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.test.baseuser }}@10.0.2.2 \
              "${script}"

          mkdir -p /srv/github/artifacts/bundle/
          scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.test.baseuser }}@10.0.2.2:/tmp/bundle.zip \
              /srv/github/artifacts/bundle/

          cd /srv/github/artifacts/bundle/
          unzip bundle.zip
          rm bundle.zip
          cd ..
          zip -r bundle.zip bundle

      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: bundle-${{ matrix.test.name }}
          retention-days: 90
          if-no-files-found: error
          path: /srv/github/artifacts/bundle.zip

      - name: Retain the environment if requested
        if: github.event_name == 'workflow_dispatch'
        run: sleep $(( $MINUTES * 60 ))
        env:
          MINUTES: ${{ inputs.retention }}

  openstack_matrix:
    name: "${{ matrix.test.description }}"
    strategy:
      fail-fast: false
      matrix:
        test: [
          {
            'name': 'kolla-ansible-master-debian-12',
            'environment_name': 'kolla',
            'description': 'OpenStack via Kolla-Ansible master on Debian 12',
            'openstack_release': 'master',
            'base': 'debian:12',
            'baseuser': 'debian'
          },
        ]
    runs-on: [self-hosted, vm]
    needs: sanity_checks
    timeout-minutes: 120
    concurrency:
      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.test.name }}
      cancel-in-progress: true

    # NOTE(mikal): git repos are checked out to /srv/github/_work/{org}/{repo}
    # which is available as GITHUB_WORKSPACE. You can find other environment
    # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables

    steps:
      - name: Filter workflow_dispatch runs
        if: github.event_name == 'workflow_dispatch' && ! contains(inputs.target, matrix.test.name)
        uses: actions/github-script@v6
        with:
          script: |
              core.setFailed('Target skipped')

      - name: Setup test environment
        uses: shakenfist/actions/setup-kerbside-environment@main
        with:
          base: ${{ matrix.test.base }}
          base_user: ${{ matrix.test.baseuser }}
          openstack_release: ${{ matrix.test.openstack_release }}
          topology: all-in-one

      - name: Build patched repositories
        run: |
          . $GITHUB_ENV
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.test.baseuser }}@10.0.2.2 \
              'cd /srv/shakenfist/kerbside-patches; ./_build/assemble-source.sh ${{ matrix.test.openstack_release }}'

      - name: Build container images
        run: |
          . $GITHUB_ENV

          script="cd /srv/shakenfist/kerbside-patches;"
          script="${script} hash=\$(./_build/calculate-container-hash.sh kolla kolla-2* nova nova-2* requirements requirements-2*);"
          script="${script} ./_build/build-containers.sh --use-ci-registry --build-targets ${{ matrix.test.openstack_release }} --build-images all --image-tag "\$hash-debian-bookworm";"

          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.test.baseuser }}@10.0.2.2 "${script}"

      - name: Bootstrap Kolla-Ansible
        run: |
          . $GITHUB_ENV
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.test.baseuser }}@10.0.2.2 \
              'cd /srv/shakenfist/kerbside-patches; sudo ./tools/bootstrap-kolla-ansible ${{ matrix.test.openstack_release }} all-in-one'

      - name: Install Kolla-Ansible
        run: |
          . $GITHUB_ENV
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.test.baseuser }}@10.0.2.2 \
              'cd /srv/shakenfist/kerbside-patches; sudo ./tools/install-kolla-ansible'

      - name: Install patched OpenStack clients
        run: |
          . $GITHUB_ENV
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.test.baseuser }}@10.0.2.2 \
              'cd /srv/shakenfist/kerbside-patches; sudo ./tools/install-openstack-clients'

      - name: Post install Kolla-Ansible setup
        run: |
          . $GITHUB_ENV
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.test.baseuser }}@10.0.2.2 \
              'cd /srv/shakenfist/kerbside-patches; sudo ./tools/postinstall-kolla-ansible'

      - name: Attempt to start an instance with a console and connect to it
        run: |
          . $GITHUB_ENV
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.test.baseuser }}@10.0.2.2 \
              'cd /srv/shakenfist/kerbside-patches; sudo ./tools/test-console'

      - name: Collect logs
        if: always()
        run: |
          . $GITHUB_ENV
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.test.baseuser }}@10.0.2.2 \
              'cd /srv/shakenfist/kerbside-patches; sudo ./tools/gather-logs'
          scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.test.baseuser }}@10.0.2.2:/tmp/bundle.zip /tmp/bundle.zip

      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: bundle-${{ matrix.test.name }}
          retention-days: 90
          if-no-files-found: error
          path: /tmp/bundle.zip

      - name: Retain the environment if requested
        if: github.event_name == 'workflow_dispatch'
        run: sleep $(( $MINUTES * 60 ))
        env:
          MINUTES: ${{ inputs.retention }}