# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.

import os
import subprocess
import sys


def test_run_tests():
    cmd = [
        sys.executable,
        "-m",
        "llama_verifications.cli.main",
        "run-tests",
        "--provider=llama_api",
        "--test-filter=test_chat_streaming_basic and earth and scout",
    ]
    result = subprocess.run(cmd, capture_output=True, text=True)
    assert "1 passed" in result.stdout
    assert result.returncode == 0, (
        f"run-tests failed with exit code {result.returncode}.\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
    )


def test_run_tests_with_model_and_provider():
    cmd = [
        sys.executable,
        "-m",
        "llama_verifications.cli.main",
        "run-tests",
        "--provider=llama_api",
        "--model=Llama-4-Scout-17B-16E-Instruct-FP8",
        "--test-filter=test_chat_streaming_basic and earth",
    ]
    result = subprocess.run(cmd, capture_output=True, text=True)
    assert "1 passed" in result.stdout
    assert result.returncode == 0, (
        f"run-tests failed with exit code {result.returncode}.\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
    )


def test_run_tests_with_model():
    cmd = [
        sys.executable,
        "-m",
        "llama_verifications.cli.main",
        "run-tests",
        "--model=Llama-4-Scout-17B-16E-Instruct-FP8",
        "--test-filter=test_chat_streaming_basic and earth",
    ]
    result = subprocess.run(cmd, capture_output=True, text=True)
    assert "1 passed" in result.stdout
    assert result.returncode == 0, (
        f"run-tests failed with exit code {result.returncode}.\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
    )


def test_run_benchmarks():
    cmd = [
        sys.executable,
        "-m",
        "llama_verifications.cli.main",
        "run-benchmarks",
        "--benchmarks=gpqa-cot-diamond",
        "--provider=llama_api",
        "--model=Llama-4-Scout-17B-16E-Instruct-FP8",
        "--num-examples=1",
    ]
    result = subprocess.run(cmd, capture_output=True, text=True)
    assert result.returncode == 0, (
        f"run-benchmarks failed with exit code {result.returncode}.\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
    )
    assert "num_correct" in result.stdout


def test_list_benchmarks():
    cmd = [
        sys.executable,
        "-m",
        "llama_verifications.cli.main",
        "list-benchmarks",
    ]
    result = subprocess.run(cmd, capture_output=True, text=True)
    assert result.returncode == 0, (
        f"list-benchmarks failed with exit code {result.returncode}.\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
    )
    assert "gpqa-cot-diamond" in result.stdout


def test_list_models():
    cmd = [
        sys.executable,
        "-m",
        "llama_verifications.cli.main",
        "list-models",
        "llama_api",
    ]
    result = subprocess.run(cmd, capture_output=True, text=True)
    assert result.returncode == 0, (
        f"list-models failed with exit code {result.returncode}.\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
    )
    assert "Llama-4-Scout-17B-16E-Instruct-FP8" in result.stdout


def test_generate_tests_report():
    cmd = [
        sys.executable,
        "-m",
        "llama_verifications.cli.main",
        "generate-tests-report",
        "--output-file=temp_file.md",
    ]
    result = subprocess.run(cmd, capture_output=True, text=True)
    assert result.returncode == 0, (
        f"generate-test-report failed with exit code {result.returncode}.\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
    )
    # smoke test to check that the file was created
    assert os.path.exists("temp_file.md")
    # cleanup
    os.remove("temp_file.md")


def test_generate_benchmarks_report():
    cmd = [
        sys.executable,
        "-m",
        "llama_verifications.cli.main",
        "generate-benchmarks-report",
        "--output-file=temp_file.md",
    ]
    result = subprocess.run(cmd, capture_output=True, text=True)
    assert result.returncode == 0, (
        f"generate-benchmarks-report failed with exit code {result.returncode}.\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
    )
    # smoke test to check that the file was created
    assert os.path.exists("temp_file.md")
    # cleanup
    os.remove("temp_file.md")
