"""End-to-end integration tests for Crank CI.

Spins up a real crankd daemon in a temp directory, exercises crankc and
crank-ctl against it, and tears everything down. No mocks.

Run with:  python -m pytest tests/test_e2e.py -v
"""

import json
import os
import signal
import socket
import sqlite3
import subprocess
import sys
import tempfile
import textwrap
import threading
import time
import unittest
from pathlib import Path

# How long to wait for daemon startup / job completion
TIMEOUT = 15


def _wait_for_socket(sock_path, timeout=TIMEOUT):
    """Block until the daemon socket appears and accepts connections."""
    deadline = time.time() + timeout
    while time.time() < deadline:
        if Path(sock_path).exists():
            try:
                s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
                s.settimeout(1)
                s.connect(sock_path)
                s.close()
                return True
            except (ConnectionRefusedError, OSError):
                pass
        time.sleep(0.1)
    return False


def _run(cmd, env=None, timeout=TIMEOUT, check=True):
    """Run a CLI command and return CompletedProcess."""
    result = subprocess.run(
        cmd, capture_output=True, text=True, env=env, timeout=timeout
    )
    if check and result.returncode != 0:
        raise AssertionError(
            f"Command failed: {' '.join(cmd)}\n"
            f"returncode={result.returncode}\n"
            f"stdout={result.stdout}\n"
            f"stderr={result.stderr}"
        )
    return result


class CrankE2ETestCase(unittest.TestCase):
    """Base class that starts a crankd daemon in a temp directory."""

    @classmethod
    def setUpClass(cls):
        cls.tmpdir = tempfile.mkdtemp(prefix="crank-e2e-")
        cls.home = cls.tmpdir
        cls.jobs_dir = Path(cls.home) / "cfg" / "jobs"
        cls.jobs_dir.mkdir(parents=True)
        (Path(cls.home) / "run").mkdir()

        cls.env = os.environ.copy()
        cls.env["CRANK_HOME"] = cls.home
        # Remove CRANK_REMOTE so everything runs local
        cls.env.pop("CRANK_REMOTE", None)

        # Start crankd as a subprocess
        cls.daemon_proc = subprocess.Popen(
            [sys.executable, "-m", "crank_cli.daemon"],
            env=cls.env,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )

        sock_path = os.path.join(cls.home, "crankd.sock")
        if not _wait_for_socket(sock_path):
            cls.daemon_proc.kill()
            stdout, stderr = cls.daemon_proc.communicate(timeout=5)
            raise RuntimeError(
                f"crankd failed to start within {TIMEOUT}s\n"
                f"stdout: {stdout.decode()}\nstderr: {stderr.decode()}"
            )

    @classmethod
    def tearDownClass(cls):
        # Graceful shutdown via SIGTERM
        cls.daemon_proc.send_signal(signal.SIGTERM)
        try:
            cls.daemon_proc.wait(timeout=10)
        except subprocess.TimeoutExpired:
            cls.daemon_proc.kill()
            cls.daemon_proc.wait(timeout=5)

        # Clean up temp directory
        import shutil
        shutil.rmtree(cls.tmpdir, ignore_errors=True)

    def _create_job(self, name, script):
        """Write a job script into the jobs directory."""
        job_path = self.jobs_dir / f"{name}.run"
        job_path.write_text(textwrap.dedent(script))
        job_path.chmod(0o755)
        return job_path

    def _crankc(self, *args, check=True):
        """Run crankc with the test environment."""
        return _run(
            [sys.executable, "-m", "crank_cli.client"] + list(args),
            env=self.env, check=check,
        )

    def _ctl(self, *args, check=True):
        """Run crank-ctl with the test environment."""
        return _run(
            [sys.executable, "-m", "crank_cli.main", "--crank-home", self.home]
            + list(args),
            env=self.env, check=check,
        )

    def _wait_build(self, job, number, timeout=TIMEOUT):
        """Poll the DB until a build reaches a terminal state."""
        db_path = Path(self.home) / "crank.sqlite"
        deadline = time.time() + timeout
        while time.time() < deadline:
            db = sqlite3.connect(str(db_path))
            row = db.execute(
                "SELECT result FROM builds WHERE name=? AND number=?",
                (job, number),
            ).fetchone()
            db.close()
            if row and row[0] not in (0, 1, 2):
                return row[0]
            time.sleep(0.2)
        self.fail(f"Build {job} #{number} did not complete within {timeout}s")


class TestDaemonLifecycle(CrankE2ETestCase):
    """Verify the daemon starts, accepts connections, and creates the DB."""

    def test_socket_exists(self):
        sock_path = Path(self.home) / "crankd.sock"
        self.assertTrue(sock_path.exists())

    def test_db_exists(self):
        db_path = Path(self.home) / "crank.sqlite"
        self.assertTrue(db_path.exists())

    def test_db_has_builds_table(self):
        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        row = db.execute(
            "SELECT name FROM sqlite_master WHERE type='table' AND name='builds'"
        ).fetchone()
        db.close()
        self.assertIsNotNone(row)


class TestCrankcProtocol(CrankE2ETestCase):
    """Test crankc against a live daemon."""

    def test_queue_job(self):
        self._create_job("e2e-queue", """\
            #!/bin/bash
            echo "queued job ran"
        """)
        result = self._crankc("queue", "e2e-queue")
        # crankc prints the build number on success
        number = int(result.stdout.strip())
        self.assertGreaterEqual(number, 1)

        # Wait for completion and verify
        code = self._wait_build("e2e-queue", number)
        self.assertEqual(code, 5)  # success

    def test_blocking_run(self):
        self._create_job("e2e-blocking", """\
            #!/bin/bash
            echo "blocking output"
        """)
        result = self._crankc("run", "e2e-blocking")
        number = int(result.stdout.strip())
        self.assertGreaterEqual(number, 1)

    def test_run_with_env_vars(self):
        self._create_job("e2e-env", """\
            #!/bin/bash
            echo "GREETING=$GREETING"
        """)
        result = self._crankc("run", "e2e-env", "GREETING=hello")
        number = int(result.stdout.strip())

        # Check the output in the DB
        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        row = db.execute(
            "SELECT output FROM builds WHERE name='e2e-env' AND number=?",
            (number,),
        ).fetchone()
        db.close()
        self.assertIn("GREETING=hello", row[0])

    def test_run_nonexistent_job(self):
        result = self._crankc("run", "no-such-job", check=False)
        self.assertNotEqual(result.returncode, 0)
        self.assertIn("not found", result.stderr.lower())

    def test_queue_then_abort(self):
        self._create_job("e2e-slow", """\
            #!/bin/bash
            sleep 60
        """)
        result = self._crankc("queue", "e2e-slow")
        number = int(result.stdout.strip())

        # Wait for it to start running
        deadline = time.time() + TIMEOUT
        while time.time() < deadline:
            db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
            row = db.execute(
                "SELECT result FROM builds WHERE name='e2e-slow' AND number=?",
                (number,),
            ).fetchone()
            db.close()
            if row and row[0] == 2:  # running
                break
            time.sleep(0.2)

        try:
            # Abort it
            result = self._crankc("abort", "e2e-slow", str(number))
            self.assertEqual(result.returncode, 0)

            # Verify it was aborted
            code = self._wait_build("e2e-slow", number)
            self.assertEqual(code, 3)  # aborted
        except Exception:
            # Ensure cleanup even if assertions fail
            self._crankc("abort", "e2e-slow", str(number), check=False)
            self._wait_build("e2e-slow", number)
            raise

    def test_failed_job(self):
        self._create_job("e2e-fail", """\
            #!/bin/bash
            echo "failing now"
            exit 1
        """)
        result = self._crankc("run", "e2e-fail", check=False)
        self.assertNotEqual(result.returncode, 0)


class TestCrankCtlCommands(CrankE2ETestCase):
    """Test crank-ctl commands against a live daemon."""

    def test_list(self):
        self._create_job("e2e-listjob", """\
            #!/bin/bash
            echo "listed"
        """)
        result = self._ctl("list")
        self.assertIn("e2e-listjob", result.stdout)

    def test_list_json(self):
        self._create_job("e2e-listjson", """\
            #!/bin/bash
            echo ok
        """)
        result = self._ctl("--json", "list")
        data = json.loads(result.stdout)
        self.assertIsInstance(data, list)
        names = [j["name"] for j in data]
        self.assertIn("e2e-listjson", names)

    def test_trigger_blocking(self):
        self._create_job("e2e-trigger", """\
            #!/bin/bash
            echo "triggered"
        """)
        result = self._ctl("trigger", "e2e-trigger")
        self.assertEqual(result.returncode, 0)

    def test_trigger_queue(self):
        self._create_job("e2e-trigq", """\
            #!/bin/bash
            echo "queued trigger"
        """)
        result = self._ctl("trigger", "e2e-trigq", "-q")
        self.assertEqual(result.returncode, 0)

    def test_trigger_with_env(self):
        self._create_job("e2e-trigenv", """\
            #!/bin/bash
            echo "VER=$VERSION"
        """)
        result = self._ctl("trigger", "e2e-trigenv", "VERSION=2.0")
        self.assertEqual(result.returncode, 0)

    def test_cat(self):
        self._create_job("e2e-catjob", """\
            #!/bin/bash
            echo "cat me"
        """)
        result = self._ctl("cat", "e2e-catjob")
        self.assertIn("cat me", result.stdout)

    def test_cat_not_found(self):
        result = self._ctl("cat", "no-such-job", check=False)
        self.assertNotEqual(result.returncode, 0)

    def test_logs(self):
        self._create_job("e2e-logjob", """\
            #!/bin/bash
            echo "log output line"
        """)
        # Trigger a run first
        self._crankc("run", "e2e-logjob")
        # Now check logs
        result = self._ctl("logs", "e2e-logjob")
        self.assertIn("log output line", result.stdout)

    def test_logs_json(self):
        self._create_job("e2e-logjson", """\
            #!/bin/bash
            echo "json log"
        """)
        self._crankc("run", "e2e-logjson")
        result = self._ctl("--json", "logs", "e2e-logjson")
        data = json.loads(result.stdout)
        self.assertIsInstance(data, list)
        self.assertGreater(len(data), 0)
        self.assertIn("json log", data[0]["output"])

    def test_history(self):
        self._create_job("e2e-histjob", """\
            #!/bin/bash
            echo "hist"
        """)
        self._crankc("run", "e2e-histjob")
        result = self._ctl("history", "e2e-histjob")
        self.assertIn("success", result.stdout)

    def test_history_json(self):
        self._create_job("e2e-histjson", """\
            #!/bin/bash
            echo ok
        """)
        self._crankc("run", "e2e-histjson")
        result = self._ctl("--json", "history", "e2e-histjson")
        data = json.loads(result.stdout)
        self.assertIsInstance(data, list)
        self.assertEqual(data[0]["result"], 5)

    def test_stats(self):
        self._create_job("e2e-statjob", """\
            #!/bin/bash
            echo "stat"
        """)
        self._crankc("run", "e2e-statjob")
        result = self._ctl("stats", "e2e-statjob")
        self.assertIn("Success rate", result.stdout)
        self.assertIn("100.0%", result.stdout)

    def test_stats_json(self):
        self._create_job("e2e-statjson", """\
            #!/bin/bash
            echo ok
        """)
        self._crankc("run", "e2e-statjson")
        result = self._ctl("--json", "stats", "e2e-statjson")
        data = json.loads(result.stdout)
        self.assertEqual(data["success_rate"], 100.0)

    def test_status_no_running(self):
        result = self._ctl("status")
        # Either shows running jobs or "No running or queued jobs."
        self.assertEqual(result.returncode, 0)

    def test_status_with_running_job(self):
        self._create_job("e2e-status-run", """\
            #!/bin/bash
            sleep 30
        """)
        self._crankc("queue", "e2e-status-run")

        # Wait for it to start running
        deadline = time.time() + TIMEOUT
        while time.time() < deadline:
            db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
            row = db.execute(
                "SELECT number, result FROM builds WHERE name='e2e-status-run' "
                "ORDER BY number DESC LIMIT 1"
            ).fetchone()
            db.close()
            if row and row[0] and row[1] == 2:
                break
            time.sleep(0.2)

        try:
            result = self._ctl("status")
            self.assertIn("e2e-status-run", result.stdout)
            self.assertIn("running", result.stdout)
        finally:
            # Cleanup
            self._crankc("abort", "e2e-status-run", str(row[0]), check=False)
            self._wait_build("e2e-status-run", row[0])

    def test_status_json(self):
        self._create_job("e2e-status-json", """\
            #!/bin/bash
            sleep 30
        """)
        self._crankc("queue", "e2e-status-json")

        # Wait for it to start running
        deadline = time.time() + TIMEOUT
        number = None
        while time.time() < deadline:
            db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
            row = db.execute(
                "SELECT number, result FROM builds WHERE name='e2e-status-json' "
                "ORDER BY number DESC LIMIT 1"
            ).fetchone()
            db.close()
            if row and row[1] == 2:
                number = row[0]
                break
            time.sleep(0.2)

        try:
            result = self._ctl("--json", "status")
            data = json.loads(result.stdout)
            self.assertIsInstance(data, list)
            names = [r["name"] for r in data]
            self.assertIn("e2e-status-json", names)
        finally:
            # Cleanup
            self._crankc("abort", "e2e-status-json", str(number), check=False)
            self._wait_build("e2e-status-json", number)

    def test_logs_specific_run(self):
        self._create_job("e2e-logs-run", """\
            #!/bin/bash
            echo "run $RUN output"
        """)
        # Run twice
        self._crankc("run", "e2e-logs-run")
        r2 = self._crankc("run", "e2e-logs-run")
        number2 = int(r2.stdout.strip())

        result = self._ctl("logs", "e2e-logs-run", "--run", str(number2))
        self.assertIn(f"#{number2}", result.stdout)

    def test_logs_last_n(self):
        self._create_job("e2e-logs-last", """\
            #!/bin/bash
            echo "log entry"
        """)
        # Run 3 times
        for _ in range(3):
            self._crankc("run", "e2e-logs-last")

        result = self._ctl("logs", "e2e-logs-last", "-n", "2")
        # Should show 2 run headers
        self.assertEqual(result.stdout.count("==="), 4)  # 2 runs x 2 delimiters

    def test_logs_no_runs(self):
        self._create_job("e2e-logs-empty", """\
            #!/bin/bash
            echo never
        """)
        result = self._ctl("logs", "e2e-logs-empty")
        self.assertIn("No runs found", result.stdout)

    def test_history_last_n(self):
        self._create_job("e2e-hist-last", """\
            #!/bin/bash
            echo ok
        """)
        for _ in range(5):
            self._crankc("run", "e2e-hist-last")

        result = self._ctl("history", "e2e-hist-last", "-n", "3")
        # Should show exactly 3 data rows (plus header + separator)
        lines = [l for l in result.stdout.strip().splitlines() if l.strip()]
        # header + separator + 3 data lines = 5
        self.assertEqual(len(lines), 5)

    def test_history_no_runs(self):
        self._create_job("e2e-hist-empty", """\
            #!/bin/bash
            echo never
        """)
        result = self._ctl("history", "e2e-hist-empty")
        self.assertIn("No runs found", result.stdout)

    def test_diff_local(self):
        self._create_job("e2e-diff-local", """\
            #!/bin/bash
            echo remote
        """)
        local_file = os.path.join(self.tmpdir, "local-script.sh")
        with open(local_file, "w") as f:
            f.write("#!/bin/bash\necho local\n")

        result = self._ctl("diff", "e2e-diff-local", "--local", local_file)
        self.assertIn("remote", result.stdout)
        self.assertIn("local", result.stdout)

    def test_diff_identical(self):
        self._create_job("e2e-diff-a", """\
            #!/bin/bash
            echo same
        """)
        self._create_job("e2e-diff-b", """\
            #!/bin/bash
            echo same
        """)
        result = self._ctl("diff", "e2e-diff-a", "e2e-diff-b")
        self.assertIn("No differences", result.stdout)

    def test_diff_different(self):
        self._create_job("e2e-diff-c", """\
            #!/bin/bash
            echo alpha
        """)
        self._create_job("e2e-diff-d", """\
            #!/bin/bash
            echo beta
        """)
        result = self._ctl("diff", "e2e-diff-c", "e2e-diff-d")
        self.assertIn("alpha", result.stdout)
        self.assertIn("beta", result.stdout)

    def test_disable_enable(self):
        self._create_job("e2e-toggle", """\
            #!/bin/bash
            echo toggle
        """)
        # Disable
        self._ctl("disable", "e2e-toggle")
        self.assertTrue((self.jobs_dir / "e2e-toggle.run.disabled").exists())
        self.assertFalse((self.jobs_dir / "e2e-toggle.run").exists())

        # Enable
        self._ctl("enable", "e2e-toggle")
        self.assertTrue((self.jobs_dir / "e2e-toggle.run").exists())
        self.assertFalse((self.jobs_dir / "e2e-toggle.run.disabled").exists())

    def test_create_and_delete(self):
        # Create
        self._ctl("create", "e2e-newjob")
        self.assertTrue((self.jobs_dir / "e2e-newjob.run").exists())
        content = (self.jobs_dir / "e2e-newjob.run").read_text()
        self.assertIn("#!/bin/bash", content)

        # Delete (without --yes should not delete)
        self._ctl("delete", "e2e-newjob")
        self.assertTrue((self.jobs_dir / "e2e-newjob.run").exists())

        # Delete with --yes
        self._ctl("delete", "e2e-newjob", "-y")
        self.assertFalse((self.jobs_dir / "e2e-newjob.run").exists())

    def test_create_from_template(self):
        self._create_job("e2e-template", """\
            #!/bin/bash
            echo "template content"
        """)
        self._ctl("create", "e2e-fromtpl", "-t", "e2e-template")
        content = (self.jobs_dir / "e2e-fromtpl.run").read_text()
        self.assertIn("template content", content)

    def test_create_already_exists(self):
        self._create_job("e2e-exists", """\
            #!/bin/bash
            echo exists
        """)
        result = self._ctl("create", "e2e-exists", check=False)
        self.assertNotEqual(result.returncode, 0)
        self.assertIn("already exists", result.stderr)

    def test_invalid_job_name_rejected(self):
        result = self._ctl("cat", "bad;name", check=False)
        self.assertNotEqual(result.returncode, 0)
        self.assertIn("Invalid job name", result.stderr)

    def test_negative_last_rejected(self):
        result = self._ctl("history", "anyjob", "-n", "-1", check=False)
        self.assertNotEqual(result.returncode, 0)

    def test_export(self):
        self._create_job("e2e-exportjob", """\
            #!/bin/bash
            echo export
        """)
        output_path = os.path.join(self.tmpdir, "backup.tar.gz")
        result = self._ctl("export", "-o", output_path)
        self.assertTrue(os.path.exists(output_path))
        self.assertGreater(os.path.getsize(output_path), 0)


class TestRetryCommand(CrankE2ETestCase):
    """Test crank-ctl retry."""

    def test_retry_failed_job(self):
        self._create_job("e2e-retry-fail", """\
            #!/bin/bash
            if [ "$RUN" = "1" ]; then exit 1; fi
            echo "retried successfully"
        """)
        # First run fails
        result = self._crankc("run", "e2e-retry-fail", check=False)
        self.assertNotEqual(result.returncode, 0)

        # Retry should succeed (RUN=2)
        result = self._ctl("retry", "e2e-retry-fail")
        self.assertEqual(result.returncode, 0)

        # Verify the retry actually ran
        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        row = db.execute(
            "SELECT result, output FROM builds WHERE name='e2e-retry-fail' "
            "ORDER BY number DESC LIMIT 1"
        ).fetchone()
        db.close()
        self.assertEqual(row[0], 5)  # success
        self.assertIn("retried successfully", row[1])

    def test_retry_specific_run(self):
        self._create_job("e2e-retry-spec", """\
            #!/bin/bash
            if [ "$RUN" = "1" ]; then exit 1; fi
            echo "retry-spec ok"
        """)
        r1 = self._crankc("run", "e2e-retry-spec", check=False)

        result = self._ctl("retry", "e2e-retry-spec", "--run", "1")
        self.assertEqual(result.returncode, 0)

    def test_retry_no_failed_build(self):
        self._create_job("e2e-retry-nofail", """\
            #!/bin/bash
            echo ok
        """)
        self._crankc("run", "e2e-retry-nofail")

        result = self._ctl("retry", "e2e-retry-nofail", check=False)
        self.assertNotEqual(result.returncode, 0)


class TestAbortAutoFind(CrankE2ETestCase):
    """Test crank-ctl abort without --run (auto-finds latest running build)."""

    def test_abort_auto_find(self):
        self._create_job("e2e-abort-auto", """\
            #!/bin/bash
            sleep 60
        """)
        self._crankc("queue", "e2e-abort-auto")

        # Wait for running
        deadline = time.time() + TIMEOUT
        number = None
        while time.time() < deadline:
            db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
            row = db.execute(
                "SELECT number, result FROM builds WHERE name='e2e-abort-auto' "
                "ORDER BY number DESC LIMIT 1"
            ).fetchone()
            db.close()
            if row and row[1] == 2:
                number = row[0]
                break
            time.sleep(0.2)

        self.assertIsNotNone(number)

        try:
            # Abort without specifying --run
            result = self._ctl("abort", "e2e-abort-auto")
            self.assertEqual(result.returncode, 0)

            code = self._wait_build("e2e-abort-auto", number)
            self.assertEqual(code, 3)
        except Exception:
            self._crankc("abort", "e2e-abort-auto", str(number), check=False)
            self._wait_build("e2e-abort-auto", number)
            raise

    def test_abort_nothing_running(self):
        self._create_job("e2e-abort-none", """\
            #!/bin/bash
            echo quick
        """)
        self._crankc("run", "e2e-abort-none")

        result = self._ctl("abort", "e2e-abort-none", check=False)
        self.assertNotEqual(result.returncode, 0)


class TestConfigCommand(CrankE2ETestCase):
    """Test crank-ctl config."""

    def test_config_view(self):
        self._create_job("e2e-cfgjob", """\
            #!/bin/bash
            echo cfg
        """)
        # Create a .conf file
        conf_path = self.jobs_dir / "e2e-cfgjob.conf"
        conf_path.write_text("TIMEOUT=30\nRETRIES=3\n")

        result = self._ctl("config", "e2e-cfgjob")
        self.assertIn("TIMEOUT=30", result.stdout)
        self.assertIn("RETRIES=3", result.stdout)

    def test_config_env_type(self):
        self._create_job("e2e-cfgenv", """\
            #!/bin/bash
            echo env
        """)
        env_path = self.jobs_dir / "e2e-cfgenv.env"
        env_path.write_text("FOO=bar\nBAZ=qux\n")

        result = self._ctl("config", "e2e-cfgenv", "-t", "env")
        self.assertIn("FOO=bar", result.stdout)

    def test_config_not_found(self):
        self._create_job("e2e-cfgmissing", """\
            #!/bin/bash
            echo missing
        """)
        result = self._ctl("config", "e2e-cfgmissing", check=False)
        self.assertNotEqual(result.returncode, 0)


class TestCommandAliases(CrankE2ETestCase):
    """Test that command aliases work."""

    def test_ls_alias(self):
        self._create_job("e2e-alias-ls", """\
            #!/bin/bash
            echo alias
        """)
        result = self._ctl("ls")
        self.assertIn("e2e-alias-ls", result.stdout)

    def test_st_alias(self):
        result = self._ctl("st")
        self.assertEqual(result.returncode, 0)

    def test_run_alias(self):
        self._create_job("e2e-alias-run", """\
            #!/bin/bash
            echo "run alias"
        """)
        result = self._ctl("run", "e2e-alias-run")
        self.assertEqual(result.returncode, 0)

    def test_log_alias(self):
        self._create_job("e2e-alias-log", """\
            #!/bin/bash
            echo "log alias"
        """)
        self._crankc("run", "e2e-alias-log")
        result = self._ctl("log", "e2e-alias-log")
        self.assertIn("log alias", result.stdout)

    def test_hist_alias(self):
        self._create_job("e2e-alias-hist", """\
            #!/bin/bash
            echo hist
        """)
        self._crankc("run", "e2e-alias-hist")
        result = self._ctl("hist", "e2e-alias-hist")
        self.assertIn("success", result.stdout)

    def test_rm_alias(self):
        self._create_job("e2e-alias-rm", """\
            #!/bin/bash
            echo rm
        """)
        result = self._ctl("rm", "e2e-alias-rm", "-y")
        self.assertEqual(result.returncode, 0)
        self.assertFalse((self.jobs_dir / "e2e-alias-rm.run").exists())

    def test_cfg_alias(self):
        self._create_job("e2e-alias-cfg", """\
            #!/bin/bash
            echo cfg
        """)
        (self.jobs_dir / "e2e-alias-cfg.conf").write_text("KEY=val\n")
        result = self._ctl("cfg", "e2e-alias-cfg")
        self.assertIn("KEY=val", result.stdout)

    def test_backup_alias(self):
        output_path = os.path.join(self.tmpdir, "backup-alias.tar.gz")
        result = self._ctl("backup", "-o", output_path)
        self.assertTrue(os.path.exists(output_path))


class TestLogFileStreaming(CrankE2ETestCase):
    """Verify that log files are written during execution, not just after."""

    def test_log_file_exists_after_run(self):
        self._create_job("e2e-logfile", """\
            #!/bin/bash
            echo "line 1"
            echo "line 2"
            echo "line 3"
        """)
        result = self._crankc("run", "e2e-logfile")
        number = int(result.stdout.strip())

        log_path = Path(self.home) / "run" / "e2e-logfile" / str(number) / "log"
        self.assertTrue(log_path.exists(), f"Log file not found: {log_path}")
        content = log_path.read_text()
        self.assertIn("line 1", content)
        self.assertIn("line 2", content)
        self.assertIn("line 3", content)

    def test_log_file_matches_db_output(self):
        self._create_job("e2e-logmatch", """\
            #!/bin/bash
            for i in $(seq 1 5); do echo "iteration $i"; done
        """)
        result = self._crankc("run", "e2e-logmatch")
        number = int(result.stdout.strip())

        # Compare log file to DB output
        log_path = Path(self.home) / "run" / "e2e-logmatch" / str(number) / "log"
        log_content = log_path.read_text()

        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        row = db.execute(
            "SELECT output FROM builds WHERE name='e2e-logmatch' AND number=?",
            (number,),
        ).fetchone()
        db.close()

        self.assertEqual(log_content, row[0])

    def test_state_file_written(self):
        self._create_job("e2e-state", """\
            #!/bin/bash
            echo "state test"
        """)
        self._crankc("run", "e2e-state")

        state_file = Path(self.home) / "e2e-state.last"
        self.assertTrue(state_file.exists())
        self.assertEqual(state_file.read_text(), "success")

    def test_failed_state_file(self):
        self._create_job("e2e-failstate", """\
            #!/bin/bash
            exit 1
        """)
        self._crankc("run", "e2e-failstate", check=False)

        state_file = Path(self.home) / "e2e-failstate.last"
        self.assertTrue(state_file.exists())
        self.assertEqual(state_file.read_text(), "failed")


class TestTailCommand(CrankE2ETestCase):
    """Test crank-ctl tail (limited — it exec's into tail -f)."""

    def test_tail_no_running_build(self):
        self._create_job("e2e-tail-none", """\
            #!/bin/bash
            echo quick
        """)
        self._crankc("run", "e2e-tail-none")

        result = self._ctl("tail", "e2e-tail-none", check=False)
        self.assertNotEqual(result.returncode, 0)
        self.assertIn("No running build", result.stderr)

    def test_tail_log_file_available(self):
        """Verify the log file exists for a completed build so tail would work."""
        self._create_job("e2e-tail-log", """\
            #!/bin/bash
            echo "tail content"
        """)
        result = self._crankc("run", "e2e-tail-log")
        number = int(result.stdout.strip())

        log_path = Path(self.home) / "run" / "e2e-tail-log" / str(number) / "log"
        self.assertTrue(log_path.exists())
        self.assertIn("tail content", log_path.read_text())


class TestConcurrency(CrankE2ETestCase):
    """Test concurrent job execution."""

    def test_multiple_jobs_in_parallel(self):
        """Queue multiple jobs and verify they all complete."""
        for i in range(4):
            self._create_job(f"e2e-par-{i}", f"""\
                #!/bin/bash
                echo "parallel job {i}"
                sleep 0.1
            """)

        numbers = []
        for i in range(4):
            result = self._crankc("queue", f"e2e-par-{i}")
            numbers.append((f"e2e-par-{i}", int(result.stdout.strip())))

        # Wait for all to complete
        for job, number in numbers:
            code = self._wait_build(job, number)
            self.assertEqual(code, 5, f"{job} #{number} should succeed")

    def test_sequential_builds_increment_number(self):
        self._create_job("e2e-seq", """\
            #!/bin/bash
            echo "seq run"
        """)
        numbers = []
        for _ in range(3):
            result = self._crankc("run", "e2e-seq")
            numbers.append(int(result.stdout.strip()))

        # Build numbers should be strictly increasing
        self.assertEqual(numbers, sorted(numbers))
        self.assertEqual(len(set(numbers)), 3)


class TestWorkspaceIsolation(CrankE2ETestCase):
    """Verify each build gets its own workspace."""

    def test_workspace_created(self):
        self._create_job("e2e-workspace", """\
            #!/bin/bash
            echo "workspace=$WORKSPACE"
            echo "job=$JOB"
            echo "run=$RUN"
            touch "$WORKSPACE/artifact.txt"
        """)
        result = self._crankc("run", "e2e-workspace")
        number = int(result.stdout.strip())

        workspace = Path(self.home) / "run" / "e2e-workspace" / str(number)
        self.assertTrue(workspace.exists())
        self.assertTrue((workspace / "artifact.txt").exists())

    def test_env_vars_set(self):
        self._create_job("e2e-envcheck", """\
            #!/bin/bash
            echo "JOB=$JOB"
            echo "RUN=$RUN"
            echo "WORKSPACE=$WORKSPACE"
        """)
        result = self._crankc("run", "e2e-envcheck")
        number = int(result.stdout.strip())

        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        row = db.execute(
            "SELECT output FROM builds WHERE name='e2e-envcheck' AND number=?",
            (number,),
        ).fetchone()
        db.close()

        self.assertIn("JOB=e2e-envcheck", row[0])
        self.assertIn(f"RUN={number}", row[0])
        self.assertIn("WORKSPACE=", row[0])


class TestAbortFlow(CrankE2ETestCase):
    """Test abort via crank-ctl."""

    def test_ctl_abort(self):
        self._create_job("e2e-ctl-abort", """\
            #!/bin/bash
            sleep 60
        """)
        # Queue it
        self._ctl("trigger", "e2e-ctl-abort", "-q")

        # Wait for it to start running
        db_path = Path(self.home) / "crank.sqlite"
        deadline = time.time() + TIMEOUT
        number = None
        while time.time() < deadline:
            db = sqlite3.connect(str(db_path))
            row = db.execute(
                "SELECT number, result FROM builds WHERE name='e2e-ctl-abort' "
                "ORDER BY number DESC LIMIT 1"
            ).fetchone()
            db.close()
            if row and row[1] == 2:
                number = row[0]
                break
            time.sleep(0.2)

        self.assertIsNotNone(number, "Job should have started running")

        try:
            # Abort via crank-ctl
            self._ctl("abort", "e2e-ctl-abort", "--run", str(number))

            # Verify aborted
            code = self._wait_build("e2e-ctl-abort", number)
            self.assertEqual(code, 3)
        except Exception:
            self._crankc("abort", "e2e-ctl-abort", str(number), check=False)
            self._wait_build("e2e-ctl-abort", number)
            raise


class TestWaitCommand(CrankE2ETestCase):
    """Test crank-ctl wait."""

    def test_wait_for_queued_job(self):
        self._create_job("e2e-wait", """\
            #!/bin/bash
            sleep 0.5
            echo "waited"
        """)
        # Queue it
        qresult = self._crankc("queue", "e2e-wait")
        number = int(qresult.stdout.strip())

        # Wait for it
        result = self._ctl("wait", "e2e-wait", "--run", str(number))
        self.assertIn("completed", result.stdout)
        self.assertIn("success", result.stdout)

    def test_wait_timeout(self):
        self._create_job("e2e-waittimeout", """\
            #!/bin/bash
            sleep 60
        """)
        self._crankc("queue", "e2e-waittimeout")

        try:
            result = self._ctl(
                "wait", "e2e-waittimeout", "-T", "2", check=False
            )
            self.assertEqual(result.returncode, 2)
            self.assertIn("timeout", result.stderr.lower())
        finally:
            # Clean up: abort the sleeping job
            db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
            row = db.execute(
                "SELECT number FROM builds WHERE name='e2e-waittimeout' "
                "AND result IN (1, 2) ORDER BY number DESC LIMIT 1"
            ).fetchone()
            db.close()
            if row:
                self._crankc("abort", "e2e-waittimeout", str(row[0]), check=False)
                self._wait_build("e2e-waittimeout", row[0])


class TestDaemonProtocolEdgeCases(CrankE2ETestCase):
    """Test raw socket protocol edge cases against the live daemon."""

    def _raw_send(self, data, timeout=5):
        """Send raw bytes to the daemon socket and return the response."""
        sock_path = os.path.join(self.home, "crankd.sock")
        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        sock.settimeout(timeout)
        sock.connect(sock_path)
        if data:
            sock.sendall(data)
        sock.shutdown(socket.SHUT_WR)
        response = b""
        try:
            while True:
                chunk = sock.recv(4096)
                if not chunk:
                    break
                response += chunk
        except socket.timeout:
            pass
        finally:
            sock.close()
        return response.decode("utf-8", errors="replace").strip()

    def test_empty_connection(self):
        """Connect and immediately close — daemon should handle gracefully."""
        resp = self._raw_send(b"")
        # Empty line = no response (daemon just closes connection)
        self.assertEqual(resp, "")

    def test_empty_line(self):
        """Send just a newline."""
        resp = self._raw_send(b"\n")
        self.assertEqual(resp, "")

    def test_run_missing_job_name(self):
        resp = self._raw_send(b"run\n")
        self.assertTrue(resp.startswith("ERR"))
        self.assertIn("missing job name", resp)

    def test_queue_missing_job_name(self):
        resp = self._raw_send(b"queue\n")
        self.assertTrue(resp.startswith("ERR"))
        self.assertIn("missing job name", resp)

    def test_abort_missing_args(self):
        resp = self._raw_send(b"abort\n")
        self.assertTrue(resp.startswith("ERR"))
        self.assertIn("usage", resp)

    def test_abort_missing_number(self):
        resp = self._raw_send(b"abort somejob\n")
        self.assertTrue(resp.startswith("ERR"))
        self.assertIn("usage", resp)

    def test_abort_invalid_number(self):
        resp = self._raw_send(b"abort somejob notanumber\n")
        self.assertTrue(resp.startswith("ERR"))
        self.assertIn("invalid build number", resp)

    def test_abort_nothing_to_abort(self):
        resp = self._raw_send(b"abort nonexistent 999\n")
        self.assertTrue(resp.startswith("ERR"))
        self.assertIn("no running or queued build found", resp)

    def test_unknown_command(self):
        resp = self._raw_send(b"foobar test\n")
        self.assertTrue(resp.startswith("ERR"))
        self.assertIn("unknown command", resp)

    def test_request_too_large(self):
        """Send more than 64KB without a newline."""
        big_data = b"x" * 70000
        try:
            resp = self._raw_send(big_data)
        except ConnectionResetError:
            # Daemon may reset the connection after sending ERR — that's fine
            return
        self.assertTrue(resp.startswith("ERR"))
        self.assertIn("too large", resp)

    def test_run_invalid_job_name(self):
        resp = self._raw_send(b"run bad;name\n")
        self.assertTrue(resp.startswith("ERR"))
        self.assertIn("Invalid job name", resp)

    def test_blocking_run_failed_job(self):
        """Blocking run of a failing job returns ERR with build number."""
        self._create_job("e2e-proto-fail", """\
            #!/bin/bash
            exit 1
        """)
        resp = self._raw_send(b"run e2e-proto-fail\n")
        self.assertTrue(resp.startswith("ERR"))
        # Should contain the build number and "failed"
        parts = resp.split()
        self.assertEqual(parts[0], "ERR")
        self.assertTrue(parts[1].isdigit())
        self.assertEqual(parts[2], "failed")


class TestClientEdgeCases(CrankE2ETestCase):
    """Test crankc edge cases."""

    def test_empty_response_from_daemon(self):
        """Simulate empty response using a fake socket server."""
        with tempfile.TemporaryDirectory() as tmpdir:
            sock_path = os.path.join(tmpdir, "crankd.sock")
            ready = threading.Event()

            def fake_server():
                srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
                srv.bind(sock_path)
                srv.listen(1)
                ready.set()
                conn, _ = srv.accept()
                conn.recv(4096)
                conn.close()  # close without sending anything
                srv.close()

            t = threading.Thread(target=fake_server, daemon=True)
            t.start()
            ready.wait(timeout=5)

            env = self.env.copy()
            env["CRANK_HOME"] = tmpdir
            result = _run(
                [sys.executable, "-m", "crank_cli.client", "run", "test"],
                env=env, check=False,
            )
            self.assertNotEqual(result.returncode, 0)
            self.assertIn("no response", result.stderr)


class TestDisplayFormatting(CrankE2ETestCase):
    """Test that display output is correctly formatted (not just JSON)."""

    def test_list_table_format(self):
        self._create_job("e2e-fmt-list", """\
            #!/bin/bash
            echo ok
        """)
        self._crankc("run", "e2e-fmt-list")
        result = self._ctl("list")
        # Should have header row with column names
        self.assertIn("Job", result.stdout)
        self.assertIn("Result", result.stdout)
        self.assertIn("e2e-fmt-list", result.stdout)
        self.assertIn("success", result.stdout)
        # Should have separator line
        self.assertIn("---", result.stdout)

    def test_logs_display_format(self):
        self._create_job("e2e-fmt-logs", """\
            #!/bin/bash
            echo "formatted output"
        """)
        self._crankc("run", "e2e-fmt-logs")
        result = self._ctl("logs", "e2e-fmt-logs")
        # Should have the run header
        self.assertIn("===", result.stdout)
        self.assertIn("e2e-fmt-logs", result.stdout)
        self.assertIn("success", result.stdout)
        self.assertIn("Started:", result.stdout)
        self.assertIn("Completed:", result.stdout)
        self.assertIn("formatted output", result.stdout)

    def test_history_table_format(self):
        self._create_job("e2e-fmt-hist", """\
            #!/bin/bash
            echo ok
        """)
        self._crankc("run", "e2e-fmt-hist")
        result = self._ctl("history", "e2e-fmt-hist")
        self.assertIn("#", result.stdout)
        self.assertIn("Result", result.stdout)
        self.assertIn("Started", result.stdout)
        self.assertIn("Duration", result.stdout)
        self.assertIn("success", result.stdout)

    def test_stats_display_format(self):
        self._create_job("e2e-fmt-stats", """\
            #!/bin/bash
            echo ok
        """)
        self._crankc("run", "e2e-fmt-stats")
        result = self._ctl("stats", "e2e-fmt-stats")
        self.assertIn("Statistics for", result.stdout)
        self.assertIn("Total runs:", result.stdout)
        self.assertIn("Success:", result.stdout)
        self.assertIn("Failed:", result.stdout)
        self.assertIn("Success rate:", result.stdout)
        self.assertIn("Avg duration:", result.stdout)

    def test_status_table_with_running(self):
        self._create_job("e2e-fmt-status", """\
            #!/bin/bash
            sleep 30
        """)
        self._crankc("queue", "e2e-fmt-status")

        # Wait for running
        deadline = time.time() + TIMEOUT
        number = None
        while time.time() < deadline:
            db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
            row = db.execute(
                "SELECT number, result FROM builds WHERE name='e2e-fmt-status' "
                "ORDER BY number DESC LIMIT 1"
            ).fetchone()
            db.close()
            if row and row[1] == 2:
                number = row[0]
                break
            time.sleep(0.2)

        try:
            result = self._ctl("status")
            self.assertIn("Job", result.stdout)
            self.assertIn("Status", result.stdout)
            self.assertIn("e2e-fmt-status", result.stdout)
        finally:
            # Cleanup
            self._crankc("abort", "e2e-fmt-status", str(number), check=False)
            self._wait_build("e2e-fmt-status", number)

    def test_wait_output_format(self):
        self._create_job("e2e-fmt-wait", """\
            #!/bin/bash
            echo ok
        """)
        r = self._crankc("run", "e2e-fmt-wait")
        number = int(r.stdout.strip())

        # Wait for already-completed build
        result = self._ctl("wait", "e2e-fmt-wait", "--run", str(number))
        self.assertIn("completed", result.stdout)
        self.assertIn("success", result.stdout)

    def test_abort_output(self):
        self._create_job("e2e-fmt-abort", """\
            #!/bin/bash
            sleep 30
        """)
        self._crankc("queue", "e2e-fmt-abort")

        deadline = time.time() + TIMEOUT
        number = None
        while time.time() < deadline:
            db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
            row = db.execute(
                "SELECT number, result FROM builds WHERE name='e2e-fmt-abort' "
                "ORDER BY number DESC LIMIT 1"
            ).fetchone()
            db.close()
            if row and row[1] == 2:
                number = row[0]
                break
            time.sleep(0.2)

        try:
            result = self._ctl("abort", "e2e-fmt-abort")
            self.assertIn("Aborting", result.stdout)
        finally:
            self._crankc("abort", "e2e-fmt-abort", str(number), check=False)
            self._wait_build("e2e-fmt-abort", number)

    def test_retry_output(self):
        self._create_job("e2e-fmt-retry", """\
            #!/bin/bash
            exit 1
        """)
        self._crankc("run", "e2e-fmt-retry", check=False)

        # Retry re-runs the failing job, so it exits non-zero
        result = self._ctl("retry", "e2e-fmt-retry", check=False)
        self.assertIn("Retrying", result.stdout)

    def test_create_output(self):
        result = self._ctl("create", "e2e-fmt-create")
        self.assertIn("Created", result.stdout)

    def test_delete_dry_run_output(self):
        self._create_job("e2e-fmt-del", """\
            #!/bin/bash
            echo del
        """)
        result = self._ctl("delete", "e2e-fmt-del")
        self.assertIn("Would delete", result.stdout)
        self.assertIn("--yes", result.stdout)

    def test_disable_enable_output(self):
        self._create_job("e2e-fmt-toggle", """\
            #!/bin/bash
            echo toggle
        """)
        result = self._ctl("disable", "e2e-fmt-toggle")
        self.assertIn("Disabled", result.stdout)

        result = self._ctl("enable", "e2e-fmt-toggle")
        self.assertIn("Enabled", result.stdout)


class TestRetryWithParams(CrankE2ETestCase):
    """Test that retry preserves env vars from the original build."""

    def test_retry_preserves_env_vars(self):
        self._create_job("e2e-retry-params", """\
            #!/bin/bash
            echo "MY_VAR=$MY_VAR"
            if [ "$RUN" = "1" ]; then exit 1; fi
        """)
        # First run with env var — fails
        self._crankc("run", "e2e-retry-params", "MY_VAR=preserved", check=False)

        # Retry should replay the env var
        self._ctl("retry", "e2e-retry-params")

        # Check the retried build has the env var in output
        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        row = db.execute(
            "SELECT output, params FROM builds WHERE name='e2e-retry-params' "
            "ORDER BY number DESC LIMIT 1"
        ).fetchone()
        db.close()
        self.assertIn("MY_VAR=preserved", row[0])

    def test_params_stored_in_db(self):
        self._create_job("e2e-params-store", """\
            #!/bin/bash
            echo ok
        """)
        self._crankc("run", "e2e-params-store", "KEY1=val1", "KEY2=val2")

        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        row = db.execute(
            "SELECT params FROM builds WHERE name='e2e-params-store' "
            "ORDER BY number DESC LIMIT 1"
        ).fetchone()
        db.close()
        params = json.loads(row[0])
        self.assertEqual(params["KEY1"], "val1")
        self.assertEqual(params["KEY2"], "val2")


class TestPipelineChainingE2E(CrankE2ETestCase):
    """Test pipeline chaining via on_success/on_failure in .conf files."""

    def test_on_success_triggers_downstream(self):
        self._create_job("e2e-pipe-a", """\
            #!/bin/bash
            echo "parent job"
        """)
        self._create_job("e2e-pipe-b", """\
            #!/bin/bash
            echo "downstream job"
        """)
        # Write conf for pipe-a
        (self.jobs_dir / "e2e-pipe-a.conf").write_text("on_success=e2e-pipe-b\n")

        # Run pipe-a
        result = self._crankc("run", "e2e-pipe-a")
        number_a = int(result.stdout.strip())

        # Wait for pipe-b to complete (triggered by pipeline)
        deadline = time.time() + TIMEOUT
        while time.time() < deadline:
            db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
            row = db.execute(
                "SELECT number, result, parentJob, parentBuild FROM builds "
                "WHERE name='e2e-pipe-b' ORDER BY number DESC LIMIT 1"
            ).fetchone()
            db.close()
            if row and row[1] not in (0, 1, 2):
                break
            time.sleep(0.3)

        self.assertIsNotNone(row)
        self.assertEqual(row[1], 5)  # success
        self.assertEqual(row[2], "e2e-pipe-a")
        self.assertEqual(row[3], number_a)

    def test_on_failure_triggers_downstream(self):
        self._create_job("e2e-pipe-fail", """\
            #!/bin/bash
            exit 1
        """)
        self._create_job("e2e-pipe-alert", """\
            #!/bin/bash
            echo "alert fired"
        """)
        (self.jobs_dir / "e2e-pipe-fail.conf").write_text("on_failure=e2e-pipe-alert\n")

        self._crankc("run", "e2e-pipe-fail", check=False)

        # Wait for alert to complete
        deadline = time.time() + TIMEOUT
        row = None
        while time.time() < deadline:
            db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
            row = db.execute(
                "SELECT result, parentJob FROM builds "
                "WHERE name='e2e-pipe-alert' ORDER BY number DESC LIMIT 1"
            ).fetchone()
            db.close()
            if row and row[0] not in (0, 1, 2):
                break
            time.sleep(0.3)

        self.assertIsNotNone(row)
        self.assertEqual(row[0], 5)  # alert succeeded
        self.assertEqual(row[1], "e2e-pipe-fail")

    def test_env_vars_inherited_by_downstream(self):
        self._create_job("e2e-pipe-parent", """\
            #!/bin/bash
            echo "parent"
        """)
        self._create_job("e2e-pipe-child", """\
            #!/bin/bash
            echo "DEPLOY_ENV=$DEPLOY_ENV"
        """)
        (self.jobs_dir / "e2e-pipe-parent.conf").write_text("on_success=e2e-pipe-child\n")

        self._crankc("run", "e2e-pipe-parent", "DEPLOY_ENV=staging")

        # Wait for child
        deadline = time.time() + TIMEOUT
        row = None
        while time.time() < deadline:
            db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
            row = db.execute(
                "SELECT result, output FROM builds "
                "WHERE name='e2e-pipe-child' ORDER BY number DESC LIMIT 1"
            ).fetchone()
            db.close()
            if row and row[0] not in (0, 1, 2):
                break
            time.sleep(0.3)

        self.assertIsNotNone(row)
        self.assertIn("DEPLOY_ENV=staging", row[1])

    def test_no_cascade_without_conf(self):
        self._create_job("e2e-pipe-solo", """\
            #!/bin/bash
            echo "solo"
        """)
        self._create_job("e2e-pipe-phantom", """\
            #!/bin/bash
            echo "should not run"
        """)
        # No conf file — no pipeline

        result = self._crankc("run", "e2e-pipe-solo")
        number = int(result.stdout.strip())

        # Wait for the parent job to actually complete (terminal state)
        self._wait_build("e2e-pipe-solo", number)

        # Now verify the phantom job was never triggered
        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        row = db.execute(
            "SELECT COUNT(*) FROM builds WHERE name='e2e-pipe-phantom'"
        ).fetchone()
        db.close()
        self.assertEqual(row[0], 0)

    def test_history_shows_parent(self):
        self._create_job("e2e-hist-parent", """\
            #!/bin/bash
            echo ok
        """)
        self._create_job("e2e-hist-child", """\
            #!/bin/bash
            echo ok
        """)
        (self.jobs_dir / "e2e-hist-parent.conf").write_text("on_success=e2e-hist-child\n")

        self._crankc("run", "e2e-hist-parent")

        # Wait for child
        deadline = time.time() + TIMEOUT
        while time.time() < deadline:
            db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
            row = db.execute(
                "SELECT result FROM builds WHERE name='e2e-hist-child' LIMIT 1"
            ).fetchone()
            db.close()
            if row and row[0] not in (0, 1, 2):
                break
            time.sleep(0.3)

        result = self._ctl("history", "e2e-hist-child")
        self.assertIn("e2e-hist-parent", result.stdout)


class TestScheduleCommandE2E(CrankE2ETestCase):
    """Test crank-ctl schedule command."""

    def test_schedule_lists_jobs(self):
        self._create_job("e2e-sched-job", """\
            #!/bin/bash
            echo scheduled
        """)
        (self.jobs_dir / "e2e-sched-job.conf").write_text("schedule=*/5 * * * *\n")

        result = self._ctl("schedule")
        self.assertIn("e2e-sched-job", result.stdout)
        self.assertIn("*/5 * * * *", result.stdout)

    def test_schedule_json(self):
        self._create_job("e2e-sched-json", """\
            #!/bin/bash
            echo ok
        """)
        (self.jobs_dir / "e2e-sched-json.conf").write_text("schedule=0 2 * * *\n")

        result = self._ctl("--json", "schedule")
        data = json.loads(result.stdout)
        names = [j["name"] for j in data]
        self.assertIn("e2e-sched-json", names)

    def test_schedule_empty(self):
        # No jobs with schedules (besides ones from other tests)
        # Create a job with conf but no schedule line
        self._create_job("e2e-sched-nosched", """\
            #!/bin/bash
            echo ok
        """)
        (self.jobs_dir / "e2e-sched-nosched.conf").write_text("timeout=30\n")

        result = self._ctl("schedule")
        # Should not list this job
        self.assertNotIn("e2e-sched-nosched", result.stdout)

    def test_sched_alias(self):
        self._create_job("e2e-sched-alias", """\
            #!/bin/bash
            echo ok
        """)
        (self.jobs_dir / "e2e-sched-alias.conf").write_text("schedule=0 * * * *\n")

        result = self._ctl("sched")
        self.assertIn("e2e-sched-alias", result.stdout)

    def test_disabled_job_not_in_schedule(self):
        self._create_job("e2e-sched-disabled", """\
            #!/bin/bash
            echo ok
        """)
        (self.jobs_dir / "e2e-sched-disabled.conf").write_text("schedule=* * * * *\n")

        # Disable the job
        self._ctl("disable", "e2e-sched-disabled")

        result = self._ctl("schedule")
        self.assertNotIn("e2e-sched-disabled", result.stdout)


class TestJobTimeout(CrankE2ETestCase):
    """Test job timeout via conf directive."""

    def test_job_killed_on_timeout(self):
        self._create_job("e2e-timeout", """\
            #!/bin/bash
            echo "starting"
            sleep 60
        """)
        (self.jobs_dir / "e2e-timeout.conf").write_text("timeout=2\n")

        result = self._crankc("run", "e2e-timeout", check=False)
        self.assertNotEqual(result.returncode, 0)

        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        row = db.execute(
            "SELECT result, output FROM builds WHERE name='e2e-timeout' "
            "ORDER BY number DESC LIMIT 1"
        ).fetchone()
        db.close()
        self.assertEqual(row[0], 4)  # failed
        self.assertIn("timed out", row[1])

    def test_job_completes_within_timeout(self):
        self._create_job("e2e-timeout-ok", """\
            #!/bin/bash
            echo "fast job"
        """)
        (self.jobs_dir / "e2e-timeout-ok.conf").write_text("timeout=30\n")

        result = self._crankc("run", "e2e-timeout-ok")
        number = int(result.stdout.strip())

        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        row = db.execute(
            "SELECT result FROM builds WHERE name='e2e-timeout-ok' AND number=?",
            (number,),
        ).fetchone()
        db.close()
        self.assertEqual(row[0], 5)  # success


class TestBeforeHookE2E(CrankE2ETestCase):
    """Test .before hooks."""

    def test_before_hook_runs(self):
        self._create_job("e2e-before", """\
            #!/bin/bash
            echo "main job"
        """)
        before_path = self.jobs_dir / "e2e-before.before"
        marker = os.path.join(self.tmpdir, "before-marker")
        before_path.write_text(f"#!/bin/bash\ntouch {marker}\n")
        before_path.chmod(0o755)

        self._crankc("run", "e2e-before")
        self.assertTrue(os.path.exists(marker))

    def test_before_hook_failure_skips_job(self):
        self._create_job("e2e-before-fail", """\
            #!/bin/bash
            echo "should not run"
        """)
        before_path = self.jobs_dir / "e2e-before-fail.before"
        before_path.write_text("#!/bin/bash\nexit 1\n")
        before_path.chmod(0o755)

        result = self._crankc("run", "e2e-before-fail", check=False)
        self.assertNotEqual(result.returncode, 0)

        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        row = db.execute(
            "SELECT result, output FROM builds WHERE name='e2e-before-fail' "
            "ORDER BY number DESC LIMIT 1"
        ).fetchone()
        db.close()
        self.assertEqual(row[0], 4)
        self.assertIn(".before hook failed", row[1])
        self.assertNotIn("should not run", row[1])


class TestAfterHookE2E(CrankE2ETestCase):
    """Test .after hooks."""

    def test_after_hook_runs(self):
        self._create_job("e2e-after", """\
            #!/bin/bash
            echo "main job"
        """)
        marker = os.path.join(self.tmpdir, "after-marker")
        after_path = self.jobs_dir / "e2e-after.after"
        after_path.write_text(f"#!/bin/bash\ntouch {marker}\n")
        after_path.chmod(0o755)

        self._crankc("run", "e2e-after")
        self.assertTrue(os.path.exists(marker))

    def test_after_hook_gets_result(self):
        self._create_job("e2e-after-env", """\
            #!/bin/bash
            echo "main"
        """)
        result_file = os.path.join(self.tmpdir, "after-result")
        after_path = self.jobs_dir / "e2e-after-env.after"
        after_path.write_text(f"#!/bin/bash\necho $RESULT_NAME > {result_file}\n")
        after_path.chmod(0o755)

        self._crankc("run", "e2e-after-env")
        self.assertTrue(os.path.exists(result_file))
        self.assertIn("success", open(result_file).read())

    def test_after_hook_runs_on_failure(self):
        self._create_job("e2e-after-fail", """\
            #!/bin/bash
            exit 1
        """)
        marker = os.path.join(self.tmpdir, "after-fail-marker")
        after_path = self.jobs_dir / "e2e-after-fail.after"
        after_path.write_text(f"#!/bin/bash\ntouch {marker}\n")
        after_path.chmod(0o755)

        self._crankc("run", "e2e-after-fail", check=False)
        self.assertTrue(os.path.exists(marker))

    def test_after_hook_failure_doesnt_change_result(self):
        self._create_job("e2e-after-hookfail", """\
            #!/bin/bash
            echo "main succeeds"
        """)
        after_path = self.jobs_dir / "e2e-after-hookfail.after"
        after_path.write_text("#!/bin/bash\nexit 1\n")
        after_path.chmod(0o755)

        result = self._crankc("run", "e2e-after-hookfail")
        number = int(result.stdout.strip())

        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        row = db.execute(
            "SELECT result FROM builds WHERE name='e2e-after-hookfail' AND number=?",
            (number,),
        ).fetchone()
        db.close()
        self.assertEqual(row[0], 5)  # still success


class TestTabProtocol(CrankE2ETestCase):
    """Test that env vars with spaces in values work end-to-end via tab protocol."""

    def test_env_var_with_spaces(self):
        """Trigger a job with MSG='hello world' and verify the full value arrives."""
        self._create_job("e2e-tabproto", """\
            #!/bin/bash
            echo "MSG=$MSG"
        """)
        result = self._crankc("run", "e2e-tabproto", "MSG=hello world")
        number = int(result.stdout.strip())

        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        row = db.execute(
            "SELECT output FROM builds WHERE name='e2e-tabproto' AND number=?",
            (number,),
        ).fetchone()
        db.close()
        self.assertIn("MSG=hello world", row[0])

    def test_multiple_env_vars_with_spaces(self):
        """Multiple env vars, each with spaces in values."""
        self._create_job("e2e-tabproto-multi", """\
            #!/bin/bash
            echo "A=$A"
            echo "B=$B"
        """)
        result = self._crankc("run", "e2e-tabproto-multi", "A=foo bar", "B=baz qux")
        number = int(result.stdout.strip())

        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        row = db.execute(
            "SELECT output FROM builds WHERE name='e2e-tabproto-multi' AND number=?",
            (number,),
        ).fetchone()
        db.close()
        self.assertIn("A=foo bar", row[0])
        self.assertIn("B=baz qux", row[0])

    def test_params_with_spaces_stored_correctly(self):
        """Env var with spaces should be stored as correct JSON in the params column."""
        self._create_job("e2e-tabproto-params", """\
            #!/bin/bash
            echo ok
        """)
        result = self._crankc("run", "e2e-tabproto-params", "LABEL=my job name")
        number = int(result.stdout.strip())

        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        row = db.execute(
            "SELECT params FROM builds WHERE name='e2e-tabproto-params' AND number=?",
            (number,),
        ).fetchone()
        db.close()
        params = json.loads(row[0])
        self.assertEqual(params["LABEL"], "my job name")


class TestAbortResult(CrankE2ETestCase):
    """Test that abort always produces result=3, even when the process traps SIGTERM."""

    def test_abort_trapped_sigterm_still_result_3(self):
        """Job traps SIGTERM and exits 0, but abort should still record result=3."""
        self._create_job("e2e-abort-trap", """\
            #!/bin/bash
            trap 'exit 0' SIGTERM
            sleep 60
        """)
        result = self._crankc("queue", "e2e-abort-trap")
        number = int(result.stdout.strip())

        # Wait for it to start running
        deadline = time.time() + TIMEOUT
        while time.time() < deadline:
            db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
            row = db.execute(
                "SELECT result FROM builds WHERE name='e2e-abort-trap' AND number=?",
                (number,),
            ).fetchone()
            db.close()
            if row and row[0] == 2:  # running
                break
            time.sleep(0.2)
        else:
            self.fail("Job did not start running within timeout")

        try:
            # Abort it
            self._crankc("abort", "e2e-abort-trap", str(number))

            # Verify result is 3 (aborted), not 5 (success)
            code = self._wait_build("e2e-abort-trap", number)
            self.assertEqual(code, 3, "Abort should produce result=3 even when process traps SIGTERM and exits 0")
        except Exception:
            self._crankc("abort", "e2e-abort-trap", str(number), check=False)
            self._wait_build("e2e-abort-trap", number)
            raise

    def test_abort_normal_job_result_3(self):
        """Abort a running job (no SIGTERM trap) — basic abort result check."""
        self._create_job("e2e-abort-normal", """\
            #!/bin/bash
            sleep 60
        """)
        result = self._crankc("queue", "e2e-abort-normal")
        number = int(result.stdout.strip())

        # Wait for it to start running
        deadline = time.time() + TIMEOUT
        while time.time() < deadline:
            db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
            row = db.execute(
                "SELECT result FROM builds WHERE name='e2e-abort-normal' AND number=?",
                (number,),
            ).fetchone()
            db.close()
            if row and row[0] == 2:  # running
                break
            time.sleep(0.2)
        else:
            self.fail("Job did not start running within timeout")

        try:
            # Abort it
            self._crankc("abort", "e2e-abort-normal", str(number))

            code = self._wait_build("e2e-abort-normal", number)
            self.assertEqual(code, 3)
        except Exception:
            self._crankc("abort", "e2e-abort-normal", str(number), check=False)
            self._wait_build("e2e-abort-normal", number)
            raise


class TestStderrForwarding(CrankE2ETestCase):
    """Test that crankc errors are visible through crank-ctl trigger/abort/retry."""

    def test_trigger_nonexistent_job_shows_error(self):
        """crank-ctl trigger on a non-existent job should show crankc's error in stderr."""
        result = self._ctl("trigger", "e2e-no-such-job", check=False)
        self.assertNotEqual(result.returncode, 0)
        # The error from crankc should be forwarded to crank-ctl's stderr
        self.assertIn("not found", result.stderr.lower())

    def test_abort_no_build_shows_error(self):
        """crank-ctl abort with no running/queued build shows crankc's error in stderr."""
        self._create_job("e2e-stderr-abort", """\
            #!/bin/bash
            echo done
        """)
        # Run and wait for completion
        self._crankc("run", "e2e-stderr-abort")

        # Now try to abort (nothing running)
        result = self._ctl("abort", "e2e-stderr-abort", check=False)
        self.assertNotEqual(result.returncode, 0)


class TestRetryWithBadParams(CrankE2ETestCase):
    """Test retry when params column has unexpected (non-dict) JSON data."""

    def test_retry_with_non_dict_params(self):
        """Retry a build whose params column contains a JSON array (not a dict)."""
        self._create_job("e2e-retry-badparams", """\
            #!/bin/bash
            if [ "$RUN" = "1" ]; then exit 1; fi
            echo "retry ok"
        """)
        # First run: fails normally
        self._crankc("run", "e2e-retry-badparams", check=False)

        # Manually overwrite the params column with non-dict JSON
        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        db.execute(
            "UPDATE builds SET params=? WHERE name='e2e-retry-badparams' AND number=1",
            ('["not", "a", "dict"]',),
        )
        db.commit()
        db.close()

        # Retry should succeed (gracefully ignore bad params)
        result = self._ctl("retry", "e2e-retry-badparams")
        self.assertEqual(result.returncode, 0)

        # Verify the retry actually ran
        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        row = db.execute(
            "SELECT result, output FROM builds WHERE name='e2e-retry-badparams' "
            "ORDER BY number DESC LIMIT 1"
        ).fetchone()
        db.close()
        self.assertEqual(row[0], 5)  # success
        self.assertIn("retry ok", row[1])

    def test_retry_with_empty_params(self):
        """Retry a build whose params column is empty string."""
        self._create_job("e2e-retry-emptyparams", """\
            #!/bin/bash
            if [ "$RUN" = "1" ]; then exit 1; fi
            echo "retry empty ok"
        """)
        self._crankc("run", "e2e-retry-emptyparams", check=False)

        # Retry with empty params (default) should work
        result = self._ctl("retry", "e2e-retry-emptyparams")
        self.assertEqual(result.returncode, 0)

        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        row = db.execute(
            "SELECT result FROM builds WHERE name='e2e-retry-emptyparams' "
            "ORDER BY number DESC LIMIT 1"
        ).fetchone()
        db.close()
        self.assertEqual(row[0], 5)

    def test_retry_with_invalid_json_params(self):
        """Retry a build whose params column contains invalid JSON."""
        self._create_job("e2e-retry-badjson", """\
            #!/bin/bash
            if [ "$RUN" = "1" ]; then exit 1; fi
            echo "retry badjson ok"
        """)
        self._crankc("run", "e2e-retry-badjson", check=False)

        # Overwrite params with invalid JSON
        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        db.execute(
            "UPDATE builds SET params=? WHERE name='e2e-retry-badjson' AND number=1",
            ("{not valid json}",),
        )
        db.commit()
        db.close()

        # Retry should still work (gracefully ignore unparseable params)
        result = self._ctl("retry", "e2e-retry-badjson")
        self.assertEqual(result.returncode, 0)

        db = sqlite3.connect(str(Path(self.home) / "crank.sqlite"))
        row = db.execute(
            "SELECT result FROM builds WHERE name='e2e-retry-badjson' "
            "ORDER BY number DESC LIMIT 1"
        ).fetchone()
        db.close()
        self.assertEqual(row[0], 5)


class TestOnCompleteHookE2E(CrankE2ETestCase):
    """Test on_complete notification hook from .conf."""

    def test_on_complete_runs(self):
        self._create_job("e2e-oncomplete", """\
            #!/bin/bash
            echo "main"
        """)
        marker = os.path.join(self.tmpdir, "oncomplete-marker")
        notify_script = self.jobs_dir / "notify.sh"
        notify_script.write_text(f"#!/bin/bash\ntouch {marker}\n")
        notify_script.chmod(0o755)
        (self.jobs_dir / "e2e-oncomplete.conf").write_text("on_complete=notify.sh\n")

        self._crankc("run", "e2e-oncomplete")
        self.assertTrue(os.path.exists(marker))

    def test_on_complete_gets_env(self):
        self._create_job("e2e-oncomplete-env", """\
            #!/bin/bash
            echo "main output"
        """)
        env_file = os.path.join(self.tmpdir, "oncomplete-env")
        notify_script = self.jobs_dir / "notify-env.sh"
        notify_script.write_text(
            f"#!/bin/bash\necho \"RESULT=$RESULT RESULT_NAME=$RESULT_NAME\" > {env_file}\n"
        )
        notify_script.chmod(0o755)
        (self.jobs_dir / "e2e-oncomplete-env.conf").write_text("on_complete=notify-env.sh\n")

        self._crankc("run", "e2e-oncomplete-env")
        self.assertTrue(os.path.exists(env_file))
        content = open(env_file).read()
        self.assertIn("RESULT=5", content)
        self.assertIn("RESULT_NAME=success", content)

    def test_on_complete_runs_on_failure(self):
        self._create_job("e2e-oncomplete-fail", """\
            #!/bin/bash
            exit 1
        """)
        marker = os.path.join(self.tmpdir, "oncomplete-fail-marker")
        notify_script = self.jobs_dir / "notify-fail.sh"
        notify_script.write_text(f"#!/bin/bash\ntouch {marker}\n")
        notify_script.chmod(0o755)
        (self.jobs_dir / "e2e-oncomplete-fail.conf").write_text("on_complete=notify-fail.sh\n")

        self._crankc("run", "e2e-oncomplete-fail", check=False)
        self.assertTrue(os.path.exists(marker))


if __name__ == "__main__":
    unittest.main()
