"""Unit tests for docscan module."""

import json

# Import the module being tested
import sys
from pathlib import Path

import pytest

sys.path.insert(0, str(Path(__file__).parent.parent))
from pytola.office.docscan.docscan import DocumentScanner, Rule


class TestRule:
    """Test cases for Rule class."""

    def test_rule_initialization_regex(self):
        """Test Rule initialization with regex pattern."""
        rule_data = {
            "name": "test_rule",
            "pattern": r"\d{3}-\d{2}-\d{4}",
            "regex": True,
            "case_sensitive": False,
            "context_lines": 2,
            "description": "Test description",
        }
        rule = Rule(rule_data)

        assert rule.name == "test_rule"
        assert rule.pattern == r"\d{3}-\d{2}-\d{4}"
        assert rule.is_regex is True
        assert rule.case_sensitive is False
        assert rule.context_lines == 2
        assert rule.description == "Test description"
        assert rule.compiled_pattern is not None

    def test_rule_initialization_text(self):
        """Test Rule initialization with simple text pattern."""
        rule_data = {
            "name": "text_rule",
            "pattern": "hello world",
            "regex": False,
            "case_sensitive": True,
            "context_lines": 5,
        }
        rule = Rule(rule_data)

        assert rule.name == "text_rule"
        assert rule.pattern == "hello world"
        assert rule.is_regex is False
        assert rule.case_sensitive is True
        assert rule.context_lines == 5
        assert rule.compiled_pattern is None

    def test_rule_default_values(self):
        """Test Rule with default values."""
        rule_data = {"name": "default_rule", "pattern": "test"}
        rule = Rule(rule_data)

        assert rule.is_regex is False
        assert rule.case_sensitive is False
        assert rule.context_lines == 3
        assert rule.description == ""

    def test_rule_invalid_regex(self):
        """Test Rule with invalid regex pattern."""
        rule_data = {"name": "invalid_rule", "pattern": "[invalid(regex", "regex": True}
        rule = Rule(rule_data)

        # Should not crash, but compiled_pattern should be None
        assert rule.compiled_pattern is None

    def test_search_regex_pattern(self):
        """Test regex search functionality."""
        rule_data = {
            "name": "email_rule",
            "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}",
            "regex": True,
            "case_sensitive": False,
        }
        rule = Rule(rule_data)

        text = """
        Contact us at support@example.com or sales@company.org
        Invalid: not-an-email
        """
        matches = rule.search(text)

        assert len(matches) == 2
        assert matches[0]["match"] == "support@example.com"
        assert matches[1]["match"] == "sales@company.org"
        assert matches[0]["type"] == "regex"

    def test_search_text_pattern(self):
        """Test text search functionality."""
        rule_data = {"name": "keyword_rule", "pattern": "confidential", "regex": False, "case_sensitive": False}
        rule = Rule(rule_data)

        text = """
        This document contains CONFIDENTIAL information.
        Please do not share confidential data.
        """
        matches = rule.search(text)

        assert len(matches) == 2
        assert "confidential" in matches[0]["match"].lower()
        assert matches[0]["type"] == "text"

    def test_search_case_sensitive(self):
        """Test case-sensitive search."""
        rule_data = {"name": "case_rule", "pattern": "TEST", "regex": False, "case_sensitive": True}
        rule = Rule(rule_data)

        text = "This is a TEST string with test lower case"
        matches = rule.search(text)

        # Should only match uppercase TEST
        assert len(matches) == 1
        assert matches[0]["match"] == "TEST"

    def test_search_empty_text(self):
        """Test search with empty text."""
        rule_data = {"name": "empty_rule", "pattern": "test"}
        rule = Rule(rule_data)

        matches = rule.search("")
        assert matches == []

    def test_search_no_matches(self):
        """Test search with no matches found."""
        rule_data = {"name": "no_match_rule", "pattern": "xyz123"}
        rule = Rule(rule_data)

        text = "This is a test string without the pattern"
        matches = rule.search(text)

        assert matches == []

    def test_get_context(self):
        """Test context extraction."""
        rule_data = {"name": "context_rule", "pattern": "test", "context_lines": 2}
        rule = Rule(rule_data)

        lines = ["Line 1", "Line 2", "Line 3 with test pattern", "Line 4", "Line 5", "Line 6"]

        context = rule._get_context(lines, 2)  # Line 3 is at index 2

        assert len(context) == 5  # 2 lines before + match line + 2 lines after
        assert "Line 1" in context
        assert "Line 3 with test pattern" in context
        assert "Line 5" in context

    def test_get_context_boundary(self):
        """Test context extraction at boundaries."""
        rule_data = {"name": "boundary_rule", "pattern": "test", "context_lines": 3}
        rule = Rule(rule_data)

        lines = ["Line 1 with test", "Line 2", "Line 3"]

        context = rule._get_context(lines, 0)  # First line

        assert len(context) == 3  # Only 3 lines total available
        assert "Line 1 with test" in context


class TestDocumentScanner:
    """Test cases for DocumentScanner class."""

    @pytest.fixture
    def test_rules(self):
        """Create test rules for scanning."""
        return [
            Rule(
                {
                    "name": "email",
                    "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}",
                    "regex": True,
                    "context_lines": 1,
                }
            ),
            Rule({"name": "phone", "pattern": r"\d{3}-\d{3}-\d{4}", "regex": True, "context_lines": 1}),
        ]

    @pytest.fixture
    def test_directory(self, tmp_path):
        """Create a temporary test directory with test files."""
        # Create test text file
        test_file = tmp_path / "test.txt"
        test_file.write_text("""
        Contact Information:
        Email: john.doe@example.com
        Phone: 555-123-4567

        Support:
        Email: support@test.org
        Phone: 555-987-6543
        """)

        # Create JSON file
        json_file = tmp_path / "data.json"
        json_file.write_text(json.dumps({"contact": "info@company.com", "phone": "555-555-5555"}))

        return tmp_path

    def test_scanner_initialization(self, test_directory, test_rules):
        """Test DocumentScanner initialization."""
        scanner = DocumentScanner(test_directory, test_rules, ["txt", "json"], use_pdf_ocr=False)

        assert scanner.input_dir == test_directory
        assert len(scanner.rules) == 2
        assert scanner.file_types == ["txt", "json"]
        assert scanner.use_pdf_ocr is False
        assert scanner.results == []

    def test_collect_files(self, test_directory):
        """Test file collection functionality."""
        scanner = DocumentScanner(test_directory, [Rule({"name": "test", "pattern": "test"})], ["txt", "json"])

        files = scanner._collect_files()

        assert len(files) == 2
        file_names = [f.name for f in files]
        assert "test.txt" in file_names
        assert "data.json" in file_names

    def test_extract_text(self, test_directory):
        """Test text extraction from plain text file."""
        scanner = DocumentScanner(test_directory, [Rule({"name": "test", "pattern": "test"})], ["txt"])

        text_file = test_directory / "test.txt"
        text, metadata = scanner._extract_text(text_file)

        assert len(text) > 0
        assert "john.doe@example.com" in text
        assert metadata.get("encoding") in ["utf-8", "latin-1", "cp1252"]

    def test_scan_single_file(self, test_directory, test_rules):
        """Test scanning a single file."""
        scanner = DocumentScanner(test_directory, test_rules, ["txt"])

        test_file = test_directory / "test.txt"
        result = scanner._scan_file(test_file)

        assert result is not None
        assert "file_path" in result
        assert "file_type" in result
        assert "matches" in result
        assert result["file_type"] == "txt"
        assert len(result["matches"]) > 0

    def test_scan_with_no_matches(self, test_directory, tmp_path):
        """Test scanning file with no matches."""
        scanner = DocumentScanner(test_directory, [Rule({"name": "nomatch", "pattern": "xyz123"})], ["txt"])

        # Create file with no matches
        no_match_file = tmp_path / "nomatch.txt"
        no_match_file.write_text("This file has no matching patterns")

        result = scanner._scan_file(no_match_file)
        assert result == {}

    def test_scan_results_structure(self, test_directory, test_rules):
        """Test that scan results have correct structure."""
        scanner = DocumentScanner(test_directory, test_rules, ["txt", "json"])

        results = scanner.scan(threads=1)

        assert "scan_info" in results
        assert "rules" in results
        assert "matches" in results

        assert "input_directory" in results["scan_info"]
        assert "scan_time" in results["scan_info"]
        assert "total_files" in results["scan_info"]
        assert "files_with_matches" in results["scan_info"]

    def test_scan_info_accuracy(self, test_directory, test_rules):
        """Test that scan info accurately reflects scan results."""
        scanner = DocumentScanner(test_directory, test_rules, ["txt", "json"])

        results = scanner.scan(threads=1)

        assert results["scan_info"]["total_files"] == 2
        assert results["scan_info"]["rules_count"] == 2
        assert results["scan_info"]["files_with_matches"] >= 1

    def test_scan_with_multiple_file_types(self, test_directory, test_rules):
        """Test scanning multiple file types."""
        scanner = DocumentScanner(test_directory, test_rules, ["txt", "json", "md"])

        # Create markdown file
        md_file = test_directory / "README.md"
        md_file.write_text("Contact: hello@world.com")

        results = scanner.scan(threads=1)

        assert results["scan_info"]["total_files"] == 3

    def test_match_context_extraction(self, test_directory):
        """Test that context is properly extracted for matches."""
        scanner = DocumentScanner(
            test_directory,
            [
                Rule(
                    {
                        "name": "email",
                        "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}",
                        "regex": True,
                        "context_lines": 2,
                    }
                )
            ],
            ["txt"],
        )

        results = scanner.scan(threads=1)

        if results["matches"]:
            file_match = results["matches"][0]
            if file_match["matches"]:
                match = file_match["matches"][0]
                assert "context" in match
                assert isinstance(match["context"], list)


class TestIntegration:
    """Integration tests for complete workflows."""

    def test_complete_scan_workflow(self, tmp_path):
        """Test complete scanning workflow from rules to results."""
        # Create test directory with files
        test_dir = tmp_path / "docs"
        test_dir.mkdir()

        # Create test file
        test_file = test_dir / "document.txt"
        test_file.write_text("""
        Project Documentation

        Contact:
        - Email: admin@example.com
        - Phone: 555-123-4567

        Important Dates:
        - Start: 2024-01-15
        - End: 2024-12-31
        """)

        # Create rules
        rules = [
            Rule(
                {
                    "name": "emails",
                    "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}",
                    "regex": True,
                    "context_lines": 2,
                }
            ),
            Rule({"name": "dates", "pattern": r"\d{4}-\d{2}-\d{2}", "regex": True, "context_lines": 1}),
        ]

        # Scan documents
        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        # Verify results
        assert results["scan_info"]["total_files"] == 1
        assert results["scan_info"]["files_with_matches"] == 1
        assert len(results["matches"]) == 1

        # Verify matches
        file_matches = results["matches"][0]
        assert len(file_matches["matches"]) >= 2  # At least email and date

        # Check for email match
        email_matches = [m for m in file_matches["matches"] if m["rule_name"] == "emails"]
        assert len(email_matches) > 0
        assert "example.com" in email_matches[0]["match"]

        # Check for date match
        date_matches = [m for m in file_matches["matches"] if m["rule_name"] == "dates"]
        assert len(date_matches) > 0

    def test_save_results_to_json(self, tmp_path):
        """Test saving scan results to JSON file."""
        # Create test data
        test_dir = tmp_path / "test_docs"
        test_dir.mkdir()

        test_file = test_dir / "test.txt"
        test_file.write_text("Contact: test@example.com")

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        # Scan
        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        # Save to JSON
        output_file = test_dir / "results.json"
        with open(output_file, "w", encoding="utf-8") as f:
            json.dump(results, f, indent=2)

        # Verify file was created and is valid JSON
        assert output_file.exists()

        with open(output_file, encoding="utf-8") as f:
            loaded_results = json.load(f)

        assert loaded_results == results
        assert "scan_info" in loaded_results
        assert "matches" in loaded_results


class TestErrorHandling:
    """Test error handling and edge cases."""

    def test_scan_nonexistent_directory(self, tmp_path):
        """Test scanning a non-existent directory."""
        nonexistent_dir = tmp_path / "nonexistent"

        rules = [Rule({"name": "test", "pattern": "test"})]

        scanner = DocumentScanner(nonexistent_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        # Should complete without crashing
        assert results["scan_info"]["total_files"] == 0
        assert results["scan_info"]["files_with_matches"] == 0

    def test_scan_empty_directory(self, tmp_path):
        """Test scanning an empty directory."""
        empty_dir = tmp_path / "empty"
        empty_dir.mkdir()

        rules = [Rule({"name": "test", "pattern": "test"})]

        scanner = DocumentScanner(empty_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        assert results["scan_info"]["total_files"] == 0
        assert results["matches"] == []

    def test_scan_with_no_files_matching_type(self, tmp_path):
        """Test scanning directory with no files of specified type."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        # Create a PDF file but scan for TXT only
        pdf_file = test_dir / "test.pdf"
        pdf_file.write_text("fake pdf content")

        rules = [Rule({"name": "test", "pattern": "test"})]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        assert results["scan_info"]["total_files"] == 0


class TestPauseResumeStop:
    """Test pause, resume, and stop functionality."""

    @pytest.fixture
    def test_directory(self, tmp_path):
        """Create test directory with files."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        # Create multiple test files
        for i in range(10):
            test_file = test_dir / f"test{i}.txt"
            test_file.write_text(f"Contact: user{i}@example.com")

        return test_dir

    def test_pause_and_resume(self, test_directory):
        """Test pause and resume functionality."""
        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_directory, rules, ["txt"])

        # Test initial state
        assert scanner.is_paused() is False
        assert scanner.is_stopped() is False

        # Test pause
        scanner.pause()
        assert scanner.is_paused() is True
        assert scanner.is_stopped() is False

        # Test resume
        scanner.resume()
        assert scanner.is_paused() is False
        assert scanner.is_stopped() is False

    def test_stop_scanner(self, test_directory):
        """Test stop functionality."""
        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_directory, rules, ["txt"])

        # Test initial state
        assert scanner.is_stopped() is False

        # Test stop
        scanner.stop()
        assert scanner.is_stopped() is True

    def test_progress_callback(self, test_directory):
        """Test progress callback functionality."""
        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        progress_updates = []

        def progress_callback(current, total):
            progress_updates.append((current, total))

        scanner = DocumentScanner(test_directory, rules, ["txt"])
        scanner.set_progress_callback(progress_callback)

        results = scanner.scan(threads=1)
        assert results["scan_info"]["total_files"] == 10

        # Verify callback was called
        assert len(progress_updates) > 0
        # Verify progress values are reasonable
        for current, total in progress_updates:
            assert current <= total
            assert total == len(list(test_directory.glob("*.txt")))


class TestFileCollection:
    """Test file collection functionality."""

    def test_collect_files_case_insensitive(self, tmp_path):
        """Test that file collection is case-insensitive."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        # Create files with mixed case extensions
        (test_dir / "test1.TXT").write_text("content1")
        (test_dir / "test2.txt").write_text("content2")
        (test_dir / "test3.TXT").write_text("content3")

        scanner = DocumentScanner(test_dir, [Rule({"name": "test", "pattern": "test"})], ["txt"])
        files = scanner._collect_files()

        # Should collect all three files
        assert len(files) == 3

    def test_collect_files_subdirectories(self, tmp_path):
        """Test that file collection includes subdirectories."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        # Create files in subdirectories
        subdir1 = test_dir / "subdir1"
        subdir1.mkdir()
        (subdir1 / "test1.txt").write_text("content1")

        subdir2 = test_dir / "subdir2"
        subdir2.mkdir()
        (subdir2 / "test2.txt").write_text("content2")

        scanner = DocumentScanner(test_dir, [Rule({"name": "test", "pattern": "test"})], ["txt"])
        files = scanner._collect_files()

        # Should collect files from subdirectories
        assert len(files) == 2

    def test_collect_files_no_duplicates(self, tmp_path):
        """Test that file collection removes duplicates."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        # Create a file
        (test_dir / "test.txt").write_text("content")

        scanner = DocumentScanner(test_dir, [Rule({"name": "test", "pattern": "test"})], ["TXT", "txt"])
        files = scanner._collect_files()

        # Should not have duplicates
        assert len(files) == 1


class TestFileMetadata:
    """Test file metadata in scan results."""

    def test_file_size_in_results(self, tmp_path):
        """Test that file size is included in results."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        test_file = test_dir / "test.txt"
        test_content = "Contact: test@example.com"
        test_file.write_text(test_content)

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        if results["matches"]:
            file_match = results["matches"][0]
            assert "file_size" in file_match
            assert file_match["file_size"] == len(test_content)

    def test_processing_time_in_results(self, tmp_path):
        """Test that processing time is recorded."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        test_file = test_dir / "test.txt"
        test_file.write_text("Contact: test@example.com")

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        if results["matches"]:
            file_match = results["matches"][0]
            assert "metadata" in file_match
            assert "processing_time_seconds" in file_match["metadata"]
            assert file_match["metadata"]["processing_time_seconds"] >= 0

    def test_encoding_metadata(self, tmp_path):
        """Test that encoding is recorded for text files."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        test_file = test_dir / "test.txt"
        test_file.write_text("Contact: test@example.com", encoding="utf-8")

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        if results["matches"]:
            file_match = results["matches"][0]
            assert "metadata" in file_match
            assert "encoding" in file_match["metadata"]
            assert file_match["metadata"]["encoding"] in ["utf-8", "latin-1", "cp1252", "utf-16"]


class TestRTFExtraction:
    """Test RTF file extraction."""

    def test_extract_rtf_simple(self, tmp_path):
        """Test simple RTF text extraction."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        # Create a simple RTF file
        rtf_file = test_dir / "test.rtf"
        rtf_content = r"{\rtf1\ansi{\fonttbl\f0\fswiss Arial;}\f0\fs24 Hello World and test@example.com}"
        rtf_file.write_bytes(rtf_content.encode("utf-8"))

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["rtf"])
        results = scanner.scan(threads=1)

        # RTF extraction may not find the email due to simple parser
        # but the scan should complete without error
        assert "scan_info" in results


class TestMultipleMatchesInFile:
    """Test handling multiple matches in a single file."""

    def test_multiple_email_matches(self, tmp_path):
        """Test file with multiple email matches."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        test_file = test_dir / "test.txt"
        test_content = """
        Contact 1: alice@example.com
        Contact 2: bob@test.org
        Contact 3: charlie@company.net
        Contact 4: david@demo.co.uk
        """
        test_file.write_text(test_content)

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        if results["matches"]:
            file_match = results["matches"][0]
            # Should find at least 4 email matches
            assert len(file_match["matches"]) >= 4

            # Check that all matches have the required fields
            for match in file_match["matches"]:
                assert "rule_name" in match
                assert "line_number" in match
                assert "match" in match
                assert "start" in match
                assert "end" in match


class TestRuleWithMultiplePatterns:
    """Test scanning with multiple rules on the same file."""

    def test_multiple_rules_on_same_file(self, tmp_path):
        """Test applying multiple rules to the same file."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        test_file = test_dir / "test.txt"
        test_content = """
        Contact: support@example.com
        Phone: 555-123-4567
        Date: 2024-01-15
        """
        test_file.write_text(test_content)

        rules = [
            Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True}),
            Rule({"name": "phone", "pattern": r"\d{3}-\d{3}-\d{4}", "regex": True}),
            Rule({"name": "date", "pattern": r"\d{4}-\d{2}-\d{2}", "regex": True}),
        ]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        if results["matches"]:
            file_match = results["matches"][0]
            # Should find matches from all three rules
            rule_names = {match["rule_name"] for match in file_match["matches"]}
            assert "email" in rule_names
            assert "phone" in rule_names
            assert "date" in rule_names


class TestTextEncodingHandling:
    """Test handling different text encodings."""

    def test_utf8_encoding(self, tmp_path):
        """Test UTF-8 encoded file."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        test_file = test_dir / "test.txt"
        test_file.write_text("Contact: 测试@example.com", encoding="utf-8")

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        # Should successfully extract text and find email
        assert "scan_info" in results

    def test_latin1_encoding(self, tmp_path):
        """Test Latin-1 encoded file."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        test_file = test_dir / "test.txt"
        test_file.write_text("Contact: test@example.com", encoding="latin-1")

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        # Should successfully extract text
        assert "scan_info" in results


class TestHTMLExtraction:
    """Test HTML file extraction."""

    def test_extract_html_simple(self, tmp_path):
        """Test simple HTML text extraction."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        html_file = test_dir / "test.html"
        html_content = """
        <html>
        <head><title>Contact Info</title></head>
        <body>
        <h1>Contact Us</h1>
        <p>Email: support@example.com</p>
        <p>Phone: 555-123-4567</p>
        </body>
        </html>
        """
        html_file.write_text(html_content)

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["html"])
        results = scanner.scan(threads=1)

        if results["matches"]:
            html_matches = [m for m in results["matches"] if m["file_type"] == "html"]
            assert len(html_matches) >= 1

    def test_extract_html_with_special_chars(self, tmp_path):
        """Test HTML with special characters."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        html_file = test_dir / "test.html"
        html_file.write_text("<html><body>Contact: &lt;test@example.com&gt;</body></html>")

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["html"])
        results = scanner.scan(threads=1)

        # Scan should complete
        assert "scan_info" in results


class TestXMLExtraction:
    """Test XML file extraction."""

    def test_extract_xml_simple(self, tmp_path):
        """Test simple XML text extraction."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        xml_file = test_dir / "test.xml"
        xml_content = """
        <?xml version="1.0" encoding="UTF-8"?>
        <contacts>
            <person>
                <name>John Doe</name>
                <email>john@example.com</email>
            </person>
            <person>
                <name>Jane Smith</name>
                <email>jane@test.org</email>
            </person>
        </contacts>
        """
        xml_file.write_text(xml_content)

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["xml"])
        results = scanner.scan(threads=1)

        if results["matches"]:
            xml_matches = [m for m in results["matches"] if m["file_type"] == "xml"]
            assert len(xml_matches) >= 1


class TestMarkdownExtraction:
    """Test Markdown file extraction."""

    def test_extract_markdown_simple(self, tmp_path):
        """Test simple Markdown text extraction."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        md_file = test_dir / "test.md"
        md_content = """
        # Contact Information

        For inquiries, please reach out to:
        - Email: support@example.com
        - Phone: 555-123-4567

        ## Sales

        Contact sales@company.com for pricing.
        """
        md_file.write_text(md_content)

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["md"])
        results = scanner.scan(threads=1)

        if results["matches"]:
            md_matches = [m for m in results["matches"] if m["file_type"] == "md"]
            assert len(md_matches) >= 1


class TestRuleDescription:
    """Test rule description functionality."""

    def test_rule_description_in_results(self, tmp_path):
        """Test that rule description is included in results."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        test_file = test_dir / "test.txt"
        test_file.write_text("Contact: test@example.com")

        rule_data = {
            "name": "email_rule",
            "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}",
            "regex": True,
            "description": "Find email addresses in documents",
        }

        rules = [Rule(rule_data)]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        if results["matches"]:
            file_match = results["matches"][0]
            if file_match["matches"]:
                match = file_match["matches"][0]
                assert "rule_description" in match
                assert match["rule_description"] == "Find email addresses in documents"


class TestBoundaryConditions:
    """Test boundary conditions and edge cases."""

    def test_empty_file(self, tmp_path):
        """Test scanning an empty file."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        empty_file = test_dir / "empty.txt"
        empty_file.write_text("")

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        # Should scan but find no matches
        assert "scan_info" in results
        assert results["scan_info"]["files_with_matches"] == 0

    def test_very_long_line(self, tmp_path):
        """Test file with very long lines."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        test_file = test_dir / "long.txt"
        # Create a very long line with an email
        long_content = "x" * 10000 + " test@example.com " + "y" * 10000
        test_file.write_text(long_content)

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        # Should still find the email
        assert "scan_info" in results

    def test_file_with_special_chars(self, tmp_path):
        """Test file with special characters."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        test_file = test_dir / "special.txt"
        # File with various special characters (using ASCII-safe version)
        special_content = """
        Contact: test@example.com
        Special chars: !@#$%^&*()_+-=[]{}|;':",./<>?
        Numbers: 1234567890
        """
        test_file.write_text(special_content, encoding="utf-8")

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        # Should handle special characters
        assert "scan_info" in results


class TestMultilinePatterns:
    """Test multiline pattern matching."""

    def test_pattern_across_lines(self, tmp_path):
        """Test pattern that appears on multiple lines."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        test_file = test_dir / "multiline.txt"
        multiline_content = """
        Line 1: Contact info
        Line 2: john@example.com
        Line 3: Another contact
        Line 4: jane@test.org
        Line 5: End of contacts
        """
        test_file.write_text(multiline_content)

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        if results["matches"]:
            file_match = results["matches"][0]
            # Should find emails on different lines
            assert len(file_match["matches"]) >= 2

            # Check line numbers are correct
            line_numbers = {m["line_number"] for m in file_match["matches"]}
            # With the initial empty line, emails are on lines 3 and 5
            assert 3 in line_numbers
            assert 5 in line_numbers


class TestConcurrentScanning:
    """Test concurrent scanning capabilities."""

    def test_scan_with_multiple_threads(self, tmp_path):
        """Test scanning with multiple threads."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        # Create multiple files
        for i in range(20):
            test_file = test_dir / f"file{i}.txt"
            test_file.write_text(f"Contact: user{i}@example.com")

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=4)

        # Should scan all files
        assert results["scan_info"]["total_files"] == 20
        assert results["scan_info"]["files_with_matches"] == 20

    def test_scan_results_consistency_across_threads(self, tmp_path):
        """Test that results are consistent when using different thread counts."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        # Create test files
        for i in range(5):
            test_file = test_dir / f"test{i}.txt"
            test_file.write_text(f"Email: contact{i}@example.com")

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        # Scan with 1 thread
        scanner1 = DocumentScanner(test_dir, rules, ["txt"])
        results1 = scanner1.scan(threads=1)

        # Scan with 4 threads
        scanner2 = DocumentScanner(test_dir, rules, ["txt"])
        results2 = scanner2.scan(threads=4)

        # Results should be consistent
        assert results1["scan_info"]["total_files"] == results2["scan_info"]["total_files"]
        assert results1["scan_info"]["files_with_matches"] == results2["scan_info"]["files_with_matches"]
        assert len(results1["matches"]) == len(results2["matches"])


class TestMatchDetails:
    """Test detailed match information."""

    def test_match_start_end_positions(self, tmp_path):
        """Test that match start and end positions are correct."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        test_file = test_dir / "test.txt"
        test_content = "Email: test@example.com and info@test.org"
        test_file.write_text(test_content)

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        if results["matches"]:
            file_match = results["matches"][0]
            # Check that start and end positions are valid
            for match in file_match["matches"]:
                assert "start" in match
                assert "end" in match
                assert match["end"] > match["start"]
                # start and end are positions in the line, not in the match
                assert (match["end"] - match["start"]) == len(match["match"])

    def test_match_line_numbers(self, tmp_path):
        """Test that line numbers are correct."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        test_file = test_dir / "test.txt"
        test_content = """Line 1: First email is first@example.com
Line 2: Second email is second@test.org
Line 3: Third email is third@example.com
Line 4: Fourth email is fourth@demo.net"""
        test_file.write_text(test_content)

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        if results["matches"]:
            file_match = results["matches"][0]
            # Check line numbers match expected values
            line_numbers = {m["line_number"] for m in file_match["matches"]}
            assert len(line_numbers) == 4  # 4 emails on 4 lines


class TestScannerConfiguration:
    """Test scanner configuration options."""

    def test_use_pdf_ocr_flag(self, tmp_path):
        """Test use_pdf_ocr configuration."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        test_file = test_dir / "test.txt"
        test_file.write_text("Contact: test@example.com")

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        # Test with OCR disabled
        scanner1 = DocumentScanner(test_dir, rules, ["txt"], use_pdf_ocr=False)
        assert scanner1.use_pdf_ocr is False

        # Test with OCR enabled
        scanner2 = DocumentScanner(test_dir, rules, ["txt"], use_pdf_ocr=True)
        assert scanner2.use_pdf_ocr is True

    def test_batch_size_configuration(self, tmp_path):
        """Test batch size configuration."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        test_file = test_dir / "test.txt"
        test_file.write_text("Contact: test@example.com")

        rules = [Rule({"name": "email", "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "regex": True})]

        scanner = DocumentScanner(test_dir, rules, ["txt"], batch_size=100)
        assert scanner.batch_size == 100

        results = scanner.scan(threads=1)
        assert "scan_info" in results


class TestRuleBehavior:
    """Test specific rule behaviors."""

    def test_regex_with_special_characters(self, tmp_path):
        """Test regex patterns with special characters."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        test_file = test_dir / "test.txt"
        test_content = """
        IP Addresses: 192.168.1.1, 10.0.0.1, 172.16.0.1
        """
        test_file.write_text(test_content)

        # IP address pattern with special regex characters
        rule_data = {
            "name": "ip_addresses",
            "pattern": r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}",
            "regex": True,
        }

        rules = [Rule(rule_data)]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        if results["matches"]:
            file_match = results["matches"][0]
            # Should find at least 3 IP addresses
            assert len(file_match["matches"]) >= 3

    def test_text_pattern_with_spaces(self, tmp_path):
        """Test text patterns with spaces."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        test_file = test_dir / "test.txt"
        test_content = """
        This is a CONFIDENTIAL document.
        Please keep it confidential.
        CONFIDENTIAL: Do not share.
        """
        test_file.write_text(test_content)

        # Text pattern with spaces
        rule_data = {
            "name": "confidential",
            "pattern": "CONFIDENTIAL",
            "regex": False,
            "case_sensitive": True,
        }

        rules = [Rule(rule_data)]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        if results["matches"]:
            file_match = results["matches"][0]
            # Should find exact uppercase "CONFIDENTIAL"
            assert len(file_match["matches"]) == 2  # Line 1 and line 3

    def test_context_lines_boundary(self, tmp_path):
        """Test context lines at file boundaries."""
        test_dir = tmp_path / "test"
        test_dir.mkdir()

        test_file = test_dir / "test.txt"
        test_content = """Line 1: First line
Line 2: SECRET@example.com here
Line 3: Third line"""
        test_file.write_text(test_content)

        # Large context_lines value to test boundary
        rule_data = {
            "name": "email",
            "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}",
            "regex": True,
            "context_lines": 10,
        }

        rules = [Rule(rule_data)]

        scanner = DocumentScanner(test_dir, rules, ["txt"])
        results = scanner.scan(threads=1)

        if results["matches"]:
            file_match = results["matches"][0]
            match = file_match["matches"][0]
            # Context should not exceed file boundaries
            assert "context" in match
            assert isinstance(match["context"], list)
            assert len(match["context"]) <= 3  # Only 3 lines in file


if __name__ == "__main__":
    pytest.main([__file__, "-v"])
