import json
import logging
import re
from copy import deepcopy
from dataclasses import dataclass, field
from datetime import datetime
from typing import ClassVar, cast

from dateutil.parser import parse
from rich import box
from rich.console import Console, ConsoleOptions, RenderResult
from rich.padding import Padding
from rich.panel import Panel
from rich.table import Table
from rich.text import Text

from epstein_files.documents.communication import Communication
from epstein_files.documents.document import CLOSE_PROPERTIES_CHAR, EXCERPT_STYLE
from epstein_files.documents.documents.categories import Uninteresting
from epstein_files.documents.config.doc_cfg import DEFAULT_TRUNCATE_TO, NO_TRUNCATE, SHORT_TRUNCATE_TO, DebugDict, EmailCfg, Metadata
from epstein_files.documents.doj_file import DojFile
from epstein_files.documents.emails.constants import *
from epstein_files.documents.emails.email_header import (EMAIL_SIMPLE_HEADER_REGEX,
     EMAIL_SIMPLE_HEADER_LINE_BREAK_REGEX, EmailHeader)
from epstein_files.documents.emails.emailers import extract_emailer_names
from epstein_files.documents.other_file import OtherFile
from epstein_files.people.interesting_people import EMAILERS_OF_INTEREST_SET
from epstein_files.output.epstein_highlighter import highlighter, temp_highlighter
from epstein_files.output.highlight_config import HIGHLIGHTED_NAMES, get_style_for_name
from epstein_files.output.html.builder import table_to_html
from epstein_files.output.html.elements import to_em
from epstein_files.output.layout_elements.file_display import FileDisplay, JustifyMethod
from epstein_files.output.rich import DEFAULT_TABLE_KWARGS, build_table, create_hyperlinks, join_texts, styled_key_value
from epstein_files.util.constant.strings import APPEARS_IN, ARCHIVE_LINK_COLOR, REDACTED, TIMESTAMP_DIM
from epstein_files.util.constant.urls import URL_SIGNIFIERS
from epstein_files.people.names import sort_names
from epstein_files.util.constants import CONFIGS_BY_ID
from epstein_files.util.env import args, site_config
from epstein_files.util.helpers.data_helpers import (AMERICAN_TIME_REGEX, TIMEZONE_INFO, flatten,
     prefix_keys, remove_timezone, uniquify)
from epstein_files.util.helpers.link_helper import link_text_obj
from epstein_files.util.helpers.string_helper import capitalize_first, collapse_newlines, is_bool_prop, strip_pdfalyzer_panels
from epstein_files.util.logging import logger

# Email bod regexes
BAD_FIRST_LINE_REGEX = re.compile(r'^(>>|Grant_Smith066474"eMailContent.htm|LOVE & KISSES)$')
BAD_LINE_REGEX = re.compile(r'^([>=];?|[>»]*=20|\d{1,2}|PAGE INTENTIONALLY LEFT BLANK|Classification: External Communication|Hide caption|Importance:?\s*High|[iI,•]|[1i] (_ )?[il]|, [-,]|L\._|_filtered|si.nature.asc|.*(yiv0232|font-family:|margin-bottom:).*)$')
BAD_SUBJECT_CONTINUATIONS = ['orwarded', 'Hi ', 'Sent ', 'AmLaw', 'Original Message', 'Privileged', 'Sorry', '---']
LINK_LINE_REGEX = re.compile(r"^[>• ]*htt")
LINK_LINE2_REGEX = re.compile(r"^[-\w.%&=/]{5,}$")
QUOTED_REPLY_LINE_REGEX = re.compile(r'(\nFrom:(.*)|wrote:)\n', re.IGNORECASE)
REPLY_TEXT_REGEX = re.compile(rf"^(.*?){REPLY_LINE_PATTERN}", re.DOTALL | re.IGNORECASE | re.MULTILINE)
XML_PLIST_REGEX = re.compile(r"[<=]?\?xml version.*</(plist|xml)>", re.DOTALL)

# Timestamp regexes
BAD_TIMEZONE_REGEX = re.compile(fr'\((UTC|GMT\+\d\d:\d\d)\)|{REDACTED}')
DATE_HEADER_REGEX = re.compile(r'(?:Date|Sent):? +(?!by|from|to|via)([^\n]{6,})\n')
TIMESTAMP_LINE_REGEX = re.compile(r"\d+:\d+")

# numbers
MAX_NUM_HEADER_LINES = 14
MAX_QUOTED_REPLIES = 1
NUM_WORDS_IN_LAST_QUOTE = 6

# TODO: Copy replace_text_with?
DERIVED_CFG_PROPS_TO_COPY = [
    'author_uncertain',
    'category',
    'is_in_chrono',
    'is_interesting',
]

# Junk mail
JUNK_EMAILERS = [
    contact.name
    for junk_hg in HIGHLIGHTED_NAMES if junk_hg.label == Uninteresting.JUNK
    for contact in junk_hg.contacts
]

BCC_LISTS = JUNK_EMAILERS + MAILING_LISTS
TRUNCATE_EMAILS_BY = BCC_LISTS + TRUNCATE_EMAILS_FROM
REWRITTEN_HEADER_MSG = "(janky OCR header fields were prettified, check source if something seems off)"

# TODO: add other forward patterns
REPLY_SPLITTERS = [f"{field}:" for field in COMMON_HEADER_FIELDS] + [
    '********************************',
    'Begin forwarded message',
]

OCR_REPAIRS: dict[str | re.Pattern, str] = {
    # '­': '-',
    # '‐': '-',
    '-­‐': '-',  # TODO: weird hyphens in 027004 and other files that rich doesn't handle well
    re.compile('»'): '>>',
    re.compile(r'grnail\.com'): 'gmail.com',
    'Newsmax. corn': 'Newsmax.com',
    re.compile(r"^(From|To)(: )?[_1.]{5,}", re.MULTILINE): rf"\1: {REDACTED}",  # Redacted email addresses
    # These 3 must come in this order!
    re.compile(r'([/vkT]|Ai|li|(I|7)v)rote:'): 'wrote:',
    re.compile(r"([<.=_HIM][<>.=_HIM14]{5,}[<>.=_HIM]|MOMMINNEMUMMIN) *(wrote:?)?"): rf"{REDACTED} \2",
    re.compile(r"([,<>_]|AM|PM)\n(>)? ?wrote:?"): r'\1\2 wrote:',
    # Headers
    re.compile(r"I ?(od|nl)ine-Images:"): 'Inline-Images:',
    re.compile(r"^((?:B?cc|To):.*)\n(>?;.*)", re.IGNORECASE | re.MULTILINE): r'\1 \2',
    re.compile(r"^From "): 'From: ',
    re.compile(r"^(Sent|Subject) (?![Ff]rom|on|using|[Rr]emote|[Vv]ia|with)", re.MULTILINE): r'\1: ',
    re.compile(r"^Subject[.•]{,2} ", re.MULTILINE): 'Subject: ',
    re.compile(r"^(Forwarded|Original) Message$", re.IGNORECASE | re.MULTILINE): r"--- \1 Message ---",  # Make forward lines match our highlight
    # Excessive quote chars
    re.compile(r"wrote:\n[>»]+(\n[>»]+)"): r"wrote:\1",
    re.compile(r"(^[>»]+\n){2,}", re.MULTILINE): r"\1",
    re.compile(r"\n[<>I ]*wrote:"): ' wrote:',
    # HTML garbage
    re.compile(r'[</=]{,3}(cl|d)?iv>|&[=n]bs[=p];|<=span>|=C\d+\b'): '',
    re.compile(r'^O= ', re.MULTILINE): 'On ',
    re.compile(r"<?=/?[bru]>"): '',
    re.compile(r'(^|\s)[<=][AC]\d+[=>]?', re.MULTILINE): r'\1',
    re.compile(r'[<=]=?/?(br|d?iv)( (class|id)="\w+")?>'): '',
    re.compile(r"\n<mailt.:?(.{,25})[»>]\s*wrote"): r'\1 wrote',
    re.compile(r"^--\w+-- (conversation-id|date-last-viewed).*(flags|remote-id\s\d+)(\s*\d{6,}.*remote-id.*\d+)?", re.MULTILINE): '',
    re.compile(r"<mailto:([-\w=.@]+)[»>]"): r'\1',
    # Names / email addresses
    r'Alireza lttihadieh': ALIREZA_ITTIHADIEH,
    r'bamaby': 'barnaby',
    r'Miroslav Laj6ak': MIROSLAV_LAJCAK,
    r'Ross G°w': ROSS_GOW,
    r'Torn Pritzker': TOM_PRITZKER,
    re.compile(r"( [AP]M,)\s+wrote:$", re.MULTILINE): r'\1 <REDACTED> wrote:',
    re.compile(r' Banno(r]?|\b)'): ' Bannon',
    re.compile(r"\bBamaby\b"): 'Barnaby',
    re.compile(r'gmax ?[1l] ?[@g]ellmax.c ?om'): 'gmax1@ellmax.com',
    re.compile(r"[ijlp']ee[vy]acation[©@a(&,P ]{1,3}g?mail.com"): 'jeevacation@gmail.com',
    'gyahoo.com': '@yahoo.com',
    # Signatures
    re.compile(r'Blac[il]cBerry'): 'BlackBerry',
    'BlackBerry by AT &T': 'BlackBerry by AT&T',
    'BlackBerry from T- Mobile': 'BlackBerry from T-Mobile',
    'Envoy& de': 'Envoyé de',
    'Envoye avec BlackBerry° d': 'Envoye avec BlackBerry® d',
    'from Samsung Mob.le': 'from Samsung Mobile',
    'gJeremyRubin': '@JeremyRubin',
    'Mail for i Phone': 'Mail for iPhone',
    'Sent from Mabfl': 'Sent from Mobile',  # NADIA_MARCINKO signature bad OCR
    'twitter glhsummers': 'twitter @lhsummers',
    re.compile(r"[cC]o-authored with i ?Phone auto-correct"): "Co-authored with iPhone auto-correct",
    re.compile(r"from my ['!()=]([Pp]hone)"): r'from my i\1',
    re.compile(r'from my BlackBerry[0°] wireless device'): 'from my BlackBerry® wireless device',
    re.compile(r'^INW$', re.MULTILINE): REDACTED,
    re.compile(r'Sent from one of my many test mobile devices while on the go and changing the world\s+:\s+so\s+my\s+apologies for any typos'): 'Sent from one of my many test mobile devices while on the go and changing the world : so my apologies for any typos',
    re.compile(r"twitter\.com[i/][lI]krauss[1lt]"): "twitter.com/lkrauss1",
    # links
    'classified-intelligence-\nmichael-flynn-trump': 'classified-intelligence-michael-flynn-trump',
    'on-accusers-rose-\nmcgowan/ ': 'on-accusers-rose-\nmcgowan/\n',
    'the-truth-\nabout-the-bitcoin-foundation/ )': 'the-truth-about-the-bitcoin-foundation/ )\n',
    'woody-allen-jeffrey-epsteins-\nsociety-friends-close-ranks/ ---': 'woody-allen-jeffrey-epsteins-society-friends-close_ranks/\n',
    ' https://www.theguardian.com/world/2017/may/29/close-friend-trump-thomas-barrack-\nalleged-tax-evasion-italy-sardinia?CMP=share btn fb': '\nhttps://www.theguardian.com/world/2017/may/29/close-friend-trump-thomas-barrack-alleged-tax-evasion-italy-sardinia?CMP=share_btn_fb',
    'search-for-secret-putin-\nfortune.html': 'search-for-secret-putin-fortune.html',
    re.compile(r"[h=][t=][t=][p=]://"): 'http://',
    re.compile(r"([h=][t=][t=][op=][s=]|Imps )://"): 'https://',
    re.compile(r'timestopics/people/t/landon jr thomas/inde\n?x\n?\.\n?h\n?tml'): 'timestopics/people/t/landon_jr_thomas/index.html',
    re.compile(r" http ?://www. ?dailymail. ?co ?.uk/news/article-\d+/Troub ?led-woman-history-drug-\n?us ?e-\n?.*html"): '\nhttp://www.dailymail.co.uk/news/article-3914012/Troubled-woman-history-drug-use-claimed-assaulted-Donald-Trump-Jeffrey-Epstein-sex-party-age-13-FABRICATED-story.html',
    re.compile(r"http.*steve-bannon-trump-tower-\n?interview-\n?trumps-\n?strategist-plots-\n?new-political-movement-948747"): "\nhttp://www.hollywoodreporter.com/news/steve-bannon-trump-tower-interview-trumps-strategist-plots-new-political-movement-948747",
    # Subject lines
    r"Arrested in\nInauguration Day Riot": "Arrested in Inauguration Day Riot",
    r"avoids testimony from alleged\nvictims": "avoids testimony from alleged victims",
    r"It's a first, but the buyer's\nanonymous": "It's a first, but the buyer's anonymous",
    r"but\nwatchdogs say probe is tainted": "watchdogs say probe is tainted",
    r"Christmas comes\nearly for most of macro": "Christmas comes early for most of macro",            # 023717
    r"but majority still made good\nmoney because": "but majority still made good money because",      # 023717
    r"COVER UP SEX ABUSE CRIMES\nBY THE WHITE HOUSE": "COVER UP SEX ABUSE CRIMES BY THE WHITE HOUSE",
    r'Priebus, used\nprivate email accounts for': 'Priebus, used private email accounts for',
    r"War on the Investigations\nEncircling Him": "War on the Investigations Encircling Him",
    r"Subject; RE": "Subject: RE",
    r"straining relations between UK and\nAmerica": "straining relations between UK and America",
    re.compile(r"as Putin Mayhem\sTests\sPresident's\sGrip\son\sGOP"): "as Putin Mayhem Tests President's Grip on GOP",
    re.compile(r"deadline re Mr Bradley Edwards vs Mr\s*Jeffrey Epstein", re.I): "deadline re Mr Bradley Edwards vs Mr Jeffrey Epstein",
    re.compile(r"Following Plea That Implicated Trump -\s*https://www.npr.org/676040070", re.I): "Following Plea That Implicated Trump - https://www.npr.org/676040070",
    re.compile(r"for Attorney General -\s+Wikisource, the"): r"for Attorney General - Wikisource, the",
    re.compile(r"JUDGE SWEET\s+ALLOWING\s+STEVEN\s+HOFFENBERG\s+TO\s+TALK\s+WITH\s+THE\s+TOWERS\s+VICTIMS\s+TO\s+EXPLAIN\s+THE\s+VICTIMS\s+SUI\n?T\s+FILING\s+AGAINST\s+JEFF\s+EPSTEIN"): "JUDGE SWEET ALLOWING STEVEN HOFFENBERG TO TALK WITH THE TOWERS VICTIMS TO EXPLAIN THE VICTIMS SUIT FILING AGAINST JEFF EPSTEIN",
    re.compile(r"Lawyer for Susan Rice: Obama administration '?justifiably concerned' about sharing Intel with\s*Trump team -\s*POLITICO", re.I): "Lawyer for Susan Rice: Obama administration 'justifiably concerned' about sharing Intel with Trump team - POLITICO",
    re.compile(r"PATTERSON NEW\s+BOOK\s+TELLING\s+FEDS\s+COVER\s+UP\s+OF\s+BILLIONAIRE\s+JEFF\s+EPSTEIN\s+CHILD\s+RAPES\s+RELEASE\s+DATE\s+OCT\s+10\s+2016\s+STEVEN\s+HOFFENBERG\s+IS\s+ON\s+THE\s+BOOK\s+WRITING\s+TEAM\s*!!!!"): "PATTERSON NEW BOOK TELLING FEDS COVER UP OF BILLIONAIRE JEFF EPSTEIN CHILD RAPES RELEASE DATE OCT 10 2016 STEVEN HOFFENBERG IS ON THE BOOK WRITING TEAM !!!!",
    re.compile(r"PROCEEDINGS FOR THE ST THOMAS ATTACHMENT OF\s*ALL JEFF EPSTEIN ASSETS"): "PROCEEDINGS FOR THE ST THOMAS ATTACHMENT OF ALL JEFF EPSTEIN ASSETS",
    re.compile(r"Subject:\s*Fwd: Trending Now: Friends for three decades"): "Subject: Fwd: Trending Now: Friends for three decades",
    # XML footers
    re.compile('<[iI] ?nteger>'): '<integer>',
    # Misc
    'AVG°': 'AVGO',
    'Saw Matt C with DTF at golf': 'Saw Matt C with DJT at golf',
    re.compile(r"[i. ]*Privileged[- ]*Redacted[i. ]*"): '<PRIVILEGED - REDACTED>',
}

METADATA_FIELDS = [
    'attachments',
    'attachment_file_ids',
    'is_junk_mail',
    'is_mailing_list',
    'recipients',
    'sent_from_device',
]

DEBUG_PROPS = METADATA_FIELDS + [
    'attachment_file_ids',
    'is_persons_first_email',
    'is_word_count_worthy',
]


@dataclass
class Email(Communication):
    """
    Attributes:
        attached_docs (list[OtherFile]): any attachments that exist as `OtherFile` objects
        actual_text (str): Best effort at the text actually sent in this email, excluding quoted replies and forwards.
        derived_cfg (EmailCfg): EmailCfg that was built instead of coming from CONFIGS_BY_ID
        header (EmailHeader): Header data extracted from the text (from/to/sent/subject etc).
        is_persons_first_email (bool): is this the first email we have for this person, whether sender or recipient?
        sent_from_device (str, optional): "Sent from my iPhone" style signature (if it exists).
        signature_substitution_counts (dict[str, int]): Number of times a signature was replaced with
            <...snipped...> per name
        _line_merge_arguments (list[tuple[int] | tuple[int, int]]): preconfigured list of line merges that will fix up
            files from the HOUSE_OVERSIGHT_ collection in memory while leaving the source files untouched
        _was_split_up (bool, optional): True if this file Email was one of the big ones that was split into pieces
            and thus should generally be hidden / not shown
    """
    attached_docs: list[OtherFile] = field(default_factory=list)
    actual_text: str = field(init=False)
    derived_cfg: EmailCfg | None = None
    is_persons_first_email: bool = False
    sent_from_device: str | None = None
    signature_substitution_counts: dict[str, int] = field(default_factory=dict)  # defaultdict breaks asdict :(
    _header: EmailHeader | None = None
    _line_merge_arguments: list[tuple[int] | tuple[int, int]] = field(default_factory=list)
    _was_split_up: bool = False

    # Class variable logging how many headers we prettified while printing, kind of janky
    rewritten_header_ids: ClassVar[set[str]] = set([])

    def __post_init__(self):
        super().__post_init__()
        self.actual_text = self._extract_actual_text()
        self.sent_from_device = self._sent_from_device()

        for signature, name in KNOWN_SIGNATURES.items():
            if self.has_unknown_participant and signature.lower() in self.text.lower():
                self.warn(f"Found known signature for {name} in unattributed email.")

    @property
    def attachment_file_ids(self) -> list[str]:
        """Strings in the Attachments: field in the header, split by semicolon."""
        return [a.file_id for a in self.attached_docs]

    @property
    def attachments(self) -> list[str]:
        """Strings in the Attachments: and Inline-Images: fields in the header, split by semicolon."""
        return self.header.all_attachments

    @property
    def config(self) -> EmailCfg | None:
        """Configured timestamp, if any."""
        if self.file_info.is_local_extract_file:
            this_file_cfg = cast(EmailCfg, CONFIGS_BY_ID.get(self.file_info.file_id))

            # Merge config for file this was extracted from into self.derived_cfg
            if not self.derived_cfg and (extracted_from_cfg := CONFIGS_BY_ID.get(self.file_info.url_slug)):
                self.derived_cfg = this_file_cfg or EmailCfg(id=self.file_id)

                if (extracted_from_description := extracted_from_cfg.complete_description):
                    self.derived_cfg.description = f"{APPEARS_IN} {extracted_from_description}"

                for prop in DERIVED_CFG_PROPS_TO_COPY:
                    derived_cfg_val = getattr(self.derived_cfg, prop)

                    if is_bool_prop(prop) and derived_cfg_val is not None:
                        continue  # Don't overwrite booleans

                    extracted_cfg_val = getattr(extracted_from_cfg, prop)
                    setattr(self.derived_cfg, prop, derived_cfg_val or extracted_cfg_val)

                self.log(f"Constructed synthetic config: {self.derived_cfg}")

            return self.derived_cfg or this_file_cfg
        else:
            return super().config

    @property
    def header(self) -> EmailHeader:
        self._header = self._header or self._extract_header()
        return self._header

    @property
    def html_margin_bottom(self) -> str:
        """Overloaded in `Email` for case of emails with attachments."""
        if self.attached_docs:
            return '6px'
        else:
            return super().html_margin_bottom

    @property
    def info(self) -> list[Text]:
        """Overloads superclass to avoid returning config_description because that's now in the Panel title."""
        if site_config.email_info_in_subtitle:
            return [self.subheader]
        else:
            return super().info

    @property
    def is_fwded_article(self) -> bool:
        """True if this email is just a forward of an article from WSJ or whatever."""
        if self.config:
            if self.config.is_fwded_article is not None:
                return self.config.is_fwded_article
            elif self.config.fwded_text_after:
                return True

        return False

    @property
    def is_interesting(self) -> bool | None:
        """Junk emails are not interesting."""
        if self.is_mailing_list:
            return False
        elif (is_interesting := super().is_interesting) is not None:
            return is_interesting
        elif self.participants.intersection(EMAILERS_OF_INTEREST_SET):
            return True

    @property
    def is_junk_mail(self) -> bool:
        return self.author in JUNK_EMAILERS

    @property
    def is_mailing_list(self) -> bool:
        return self.author in MAILING_LISTS or self.is_junk_mail

    @property
    def is_word_count_worthy(self) -> bool:
        """True if this file should be included in word counts."""
        if self.is_fwded_article:
            return bool(self.config.fwded_text_after) or len(self.actual_text) < 150
        else:
            return not self.is_mailing_list

    @property
    def metadata(self) -> Metadata:
        metadata = super().metadata
        metadata.update(self.truthy_props(METADATA_FIELDS))

        if not self.header.is_empty:
            metadata['header'] = self.header.as_dict()

        if self.attached_docs:
            metadata['attachment_file_ids'] = [f.file_id for f in self.attached_docs]

        return metadata

    @property
    def people(self) -> list[str]:
        """Includes people in attachments."""
        return sorted(uniquify(super().people + flatten([ad.people for ad in self.attached_docs])))

    @property
    def prettified_text(self) -> Text:
        """Cleaned up text ready for printing."""
        should_rewrite_header = self.header.was_initially_empty and self.header.num_header_rows > 0
        num_chars = self._truncate_to_length()
        trim_footer_txt = None
        text = self.text

        # Truncate long emails but leave a note explaining what happened w/link to source document
        if len(text) > num_chars:
            text = text[0:num_chars]
            trim_footer_txt = self.truncation_note(num_chars)

        text = collapse_newlines(text)

        # Rewrite broken headers where the values are on separate lines from the field names
        if should_rewrite_header:
            configured_actual_text = self.config.actual_text if self.config and self.config.actual_text else None
            num_lines_to_skip = self.header.num_header_rows
            lines = []

            # Emails w/configured 'actual_text' are particularly broken; need to shuffle some lines
            if configured_actual_text is not None:
                num_lines_to_skip += 1
                lines += [cast(str, configured_actual_text), '\n']

            lines += text.split('\n')[num_lines_to_skip:]
            text = self.with_header('\n'.join(lines))
            text = _add_line_breaks(text)
            self.rewritten_header_ids.add(self.file_id)

        if args.char_nums:
            text = self._inject_line_numbers(text, args.char_nums)

        lines = [create_hyperlinks(line) for line in text.split('\n')]

        if self.config and self.config.highlighted_pattern:
            logger.debug(f"self.config.highlighted_pattern='{self.config.highlighted_pattern}'")
            txt_highlighter = temp_highlighter(self.config.highlighted_pattern)
        else:
            txt_highlighter = highlighter

        txt = txt_highlighter(join_texts(lines, '\n'))

        if trim_footer_txt:
            txt.append('...\n\n').append(trim_footer_txt)

        return txt

    @property
    def recipients_str(self) -> str:
        return ';'.join([str(r) for r in self.recipients])

    @property
    def subheader(self) -> Text:
        prefix = 'fwded article' if self.is_fwded_article else 'email'
        author_txt = self.author_txt

        if self.config and self.config.is_attribution_uncertain:
            author_txt += Text(f" {QUESTION_MARKS}", style=self.author_style)

        return site_config.email_subheader(prefix, author_txt, self.recipients_txt(), self.timestamp)

    @property
    def subject(self) -> str:
        if self.config and self.config.subject:
            return self.config.subject
        else:
            return self.header.subject or ''

    @property
    def uninteresting_txt(self) -> Text | None:
        """Text to print for uninteresting files."""
        if (uninteresting_txt := super().uninteresting_txt):
            if self.subject:
                uninteresting_txt.append(f' ("{self.subject}")', style='light_yellow3')

            return uninteresting_txt

    @property
    def _summary(self) -> Text:
        """One line summary for logging only."""
        txt = self._summary_with_author

        if len(self.recipients) > 0:
            txt.append(', ').append(styled_key_value('recipients', self.recipients_txt()))

        return txt.append(CLOSE_PROPERTIES_CHAR)

    def file_display(self, align: JustifyMethod | None = None) -> FileDisplay:
        """Allows for proper right vs. left justify."""
        return FileDisplay(
            body_panel=self._body_as_table(self.prettified_text, self.config_description_txt),
            file_info=self.file_id_panel,
            indent=site_config.info_indent,
            justify=align,
            margin_bottom=self.html_margin_bottom,
            subheaders=self.info,
        )

    def is_from_or_to(self, name: str) -> bool:
        """True if `name` is either the author or one of the recipients."""
        return name in self.participants

    def is_note_to_self(self, recipients: list[Name] | None = None) -> bool:
        return (recipients if recipients is not None else self.recipients) == [self.author]

    def recipients_txt(self, max_full_names: int = 2) -> Text:
        """Comma separated colored names (last name only if more than `max_full_names` recipients)."""
        recipients = [r or UNKNOWN for r in self.recipients] if len(self.recipients) > 0 else [UNKNOWN]

        names = [
            Text(r if len(recipients) <= max_full_names else extract_last_name(r), get_style_for_name(r)) + \
                (Text(f" {QUESTION_MARKS}", get_style_for_name(r)) if self.is_recipient_uncertain else Text(''))
            for r in recipients
        ]

        return join_texts(names, join=', ')

    def to_html(self) -> str:
        html = super().to_html()

        if (attachments_table := self._attached_docs_table()):
            indent_props = {'margin-left': to_em(site_config.attachment_indent)}
            html += table_to_html(attachments_table, indent_props)

        return html

    def with_header(self, text: str) -> str:
        """Add the header lines to `text`."""
        return self.header.rewrite_header() + '\n' + text

    def _attached_docs_table(self) -> Table | None:
        if not self.attached_docs:
            return None

        attachments_table_title = f"| Email Attachments for {self.file_info.url_slug}:" # ╏┇┣

        if (doc := self.attached_docs[0]).config and doc.config.show_full_panel:
            if len(self.attached_docs) > 1:
                raise ValueError(f"Can't show more than one panelized attachment for {self}!")

            table = Table(title=attachments_table_title, title_justify='left')
            table.add_column(doc.config.description)
            table.add_row(highlighter(Text(self.attached_docs[0].text, EXCERPT_STYLE)))
            return table
        else:
            return OtherFile.files_preview_table(self.attached_docs, title=attachments_table_title)

    def _body_as_panel(self, text: str | Text, description: Text | None = None) -> Panel:
        """Renders the info info text in the panel's bottom border."""
        return Panel(
            text,
            border_style=self.border_style,
            expand=False,
            title=Text(' ').join(description.split(' ')) if description else None,  # split then join makes rich color subtitle correctly
            title_align='right',
        )

    def _body_as_table(self, text: str | Text, description: Text | None = None) -> Table:
        """Renders the info text as a top row in a table-ish view."""
        panel = Table(
            border_style=self.border_style,
            box=box.ROUNDED,
            header_style='on gray11',
            show_header=bool(description)
        )

        if description and site_config.email_info_in_subtitle:
            description = Text('', justify='right').append(description)

        panel.add_column(description or '')
        panel.add_row(text)
        return panel

    def _debug_props(self) -> DebugDict:
        props = super()._debug_props()
        local_props = self.truthy_props(DEBUG_PROPS)

        if not self.header.is_empty:
            local_props['header'] = self.header.as_dict()
        if self.is_note_to_self():
            local_props['is_note_to_self'] = self.is_note_to_self()

        props.update(prefix_keys(self._debug_prefix, local_props))
        return props

    def _extract_actual_text(self) -> str:
        """The text that comes before likely quoted replies and forwards etc."""
        if self.config and self.config.actual_text is not None:
            return self.config.actual_text

        text = '\n'.join(self.text.split('\n')[self.header.num_header_rows:]).strip()

        if self.config and self.config.fwded_text_after:
            return text.split(self.config.fwded_text_after)[0].strip()
        elif self.header.num_header_rows == 0:
            return self.text

        self.log_top_lines(20, "Raw text:", logging.DEBUG)
        self.log(f"With {self.header.num_header_rows} header lines removed:\n{text[0:500]}\n\n", logging.DEBUG)
        reply_text_match = REPLY_TEXT_REGEX.search(text)

        if reply_text_match:
            actual_num_chars = len(reply_text_match.group(1))
            actual_text_pct = f"{(100 * float(actual_num_chars) / len(text)):.1f}%"
            logger.debug(f"'{self.file_id}': actual_text() reply_text_match is {actual_num_chars:,} chars ({actual_text_pct} of {len(text):,})")
            text = reply_text_match.group(1)

        # If all else fails look for lines like 'From: blah', 'Subject: blah', and split on that.
        for field_name in REPLY_SPLITTERS:
            field_string = f'\n{field_name}'

            if field_string not in text:
                continue

            pre_from_text = text.split(field_string)[0]
            actual_num_chars = len(pre_from_text)
            actual_text_pct = f"{(100 * float(actual_num_chars) / len(text)):.1f}%"
            logger.debug(f"'{self.file_id}': actual_text() fwd_text_match is {actual_num_chars:,} chars ({actual_text_pct} of {len(text):,})")
            text = pre_from_text
            break

        return text.strip()

    def _extract_author(self) -> Name:
        """Overloads superclass method, called at instantiation time."""
        if self.header.author:
            authors = extract_emailer_names(self.header.author)
            return authors[0] if (len(authors) > 0 and authors[0]) else None

    def _extract_header(self) -> EmailHeader:
        """Extract an `EmailHeader` from the OCR text."""
        header_match = EMAIL_SIMPLE_HEADER_REGEX.search(self.text)

        if header_match:
            header = EmailHeader.from_header_lines(header_match.group(0))

            # DOJ file OCR text is broken in a less consistent way than the HOUSE_OVERSIGHT files
            if header.is_empty and not self.file_info.is_doj_file:
                header.repair_empty_header(self.lines)
        else:
            log_level = logging.INFO if self.config else logging.WARNING
            self.log_top_lines(msg='No email header match found!', level=log_level)
            header = EmailHeader(field_names=[])

        logger.debug(f"{self.file_id} extracted header\n\n{header}\n")
        return header

    def _extract_recipients(self) -> list[Name]:
        """Scan the To:, BCC: and CC: fields for known names, falling back to raw strings if no names identified."""
        recipients = flatten([extract_emailer_names(r) for r in self.header.recipients])
        recipients = uniquify(recipients)

        # Assume mailing list emails are to Epstein
        if self.author in BCC_LISTS and (self.is_note_to_self(recipients) or not recipients):
            recipients: list[Name] = [JEFFREY_EPSTEIN]

        # Remove self CCs but preserve self emails
        if not (self.is_note_to_self(recipients) or self.author is None):
            if self.author in self.recipients:
                self.log(f"Removing email to self for {self.author}")

            recipients = [r for r in recipients if r != self.author]

        return sort_names(recipients)

    def _extract_timestamp(self) -> datetime:
        """Find the time this email was sent."""
        if self.header.sent_at and (timestamp := _parse_timestamp(self.header.sent_at)):
            return timestamp

        searchable_lines = self.lines[0:MAX_NUM_HEADER_LINES]
        searchable_text = '\n'.join(searchable_lines)

        if (date_match := DATE_HEADER_REGEX.search(searchable_text)):
            if (timestamp := _parse_timestamp(date_match.group(1))):
                return timestamp

        logger.debug(f"Failed to find timestamp, falling back to parsing {MAX_NUM_HEADER_LINES} lines...")

        for line in searchable_lines:
            if not TIMESTAMP_LINE_REGEX.search(line):
                continue

            if (timestamp := _parse_timestamp(line)):
                logger.debug(f"Fell back to timestamp {timestamp} in line '{line}'...")
                return timestamp

        no_timestamp_msg = f"No timestamp found in '{self.file_path.name}'"

        if self.is_duplicate:
            logger.warning(f"{no_timestamp_msg} but timestamp should be copied from {self.duplicate_of_id}")
        else:
            logger.error(f"{no_timestamp_msg}, top lines:\n" + '\n'.join(self.lines[0:MAX_NUM_HEADER_LINES + 10]))

        return FALLBACK_TIMESTAMP

    def _idx_of_nth_quoted_reply(self, n: int = MAX_QUOTED_REPLIES) -> int | None:
        """Get position of the nth 'On June 12th, 1985 [SOMEONE] wrote:' style line in self.text."""
        header_offset = len(self.header.header_chars)
        text = self.text[header_offset:]

        for i, match in enumerate(QUOTED_REPLY_LINE_REGEX.finditer(text)):
            if i >= n:
                return match.end() + header_offset - 1

    def _merge_lines(self, idx1: int, idx2: int | None = None) -> None:
        """Combine lines numbered 'idx' and 'idx2' into a single line (idx2 defaults to idx + 1)."""
        if idx2 is None:
            self._line_merge_arguments.append((idx1,))
            idx2 = idx1 + 1
        else:
            self._line_merge_arguments.append((idx1, idx2))

        if idx2 < idx1:
            lines = self.lines[0:idx2] + self.lines[idx2 + 1:idx1] + [self.lines[idx1] + ' ' + self.lines[idx2]] + self.lines[idx1 + 1:]
        elif idx2 == idx1:
            raise RuntimeError(f"idx2 ({idx2}) must be greater or less than idx ({idx1})")
        else:
            lines = self.lines[0:idx1]

            if idx2 == (idx1 + 1):
                lines += [self.lines[idx1] + ' ' + self.lines[idx1 + 1]] + self.lines[idx1 + 2:]
            else:
                lines += [self.lines[idx1] + ' ' + self.lines[idx2]] + self.lines[idx1 + 1:idx2] + self.lines[idx2 + 1:]

        self._set_text(lines=lines)

    def _remove_line(self, idx: int) -> None:
        """Remove a line from `self.lines`."""
        num_lines = idx * 2
        self.log_top_lines(num_lines, msg=f'before removal of line {idx}')
        del self.lines[idx]
        self._set_text(lines=self.lines)
        self.log_top_lines(num_lines, msg=f'after removal of line {idx}')

    def _repair(self) -> None:
        """Repair particularly janky files. Note that OCR_REPAIRS are applied *after* other line adjustments."""
        # Some DOJ cleanup needs to happen first if this is a DOJ file.
        super()._repair()

        if self.file_info.is_doj_file:
            new_text = DojFile._remove_bad_lines(strip_pdfalyzer_panels(self.text))
            self._set_text(text=self.repair_ocr_text(DOJ_EMAIL_OCR_REPAIRS, new_text))

        if BAD_FIRST_LINE_REGEX.match(self.lines[0]):
            self._set_text(lines=self.lines[1:])

        # print(self._numbered_lines())
        # import pdb;pdb.set_trace()
        self._set_text(lines=[line for line in self.lines if not BAD_LINE_REGEX.match(line)])
        old_text = self.text

        if self.file_id in LINE_REPAIR_MERGES:
            for merge_args in LINE_REPAIR_MERGES[self.file_id]:
                self._merge_lines(*merge_args)

        if self.file_id in ['025233']:
            self.lines[4] = f"Attachments: {self.lines[4]}"
            self._set_text(lines=self.lines)
        elif self.file_id == '029977':
            self._set_text(text=self.text.replace('Sent 9/28/2012 2:41:02 PM', 'Sent: 9/28/2012 2:41:02 PM'))

        # Bad line removal
        if self.file_id == '025041':
            self._remove_line(4)
            self._remove_line(4)
        elif self.file_id == '029692':
            self._remove_line(3)

        if old_text != self.text:
            self.log(f"Modified text, old:\n\n" + '\n'.join(old_text.split('\n')[0:12]) + '\n')
            self.log_top_lines(12, 'Result of modifications')

        repaired_text = self._repair_links_and_quoted_subjects(self.repair_ocr_text(OCR_REPAIRS, self.text))
        # logger.debug(f"repaired_text\n---\n{self.text}\n---")
        self._set_text(text=repaired_text)
        self._strip_unwanted_text()
        # TODO: we're currently calling this twice to handle lines with HTML garbage that turn into something matching BAD_LINE_REGEX
        self._set_text(lines=[line for line in self.lines if not BAD_LINE_REGEX.match(line)])

    def _repair_links_and_quoted_subjects(self, text: str) -> str:
        """Repair links that the OCR has broken into multiple lines as well as 'Subject:' lines."""
        lines = text.split('\n')
        subject_line = next((line for line in lines if line.startswith('Subject:')), None) or ''
        subject = subject_line.split(':')[1].strip() if subject_line else ''
        new_lines = []
        i = 0

        while i < len(lines):
            line = lines[i]

            if LINK_LINE_REGEX.search(line):
                while i < (len(lines) - 1) \
                        and not (lines[i + 1].startswith('htt') or lines[i + 1].startswith('www') or lines[i + 1].endswith('com')) \
                        and (lines[i + 1].endswith('/') \
                            or any(s in lines[i + 1] for s in URL_SIGNIFIERS) \
                            or LINK_LINE2_REGEX.match(lines[i + 1])):
                    logger.debug(f"{self.filename}: Joining link lines\n   1. {line}\n   2. {lines[i + 1]}\n")
                    line += lines[i + 1]
                    i += 1

                line = line if '[' in line or ']' in line else line.replace(' ', '')
            elif ' http' in line and line.endswith('html'):
                pre_link, post_link = line.split(' http', 1)
                line = f"{pre_link} http{post_link.replace(' ', '')}"
            elif line.startswith('Subject:') and i < (len(lines) - 2) and len(line) >= 40:
                next_line = lines[i + 1]
                next_next = lines[i + 2]

                if len(next_line) <= 1 or any([cont in next_line for cont in BAD_SUBJECT_CONTINUATIONS]):
                    pass
                elif (subject.endswith(next_line) and next_line != subject) \
                        or (HEADER_FIELD_COLON_REGEX.search(next_next) and not HEADER_FIELD_COLON_REGEX.search(next_line)):
                    self.log(f"Fixing broken subject line\n  line: '{line}'\n    next: '{next_line}'\n    next: '{next_next}'\nsubject='{subject}'\n")
                    line += f" {next_line}"
                    i += 1

            new_lines.append(line)
            i += 1

        logger.debug(f"----after line repair---\n" + '\n'.join(new_lines[0:20]) + "\n---")
        return '\n'.join(new_lines)

    def _sent_from_device(self) -> str | None:
        """Find any 'Sent from my iPhone' style signature line if it exist in the 'actual_text'."""
        if (sent_from_match := SENT_FROM_REGEX.search(self.actual_text)):
            sent_from = sent_from_match.group(0).strip()

            if sent_from.startswith('Typos, misspellings courtesy of iPhone'):  # special handling for linda stone periods
                return sent_from.removesuffix('.')
            elif sent_from.startswith('sent'):
                return capitalize_first(sent_from)
            else:
                return sent_from

    def _strip_unwanted_text(self) -> None:
        """Add newlines before quoted replies, snip signatures and XML, etc.."""
        # Insert line breaks now unless header is broken, in which case we'll do it later after fixing header
        # self.(f"text before _add_line_breaks:\n\n{self.text}\n---")
        text = self.text if self.header.was_initially_empty else _add_line_breaks(self.text)
        text = REPLY_REGEX.sub(r'\n\1', text)  # Newlines between quoted replies
        text = FORWARDED_TOO_MUCH_SPACE_REGEX.sub(r'\1\n', text)
        # self.warn(f"text after _add_line_breaks:\n\n{text}\n---")

        for name, signature_regex in EMAIL_SIGNATURE_REGEXES.items():
            signature_replacement = f'<...snipped {name.lower()} email signature...>'
            text, num_replaced = signature_regex.subn(signature_replacement, text)
            self.signature_substitution_counts[name] = self.signature_substitution_counts.get(name, 0)
            self.signature_substitution_counts[name] += num_replaced

        # Remove XML cruft in some files
        text, num_plists_stripped = XML_PLIST_REGEX.subn(XML_STRIPPED_MSG, text)

        if num_plists_stripped:
            self.log(f"Replaced {num_plists_stripped} XML plists...")

        self._set_text(text=collapse_newlines(text).strip())

    def _truncate_to_length(self) -> int:
        """Decide how many chars we should limit the dislpay of this email to."""
        quote_cutoff = self._idx_of_nth_quoted_reply()  # Trim if there's many quoted replies
        includes_truncate_term = next((term for term in TRUNCATE_TERMS if term in self.text), None)
        num_chars: int

        if args.whole_file:
            num_chars = len(self.text)
        elif args.truncate:
            num_chars = args.truncate
        elif self.config and (truncate_at := self.config.truncate_at):
            if self.config.is_excerpt:
                raise ValueError(f"Emails don't support truncate_to as a tuple")

            num_chars = len(self.text) if truncate_at == NO_TRUNCATE else truncate_at
        elif self.author in TRUNCATE_EMAILS_BY \
                or any([self.is_from_or_to(n) for n in TRUNCATE_EMAILS_FROM_OR_TO]) \
                or includes_truncate_term:
            num_chars = min(quote_cutoff or DEFAULT_TRUNCATE_TO, SHORT_TRUNCATE_TO)
        else:
            if quote_cutoff and quote_cutoff < DEFAULT_TRUNCATE_TO:
                trimmed_words = self.text[quote_cutoff:].split()

                if '<...snipped' in trimmed_words[:NUM_WORDS_IN_LAST_QUOTE]:
                    num_trailing_words = 0
                elif trimmed_words and trimmed_words[0] in ['From:', 'Sent:']:
                    num_trailing_words = NUM_WORDS_IN_LAST_QUOTE
                else:
                    num_trailing_words = NUM_WORDS_IN_LAST_QUOTE

                if trimmed_words:
                    last_quoted_text = ' '.join(trimmed_words[:num_trailing_words])
                    num_chars = quote_cutoff + len(last_quoted_text) + 1 # Give a hint of the next line
                else:
                    num_chars = quote_cutoff
            else:
                # TODO: Added some padding to self.length because newlines may be added in prettification but this sucks
                num_chars = min(self.length + 100, DEFAULT_TRUNCATE_TO)

            # Always print whole email for 1st email for actual people
            if self.is_persons_first_email and num_chars < self.length and \
                    not (self.is_duplicate or self.is_fwded_article or self.is_mailing_list):
                self.log(f"{self} Overriding cutoff {num_chars} for first email")
                num_chars = self.length + 100

        log_args = {
            'num_chars': num_chars,
            'is_persons_first_email': self.is_persons_first_email,
            'author_truncate': self.author in TRUNCATE_EMAILS_BY,
            'is_fwded_article': self.is_fwded_article,
            'is_quote_cutoff': quote_cutoff == num_chars,
            'includes_truncate_term': json.dumps(includes_truncate_term) if includes_truncate_term else None,
            'quote_cutoff': quote_cutoff,
        }

        log_args_str = ', '.join([f"{k}={v}" for k, v in log_args.items() if v])
        self.log(f"Truncate determination: {log_args_str}")
        return num_chars

    def __rich_console__(self, console: Console, options: ConsoleOptions) -> RenderResult:
        prettified_txt = self.prettified_text
        max_line_len = max(*[len(line) for line in prettified_txt.split('\n')])
        panel_subtitle = None

        if self.config_description_txt and site_config.email_info_in_subtitle:
            max_line_len = max(max_line_len, len(self.config_description_txt.plain))
            panel_subtitle = Text('').append(self.config_description_txt)
            panel_subtitle.justify = 'right'

        if args.panelize_emails:
            body = self._body_as_panel(prettified_txt, panel_subtitle)
        else:
            body = self._body_as_table(prettified_txt, panel_subtitle)

        yield self.rich_header()
        body_bottom_padding = 0 if self.attached_docs else 1
        yield Padding(body, (0, 0, body_bottom_padding, site_config.other_files_table_indent))

        if (attachments_table := self._attached_docs_table()):
            yield Padding(attachments_table, (0, 0, 1, site_config.attachment_indent))

    @staticmethod
    def build_emails_table(emails: list['Email'], name: Name = '', title: str = '', show_length: bool = False) -> Table:
        """Turn a list of `Email` objects into a `Table` with sender, recipient, and subject line."""
        if title and name:
            raise ValueError(f"Can't provide both 'author' and 'title' args")
        elif name == '' and title == '':
            raise ValueError(f"Must provide either 'author' or 'title' arg")

        author_style = get_style_for_name(name, allow_bold=False)
        link_style = author_style if name else ARCHIVE_LINK_COLOR
        min_width = len(name or UNKNOWN)
        max_width = max(20, min_width)

        columns = [
            {'name': 'Sent At', 'justify': 'left', 'style': TIMESTAMP_DIM},
            {'name': 'From', 'justify': 'left', 'min_width': min_width, 'max_width': max_width},
            {'name': 'To', 'justify': 'left', 'min_width': min_width, 'max_width': max_width + 2},
            {'name': 'Length', 'justify': 'right', 'style': 'wheat4'},
            {'name': 'Subject', 'justify': 'left', 'min_width': 35, 'style': 'honeydew2'},
        ]

        table = build_table(
            title or None,
            cols=[col for col in columns if show_length or col['name'] not in ['Length']],
            border_style=DEFAULT_TABLE_KWARGS['border_style'] if title else author_style,
            header_style="bold",
            highlight=True,
        )

        for email in emails:
            fields = [
                link_text_obj(email.file_info.external_url, email.timestamp_without_seconds, style=link_style),
                email.author_txt,
                email.recipients_txt(max_full_names=1),
                f"{email.length}",
                email.subject,
            ]

            if not show_length:
                del fields[3]

            table.add_row(*fields)

        return table


def _add_line_breaks(email_text: str) -> str:
    text = EMAIL_SIMPLE_HEADER_LINE_BREAK_REGEX.sub(r'\n\1\n', email_text).strip()
    logger.debug(f"text after EMAIL_SIMPLE_HEADER_LINE_BREAK_REGEX _add_line_breaks()\n---\n{text}\n---")
    return FORWARDED_TOO_MUCH_SPACE_REGEX.sub(r'\1\n', text)


def _parse_timestamp(timestamp_str: str) -> None | datetime:
    try:
        if (american_date_match := AMERICAN_TIME_REGEX.search(timestamp_str)):
            timestamp_str = american_date_match.group(1)
        else:
            timestamp_str = timestamp_str.replace('(GMT-05:00)', 'EST')
            timestamp_str = BAD_TIMEZONE_REGEX.sub(' ', timestamp_str).strip()

        timestamp = parse(timestamp_str, fuzzy=True, tzinfos=TIMEZONE_INFO)
        logger.debug(f'Parsed timestamp "%s" from string "%s"', timestamp, timestamp_str)
        return remove_timezone(timestamp)
    except Exception as e:
        logger.debug(f'Failed to parse "{timestamp_str}" to timestamp!')
