import re
import webbrowser
import os
import threading
from ..dialogs import KeyHandler,FormDialog,ElementInterface,Element,ValuesElementInterface,ValuesElement,Dialog,SingleTextElementInterface,SingleTextElement,TextElementInterface,TextElement,LabelElement,BooleanElement,ListElement,ButtonElement
from .editor import EditDialog
from .explorer import open_file
from ..google_contacts import ContactsInputDialog,split_email_list, validate_and_normalize,clean_emails
from ..framework import beep, ensure_dir_exists
from .. import punctuator
from datetime import datetime, timedelta, timezone
import pytz
import functools
from ..google_api import online_only,is_connected,create_service,execute_with_retry,GoogleDialog
from ..secrets_paths import (
    get_gmail_labels_cache_path_with_fallback,
    get_gmail_folder_cache_path_with_fallback,
    get_gmail_message_cache_path_with_fallback
)
import json
import email.utils
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from email.mime.base import MIMEBase
from email import encoders
import html 
from bs4 import BeautifulSoup, NavigableString, Comment
#from icalendar import Calendar, Event
import hashlib
import csv
import io
import base64
from collections import namedtuple
ContentElement = namedtuple('ContentElement', ['type', 'content', 'level', 'attributes', 'inline_elements'])
Tag = namedtuple('Tag',['name','start','length','attributes'])
# Dictionary to map file extensions to MIME types
MIME_TYPES = {
    # Text Files
    '.txt': 'text/plain',
    '.html': 'text/html',
    '.htm': 'text/html',
    '.css': 'text/css',
    '.js': 'application/javascript',
    '.csv': 'text/csv',
    '.xml': 'application/xml',
    '.json': 'application/json',
    # Image Files
    '.jpg': 'image/jpeg',
    '.jpeg': 'image/jpeg',
    '.png': 'image/png',
    '.gif': 'image/gif',
    '.bmp': 'image/bmp',
    '.tiff': 'image/tiff',
    '.tif': 'image/tiff',
    '.svg': 'image/svg+xml',
    '.webp': 'image/webp',
    # Audio Files
    '.mp3': 'audio/mpeg',
    '.wav': 'audio/wav',
    '.ogg': 'audio/ogg',
    '.mid': 'audio/midi',
    '.midi': 'audio/midi',
    '.aac': 'audio/aac',
    # Video Files
    '.mp4': 'video/mp4',
    '.avi': 'video/x-msvideo',
    '.mpeg': 'video/mpeg',
    '.mpg': 'video/mpeg',
    '.webm': 'video/webm',
    '.ogv': 'video/ogg',
    '.mov': 'video/quicktime',
    '.wmv': 'video/x-ms-wmv',
    # Application Files
    '.pdf': 'application/pdf',
    '.zip': 'application/zip',
    '.gz': 'application/gzip',
    '.tar': 'application/x-tar',
    '.doc': 'application/msword',
    '.dot': 'application/msword',
    '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
    '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
    '.xls': 'application/vnd.ms-excel',
    '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
    '.ppt': 'application/vnd.ms-powerpoint',
    '.odt': 'application/vnd.oasis.opendocument.text',
    '.ods': 'application/vnd.oasis.opendocument.spreadsheet',
    '.odp': 'application/vnd.oasis.opendocument.presentation',
    '.rtf': 'application/rtf',
    '.epub': 'application/epub+zip',
    '.swf': 'application/x-shockwave-flash',
    '.json': 'application/json',
    '.ics': 'text/calendar',
    # Executable Files
    '.exe': 'application/vnd.microsoft.portable-executable',
    '.sh': 'application/x-sh',
    # Font Files
    '.woff': 'font/woff',
    '.woff2': 'font/woff2',
    '.ttf': 'font/ttf',
    '.otf': 'font/otf',
}
def clean_unspeakables(text):
    return punctuator.substitute(text,punctuator.unspeakables)
def wrap_text(text, width=160):
    # Use regex to split at or before the specified width, preserving all spaces
    lines = re.findall(r'.{1,%d}(?=\s|$)' % width, text)
    # No rstrip() or additional processing to ensure full preservation of spaces
    return lines
import re
def find_url_before_offset(input: str, offset: int):
    # Pattern to match a URL, now including ? and # in the allowed characters
    url_pattern = r'\b[\w.-]+://[\w\.-?/#]+\b'
    # Adjust offset if the character before it is a non-URL character
    if offset > 0 and not re.match(r'[\w\.-?/#]', input[offset - 1]):
        offset -= 1
    # Limit search up to the adjusted offset
    substring = input[:offset]
    # Find the last URL match in the substring
    matches = list(re.finditer(url_pattern, substring))
    # Check if we found any matches
    if not matches:
        return None
    # Get the last match
    last_match = matches[-1]
    # Check if the match ends exactly at the offset or at the adjusted offset
    if last_match.end() == offset or (offset < len(input) and not re.match(r'[\w\.-?/#]', input[offset])):
        # Return (start position, length, matched URL string)
        return (last_match.start(), last_match.end() - last_match.start(), last_match.group(0))
    return None
def is_url(input: str, offset: int):
    # Pattern to match a URL 
    url_pattern = r'\b[\w.-]+://[\w\.-?/#]+\b'
    # Adjust offset if the character before it is a non-URL character
    if offset > 0 and not re.match(r'[\w\.-?/#]', input[offset - 1]):
        offset -= 1
    # Limit search up to the adjusted offset
    substring = input[:offset]
    # Find the last URL match in the substring
    matches = list(re.finditer(url_pattern, substring))
    # Check if we found any matches
    if not matches:
        return None
    # Get the last match
    last_match = matches[-1]
    # Check if the match ends exactly at the offset or at the adjusted offset
    if last_match.end() == offset or (offset < len(input) and not re.match(r'[\w\.-?/#]', input[offset])):
        # Return (start position, length, matched URL string)
        return (last_match.start(), last_match.end() - last_match.start(), last_match.group(0))
    return None
def url_before_offset(input: str, offset: int):
    # Pattern to match a URL
    url_pattern = r'\b[\w.-]+://[\w.-]+\b'
    # Adjust offset if the character before it is a non-URL character
    if offset > 0 and not re.match(r'[\w.-/?#]', input[offset - 1]):
        offset -= 1
    # Limit search up to the adjusted offset
    substring = input[:offset]
    # Find the last URL match in the substring
    matches = list(re.finditer(url_pattern, substring))
    # Check if we found any matches
    if not matches:
        return None
    # Get the last match
    last_match = matches[-1]
    # Check if the match ends exactly at the offset or at the adjusted offset
    if last_match.end() == offset or (offset < len(input) and not re.match(r'[\w.-]', input[offset])):
        # Return (start position, length, matched URL string)
        return (last_match.start(), last_match.end() - last_match.start(), last_match.group(0))
    return None
def is_url(url):
    # Permissive regex for URL validation
    url_pattern = re.compile(
        r"^(https?://)?(www\.)?([a-zA-Z0-9-]+\.)+[a-zA-Z]{2,6}(/[\w.-]*)*"
    )
    return bool(url_pattern.match(url))
def html_reply(original_soup,reply_lines):
    # Join the reply text lines with HTML line breaks
    reply_content='<br>'.join(reply_lines)
    body = original_soup.body if original_soup.body else original_soup
    body.insert(0, BeautifulSoup(reply_content, 'html.parser'))
    # Return the modified HTML as a string
    return str(original_soup)
def label_priority(label):
    if label['type']=='user':
        priority=1
    else:
        if label['name']=='INBOX':
            priority=2
        elif label['name']=='SENT': 
            priority=3
        else:
            priority=4
    return (priority,label['name'])
def overlap(tag: Tag, line_start: int, line_length:int) -> tuple:
    tag_end = tag.start + tag.length
    line_end = line_start+line_length
    # Check if there is no overlap
    #cater for zero length tags at start of line 
    if ((tag.start >= line_end) or (tag_end <= line_start)) and not ((tag.start==line_start) and (tag.length==0)):
        return None
    # Determine the overlapping start and end positions relative to full_text
    overlap_start = max(line_start, tag.start)
    overlap_end = min(line_end, tag_end)
    # Convert to positions relative to the line
    relative_start = overlap_start - line_start
    overlap_length = overlap_end - overlap_start
    return (relative_start, overlap_length)
def reply_addresses(reply_type,owner,input_dict):
    addresses={'From':owner}
    if 'reply' in reply_type:
        addresses['To']=input_dict['From']
        if 'all' in reply_type:
            addresses['To']= ', '.join([addresses['To'],input_dict.get('To','')])
            addresses['Cc']=','.join([input_dict.get('Cc',''),input_dict.get('Bcc','')]).strip(' ,') #remove any extraneous spaces and commaas on start and end of string if one of the cc and bcc strings was empty 
    return addresses
def prefixed_subject(reply_type,base_subject):
    if reply_type == 'forward':
        prefix ='FW:'  
    else: 
        prefix='RE:'
    #strip leading Re: or Fw: off the subject to avoid, e.g. 're:Fw
    if base_subject[:3].lower() in ['re:','fw:']:
        base_subject=base_subject[3:]
    return prefix+base_subject 
def message_filler(action,addresses={},subject='',summary={},prefer_text_format=None,content={},include_attachments=False):
    if action=='view':
        address_list_class = Dialog  
        subject_class = LabelElement
    else:
        address_list_class=people.ContactsInputDialog  
        subject_class=SingleTextElement 
    address_fields = ['To','Cc']
    #do not display Bcc field for viewed messages unless there is a 'Bcc' header due to either the owner of this mailbox being in the Bcc field themselves or this being the Sent folder and Bcc entries have been provided when the message was created  
    if ('Bcc' in addresses) or (action!='view'):
        address_fields.append('Bcc')        
        fields=[[LabelElement,'From',addresses.get('From','')]]
        fields.extend([[address_list_class,group,Element,split_email_list(addresses.get(group,''))] for group in address_fields])
    fields.append([subject_class,'Subject',subject])
    if include_attachments:
        attachments=attachments_from_content(content)
        if len(attachments):
            fields.append([ListElement,'attachments','',[a[0] for a in attachments]])
    fields.append(            [BodyDialog,'Body',TaggedElement,[action,prefer_text_format,summary]])
    return fields
def content_from_message(message):
    email_content = {"text/plain": None, "text/html": None, "inline_attachments": [], "attachment_ids": []}
    if message:
        # Check if the message is multipart
        if 'parts' in message['payload']:
            for part in message['payload']['parts']:
                process_part(part, email_content)
        else:
            process_part(message['payload'], email_content)
    return email_content 
def attachments_from_content(content):
    attachments=[]
    for a in content['attachment_ids']:
        #recursive walking of payload seems to sometimes add two versions of attachment with same filename but different attachment id. Currently just take the first one as they should represent the same ultimate file 
        if not a['filename'] in attachments:
            attachments.append((a['filename'],a['attachment_id'],'id'))
    count=1
    for a in content['inline_attachments']:
        attachments.append((f'inline attachment {count}',i['data'],'inline'))                
        count+=1
    return attachments 
def get_mime_type(file_extension):
    """
    Returns the correct MIME type for a given file extension.
    Defaults to 'application/octet-stream' if the extension is not found.
    :param file_extension: The file extension (e.g., '.pdf')
    :return: The corresponding MIME type
    """
    # Ensure the file extension is lowercase for consistent lookup
    return MIME_TYPES.get(file_extension.lower(), 'application/octet-stream')
def header_lines(summary):
    lines =[
            '',
            'Original Message------------------------',
            f"Sent: {reformat(summary.get('Date'))}",
            f"From: {summary.get('From','')}",
            f"To: {summary.get('To','')}",                
            f"Cc:, {summary.get('Cc','')}",
            f"Bcc: {summary.get('Bcc','')}",
            f"Subject: {summary.get('Subject','')}",                                                                
            ]
    return lines
def extract_addresses(headers):
    addresses = {
        'From': [],
        'To': [],
        'Cc': [],
        'Bcc': []
    }
    for header in headers:
        if header['name'] in addresses:
            addresses[header['name']].extend(email.utils.getaddresses([header['value']]))
    return addresses
def make_api_message(dialog,input,owner):
    """
    Creates a message for the gmail api from the headers and body in input which is the results dict returned by MessageDialog()   
    '__format__' is used as argument to MIMEText specifying whether the body string in input is 'html' or 'plain'  (i.e. plain text)  
    """
    message = MIMEMultipart()    
    #input['Body'] is a string and input['__format__'] is either 'plain' or 'html' 
    message.attach(MIMEText(input['Body'], input['__format__']))
    addresses = []
    keys=('To','Cc','Bcc','From')
    email_lists={key:split_email_list(input[key],owner) for key in keys}  
    email_lists['From'] = email_lists['From'] if email_lists['From'] else [owner]
    for key in keys:
        #validate and normalize emails then join with ',' rather than ';' and place in the message dict 
        clean_list=clean_emails(email_lists[key])
        message[key] = ','.join(clean_list)
        #add the emails to the list of addresses for updating the recent contacts cache  and app.recent_contacts list 
        addresses.extend(clean_list)
    #update the app.recent_contacts list and also update the cache  
    people.update_recent_contacts(dialog.app,addresses)
    message['Subject'] = input['Subject']
    if 'replyHeaders' in input:
        message['In-Reply-To']=input['replyHeaders']['In-Reply-To']
        message['References']=input['replyHeaders']['References']
    for file_path  in input.get('attachments',[]):
        extension = os.path.splitext(file_path)[-1]
        mime_type=get_mime_type(extension)
        filename = os.path.basename(file_path)
        # Open the file in binary mode
        with open(file_path, 'rb') as f:
            # Create a MIMEBase object
            mime_part = MIMEBase(*mime_type.split('/'))
            mime_part.set_payload(f.read())
            encoders.encode_base64(mime_part)  # Encode the attachment in base64
            # Add header for the attachment
            mime_part.add_header('Content-Disposition', f'attachment; filename={filename}')
            # Attach the MIMEBase object to the message
            message.attach(mime_part)        
    # Encode the message in base64.
    encoded_message = base64.urlsafe_b64encode(message.as_bytes()).decode()
    return {'raw': encoded_message}
label_id =lambda labels,name:[l['id'] for l in labels if (l['name']==name)][0]
def zzz_preprocess_html(html_content):
    """
    Preprocesses HTML content to insert newlines for block-level tags and spaces
    after periods if necessary before extracting text.
    """
    # Replace block-level elements with newlines
    for br in html_content.find_all("br"):
        br.replace_with("\n")
    for p in html_content.find_all(["p", "div"]):
        p.append("\n")
    # Convert the modified HTML back to a string
    html_str = str(html_content)
    # Optionally, insert space after periods followed by a tag (careful with abbreviations and decimal numbers)
    # This regex looks for a period followed by any HTML tag and inserts a space after the period if not already present.
    # Use cautiously as it might introduce spaces in unintended places like within URLs or email addresses.
    html_str = re.sub(r'\.([<])', r'. \1', html_str)
    return html_str
def b64_decode(bytes):
    return base64.urlsafe_b64decode(bytes).decode('utf-8')
def zzz_alternative_extract_text(html):
    """
    Extracts text from HTML, preserving as much formatting as possible,
    particularly with regard to newlines and sentence spacing.
    """
    soup = BeautifulSoup(html, 'html.parser')
    preprocessed_html = preprocess_html(soup)
    text = BeautifulSoup(preprocessed_html, 'html.parser').get_text()
    return text
def zzz_extract_text(html_content):
    # Parse the HTML content
    soup = BeautifulSoup(html_content, 'html.parser')
    renderer = HTMLRenderer()
    content_elements=renderer.parse_soup(soup)
    current_element = renderer.get_current_element()
    e_list=[]
    while current_element:
        e_list.append(renderer.generate_audio_description(current_element))
        current_element = renderer.navigate('next')
    text = '\n'.join(e_list) 
    # Remove script and style elements
    #for script_or_style in soup(['script', 'style']):
        #script_or_style.decompose()
    # Convert HTML entities to plain text
    text = html.unescape(text)
    # Replace multiple spaces and newlines for readability
    lines = (line.strip() for line in text.splitlines())
    chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
    text = '\n'.join(chunk for chunk in chunks if chunk)
    # Replace Unicode characters as needed
    text = text.replace('\u2007', ' ')  # Replace Figure Space with a regular space
    text = text.replace('\u00a0', ' ')  # Replace non breaking space  with a regular space
    text = text.replace('\u034f', '')   # Remove Combining Double Macron or replace with appropriate ASCII
    text = text.replace('\u00ad', '')   # Remove soft hyphens 
    text = text.replace('\u200c', '')   # Remove zero width non-joiners &zwnj      
    return text
def hash_id(attachment_id):
    """
    unused and cannot remember why this was ever included 
    Generate a SHA-256 hash for a given attachment ID.
    Args:
    - attachment_id: The attachment ID to be hashed.
    Returns:
    - A hexadecimal string representation of the SHA-256 hash of the attachment ID.
    """
    # Create a SHA-256 hash object
    hash_obj = hashlib.sha256()
    # Update the hash object with the attachment ID, encoded as bytes
    hash_obj.update(attachment_id.encode('utf-8'))
    # Return the hexadecimal digest of the hash
    return hash_obj.hexdigest()
def process_part(part, email_content):
    """Process each part of the email, extracting the body, attachment IDs, and inline attachments."""
    part_body = part['body']
    filename=part.get('filename')
    # Handling body content
    if part['mimeType'] in ( 'text/plain','text/html')and 'data' in part_body:
        text = b64_decode(part_body['data']) 
        key = part['mimeType'] 
        if email_content[key] is None:
            email_content[key] = text
        else:
            email_content[key] += text        
    elif 'attachmentId' in part_body:   
        email_content['attachment_ids'].append({'filename':filename,'attachment_id':part_body['attachmentId']})
        if 'headers' in part and any(header['name'].lower() == 'content-disposition' and header['value'].lower() in ['inline','attachment'] for header in part['headers']):
            attachment = part['payload']['body']
            email_content['inline_attachments'].append({'filename':filename,'data':base64.urlsafe_b64decode(attachment['data'])})
    elif 'parts' in part:  # Nested multi-part
        parse_parts(part['parts'], email_content)
def parse_parts(parts, email_content):
    """Recursively parse parts of the email."""
    email_body_parts = []
    for part in parts:
        process_part(part, email_content)
def zzz_parse_calendar_content(calendar_content):
    """
    Parses calendar (iCalendar) content and extracts key details.
    """
    cal = Calendar.from_ical(calendar_content)
    event_details = []
    for component in cal.walk():
        if component.name == "VEVENT":
            summary = str(component.get('summary'))
            start = component.get('dtstart').dt
            end = component.get('dtend').dt
            location = str(component.get('location', 'No location provided'))
            # Ensure datetime is timezone-aware and converted to a simple string format
            if isinstance(start, datetime):
                start = start.astimezone(pytz.utc).strftime('%Y-%m-%d %H:%M:%S UTC')
            if isinstance(end, datetime):
                end = end.astimezone(pytz.utc).strftime('%Y-%m-%d %H:%M:%S UTC')
            event_details.append(f"Event: {summary}, Location: {location}, Start: {start}, End: {end}")
    return " ".join(event_details)
def parse_csv_string(csv_string):
    # Use io.StringIO to convert the string to a file-like object
    csv_file = io.StringIO(csv_string)
    # Create a csv reader object
    reader = csv.reader(csv_file)
    # Read the first (and only) line from the reader and return it as a list
    return next(reader)
def parse_addresses(header):
    # Regular expression to match email addresses and names
    pattern = r'(?:"?([^"]*)"?\s)?(?:<)?([^@\s]+@[^@\s]+\.[^@\s]+)(?:>)?'
    matches = re.findall(pattern, header)
    return [(name.strip(""""'- """) if name else '', email) for name, email in matches]
def rfc2822_from_milliseconds(msec_date):
    """
    takes a millisecond timestamp string such as that in the 'internalDate' value of an email and returns a date in the same format as the 'date' field
    used to create the 'date' value for a summary if the 'date' field is not present in the response from the server, such as in a draft email  
    """
    date_seconds= int(msec_date) / 1000
    dt_obj = datetime.fromtimestamp(date_seconds, tz=timezone.utc)
    # Format to RFC 2822 to be compatible with the 'date' field of emails 
    return dt_obj.strftime('%a, %d %b %Y %H:%M:%S %z')
def parse_email_date(date_str):
    dt_tuple = email.utils.parsedate_tz(date_str)
    if dt_tuple is None:
        return None  # Date string could not be parsed
    # Convert the parsed tuple to a datetime object
    date_without_timezone = datetime(*dt_tuple[:6])
    # Handle the timezone offset if present
    tz_offset_secs = dt_tuple[-1]
    if tz_offset_secs is not None:
        tz_offset = timedelta(seconds=tz_offset_secs)
        date_with_timezone = date_without_timezone.replace(tzinfo=timezone(tz_offset))
    else:
        # No timezone information, assume naive datetime object
        date_with_timezone = date_without_timezone
    return date_with_timezone
def sortable_date(date_str):
    date_obj=parse_email_date(date_str)
    #return date_obj
    # Convert the datetime object to London. Not sure if this will go wrong in other timezones but might just work anyway 
    return  date_obj.astimezone(pytz.timezone('Europe/London') )
def reformat(date_str, user_timezone=pytz.timezone('Europe/London') ):
    if date_str is None:
        return 
    date_obj=parse_email_date(date_str)
    # Convert the datetime object to the user's timezone
    date_time = date_obj.astimezone(user_timezone)
    # Format the datetime object for display
    #formatted_date = date_time.strftime('%d %b %Y, %H:%M')
    time_str = date_time.strftime(', %H:%M')
    if date_time.date() == datetime.today().date():
        date_str = 'Today'
    else: 
        date_str = date_time.strftime('%a %d %B')
        if not (date_time.year == datetime.today().year):
            date_str+=date_time.strftime(' %Y')   
    formatted_date =  date_str+time_str
    return formatted_date
def split_and_wrap(full_text):
        # Split the text into a list of lines by splitting at '\n'
    lines = full_text.split('\n')
    start_pos=0
    tuple_list=[]
    for line in lines:
        #ad to a list of text lines each preceded by the  offset in full_text of this line substring
        #start_pos for the first line of a wrapped list of tuples is adjusted on each iteration to account for the \n that has been lost in the split, whereas the wrapped_tuples function does not need to adjust the start_pos on subsequent lines in the list as wrap function does not remove \n line breaks, but simply breaks at or just before max_width 
        tuple_list.extend(wrapped_tuples(start_pos,line))
        start_pos+=(len(line)+1)
    return tuple_list
def wrapped_tuples(start_pos,line):
    tuples_list=[]
    wrapped_lines = wrap_text(line,width=160)   
    if len(wrapped_lines):
        tuples_list.append( (start_pos,wrapped_lines[0]))
        start_pos+=len(wrapped_lines[0])
        for i in range(1,len(wrapped_lines)):
            tuples_list.append((start_pos,wrapped_lines[i])) 
            start_pos+=len(wrapped_lines[i])
    return tuples_list
def filter_dict_by_keys(original_dict, filter_keys):
    """
    Remove all entries from the dictionary if the entry key is not in the filter list.
    Args:
    original_dict (dict): The original dictionary.
    filter_keys (list): The list of keys to keep in the dictionary.
    Returns:
    dict: A new dictionary containing only the filtered entries.
    """
    # Convert the filter list to a set for O(1) average time complexity for lookups
    filter_set = set(filter_keys)
    # Use dictionary comprehension to filter the original dictionary
    filtered_dict = {key: value for key, value in original_dict.items() if key in filter_set}
    return filtered_dict
def folder_from_label_ids(label_ids,labels,names):
    """
    returns the folder name from the list of label ids in a message provided as the first parameter
    The second parameter is a dict of all labels in the mailbox keyed on label_id
    The names list in the 3rd parameter contains the list of all folder names (typically created using MailDialog.folder_names())
    When a message has more than one label_id  corresponding to a folder the user folder (if any) is returned in preference to a  system folder and system folders are sorted <INBOX,SENT,other system labels>
    order of preference is achieved via the custom sort function label_priority(label)
    """
    message_labels = [labels[label_id] for label_id in label_ids]
    for label in sorted(message_labels,key=label_priority):
        if (label['name'] in names): 
            return label['name']
    return None 
def unique_values(dicts_list,key):
    values=set()
    for d in dicts_list:
        values.add(d.get(key))
    values.discard(None)
    return list(values)
def add_summary(summaries_dict,mailbox_labels_dict,mailbox_folders,request_id, response, exception):
    """
    callback invoked from get_batch_summaries to add each individual summary to the summaries dict
    take care re inconsistent capitalisation of keys in summary dict 
    if a key relates to a field from the payload headers then it starts with a capital letter to be consistent with the name value  in the dict for this header within the headers list  
    otherwise to be consistent with the keys in the response dict it starts with lower case and any subsequent word within the key starts with a capital   
    """
    if exception is not None:
        # Handle error
        print(exception)
    else:
        summary = {header['name']: header['value'] for header in response['payload'].get('headers',[])}
        summary['isUnread'] = 'Yes' if 'UNREAD' in response['labelIds'] else 'No'
        summary['id']=response['id']
        summary['labelIds']=response['labelIds']
        summary['folder']=folder_from_label_ids(response['labelIds'],mailbox_labels_dict,mailbox_folders)         
        if summary['folder'] is None:
            summary['folder']='Archived (no folder)'
        summary['snippet']=html.unescape(response['snippet'])
        summary['mimeType']=response['payload']['mimeType']
        summary['sizeEstimate']=response['sizeEstimate']
        if not 'Date' in summary:
            summary['Date'] = rfc2822_from_milliseconds(response['internalDate'])
            summary['dateSource']='internalDate'
        #can delete the raw_response key:value below once happy that summary contains everything useful 
        summary['rawResponse'] = response
        summaries_dict[summary['id']]=summary 
def is_cached(message_id, filename=None):
    """Checks if the email body or attachment is cached."""
    path = get_gmail_message_cache_path_with_fallback(message_id, filename)
    return os.path.exists(path), path
class UrlHandler:
    def handle(self, url):
        # Extract protocol from the URL
        protocol = self.extract_protocol(url)
        # Construct the handler method name and call it, defaulting to default_handler
        method_name = f"{protocol}_handler"
        return getattr(self, method_name, self.default_handler)(url)
    def extract_protocol(self, url):
        # Match the protocol at the beginning of the URL
        match = re.match(r"^([a-zA-Z][a-zA-Z0-9+-.]*):", url)
        return match.group(1) if match else None
    # Define default handler
    def default_handler(self, url):
        print(f"unsupported protocol for URL: {url}")
    def launch_browser(self,url):
            threading.Thread(target=webbrowser.open, args=(url,)).start()    
    # Define protocol-specific handlers
    def handle_http(self, url):
        self.launch_browser(url)
    def handle_https(self, url):
        self.launch_browser(url)
    def handle_ftp(self, url):
        self.launch_browser(url)
    def handle_tel(self, url):
        print(f"Implement tel url handler that uses windows phone app to  call: {url}")
    def handle_mailto(self, url):
        print(f"Implement email link handler that repurposes this dialog with new message to : {url}")
class SoupBowl:
    def __init__(self,html_content):
        self.handlers = {"a":self.link_handler,"article":self.block_handler,"aside":self.block_handler,"blockquote":self.block_handler, "br":self.block_handler,"div":self.block_handler, "footer":self.block_handler,"header":self.block_handler,"h1":self.heading_handler, "h2":self.heading_handler, "h3":self.heading_handler, "h4":self.heading_handler, "h5":self.heading_handler, "h6":self.heading_handler, "img":self.img_handler,"nav":self.block_handler, "p":self.block_handler, "pre":self.block_handler,"section":self.block_handler}
        self.current_pos = 0
        self.current_tag_index=0
        self.full_text=''
        self.tags=[]        
        self.soup = BeautifulSoup(html_content,'html.parser')
        self._render(self.soup.body if self.soup.body else self.soup)
    def _render(self,element):
        """ 
        Recursively traverse the element tree and build the full text and the interesting tags  attributes, skipping elements of no interest to winston, such as aria-hidden elements, script and style. 
        """
        if isinstance(element, NavigableString):
            if not isinstance(element,Comment): #ignore comments
                # Handle plain text by cleaning chars which tts cannot speak then appending it
                text=clean_unspeakables(element)
                self.full_text+=text
                self.current_pos += len(text)
            return
        elif element.has_attr('aria-hidden') and element['aria-hidden'].lower() == 'true':
            return   # Ignore this element and its children
        elif  element.name in ['script','style']: 
            return   # Ignore this element and its children 
        else:
            handler=self.handlers.get(element.name,self.default_handler)
            tag=handler(element)
            if tag: 
                self.current_tag_index+=1
                tag.index=self.current_tag_index 
                self.tags.append(tag)
        # If the element has children, recursively process them
        for child in element.children:
            self._render(child)
    def get_tagged_lines(self):
        tagged_lines=self._apply_tags(self.full_text,self.tags)
        return tagged_lines
    def _apply_tags(self,full_text,tags):
        start_and_text_tuples =split_and_wrap(full_text)
        tagged_lines = []
        for line_start,line in start_and_text_tuples: 
            tagged_lines.append(TaggedLine(line_start,line,tags))
        return tagged_lines
    def link_handler(self,soup_link):
        text = clean_unspeakables(soup_link.get_text())
        url = soup_link.get('href')
        t = Tag('link',self.current_pos, len(text), {'url':url},description_extension =text)
        children = [c for c in soup_link.children]
        if (len(children)==1) and (children[0].name=='img') and not children[0].get('alt'):
            t.unlabelled_graphic=True
        return t 
    def default_handler(self,soup_element):
        #print(f'default handler for {soup_element.name}')
        pass
    def heading_handler(self,soup_heading):    
        text = clean_unspeakables(soup_heading.get_text())
        description=f"heading level {soup_heading.name[-1]}"
        return Tag(soup_heading.name,self.current_pos, len(text), attributes={},description=description,description_extension=text)
    def block_handler(self,soup_element):
        if not self.full_text.endswith('\n'):  # Avoid duplicate line breaks
            self.full_text+='\n'
            self.current_pos += 1
    def img_handler(self,soup_img):
        description = soup_img.get('alt', 'unlabelled')+' graphic '
        # insert a placeholder into the text parts and update the current position so that multiple images on a single line do not occupy the same slot in the eventual TaggedElement()  
        placeholder = '\u200c' #'\u200c' #This is  non-breaking space not spoken in punc.ini. do not use ' graphic ' as this leads to saying of simply space when entering the tag
        self.full_text+=placeholder
        pos = self.current_pos
        self.current_pos += len(placeholder)
        t = Tag(soup_img.name,pos, len(placeholder), {},description)        
        t.unlabelled_graphic = False if soup_img.get('alt') else True 
        return t 
class TaggedLine:
    def __init__(self,line_start,line,tags):
        self.text=line
        self.tags=[]
        for tag in tags:
            # Check if the tag overlaps with the line 
            overlap_start_length=overlap(tag,line_start,len(line))
            if overlap_start_length:
                line_tag=Tag(tag.name,*overlap_start_length,tag.attributes,tag.description,tag.index,tag.description_extension)
                line_tag.unlabelled_graphic = tag.unlabelled_graphic
                self.tags.append(line_tag)
class zzz_HTMLRenderer:
    def __init__(self):
        self.content_elements = []
        self.current_index = 0
    def get_current_element(self):
        if 0 <= self.current_index < len(self.content_elements):
            return self.content_elements[self.current_index]
        return None
    def navigate(self, direction):
        if direction == 'next' and self.current_index < len(self.content_elements) - 1:
            self.current_index += 1
            return self.get_current_element()            
        elif direction == 'previous' and self.current_index > 0:
            self.current_index -= 1
            return self.get_current_element()
        return None
    def jump_to_next(self, element_type):
        for i in range(self.current_index + 1, len(self.content_elements)):
            if self.content_elements[i].type == element_type:
                self.current_index = i
                return self.get_current_element()
        return None
    def parse_soup(self, soup):
        assert isinstance(soup,BeautifulSoup),f'parse_soup invalid argument type: {type(soup)}' 
        #cope with badly formed html emails which do not have an encapsulating <body> element 
        body_soup = soup if soup.body is None else soup.body 
        self.content_elements = self._parse_element(body_soup, 0)
        return self.content_elements 
    def _parse_element(self, element, level):
        assert element is not None,f'_parse_element invalid element type: NoneType' 
        elements = []
        current_text = ""
        inline_elements = []
        for child in element.children:
            assert len(elements)<1000, "length of html elements list exceeded 1000"
            assert len(inline_elements)<1000, "length of inline_elements list exceeded 1000"            
            if isinstance(child, NavigableString):
                current_text += child.text #was strip() rather than text 
            elif child.name in ['a', 'strong', 'em', 'span']:
                inline_elements.append((len(current_text), child))
                current_text += child.get_text(strip=True)
            elif child.name in ['dummy out']: # 'img', 'area', 'input']:
                if current_text or inline_elements:
                    elements.append(ContentElement('text', current_text, level, {}, inline_elements))
                    current_text = ""
                    inline_elements = []
                alt_text = child.get('alt', f'{child.name.capitalize()} with no description')
                elements.append(ContentElement(child.name, alt_text, level, {'src': child.get('src')}, []))
            elif child.name in ['object', 'applet']:
                if current_text or inline_elements:
                    elements.append(ContentElement('text', current_text, level, {}, inline_elements))
                    current_text = ""
                    inline_elements = []
                alt_text = child.get('alt') or child.get_text(strip=True) or f'{child.name.capitalize()} content'
                elements.append(ContentElement(child.name, alt_text, level, {}, []))
            elif child.name == 'figure':
                if current_text or inline_elements:
                    elements.append(ContentElement('text', current_text, level, {}, inline_elements))
                    current_text = ""
                    inline_elements = []
                figcaption = child.find('figcaption')
                caption_text = figcaption.get_text(strip=True) if figcaption else 'Figure with no caption'
                elements.append(ContentElement('figure', caption_text, level, {}, []))
            elif child.name in ['audio', 'video']:
                if current_text or inline_elements:
                    elements.append(ContentElement('text', current_text, level, {}, inline_elements))
                    current_text = ""
                    inline_elements = []
                desc_text = f'{child.name.capitalize()} content: {child.get("title", "No title provided")}'
                elements.append(ContentElement(child.name, desc_text, level, {}, []))
            else:
                if current_text or inline_elements:
                    elements.append(ContentElement('text', current_text, level, {}, inline_elements))
                    current_text = ""
                    inline_elements = []
                if child.name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
                    elements.append(ContentElement('heading', child.get_text(strip=True), int(child.name[1]), {'level': child.name}, []))
                elif child.name in ['ul', 'ol']:
                    elements.append(ContentElement('list_start', child.name, level, {}, []))
                    elements.extend(self._parse_element(child, level + 1))
                    elements.append(ContentElement('list_end', child.name, level, {}, []))
                elif child.name == 'li':
                    elements.extend(self._parse_element(child, level))
                elif child.name in ['div', 'p']:
                    elements.extend(self._parse_element(child, level))
        if current_text or inline_elements:
            elements.append(ContentElement('text', current_text, level, {}, inline_elements))
        return elements
    def generate_audio_description(self, element):
        if element.type == 'heading':
            return f"Heading level {element.attributes['level']}: {element.content}"
        elif element.type in ['img', 'area', 'input']:
            return f"{element.type.capitalize()}: {element.content}"
        elif element.type in ['object', 'applet']:
            return f"{element.type.capitalize()} content: {element.content}"
        elif element.type == 'figure':
            return f"Figure: {element.content}"
        elif element.type in ['audio', 'video']:
            return element.content
        elif element.type == 'list_start':
            return f"Start of {element.content} list"
        elif element.type == 'list_end':
            return f"End of {element.content} list"
        elif element.type == 'text':
            description = ""
            last_pos = 0
            for pos, inline_element in element.inline_elements:
                description += element.content[last_pos:pos]
                if inline_element.name == 'a':
                    description += f"link: {inline_element.get_text(strip=True)}"
                elif inline_element.name in ['strong', 'em']:
                    description += f"emphasize: {inline_element.get_text(strip=True)}"
                else:
                    description+=inline_element.get_text(strip=True)
                last_pos = pos + len(inline_element.get_text(strip=True))
            description += element.content[last_pos:]
            return description.strip()
        else:
            return element.content
class SearchParameters:
    def __init__(self,labels,search_string):
        self.labels=labels
        self.search_string=search_string
class MailElementInterface(ValuesElementInterface):
    pass
class MailElement(MailElementInterface,ValuesElement):
    def __init__(self,*args,summary=None,**kwargs):
        self.summary=summary
        super().__init__(*args,**kwargs)
    def __str__(self):
        speakables =[] if self.summary['isUnread']=='No' else ['Unread'] 
        if 'has_attachment' in self.summary:
            speakables.append('has attachment')
        addresses=parse_addresses(self.values[0])
        string_addresses= [a[0] if len(a[0]) else a[1] for a in addresses]
        first_addresses=string_addresses[:3]
        if len(addresses)>3:
            first_addresses.append(' and more addresses')
        speakables.append(', '.join(first_addresses))
        speakables.extend(self.values[1:])
        return ', '.join(speakables)    
    @online_only    
    @KeyHandler.hotkey("Control+U")
    def command_mark_unread(self,modified_name):
        self.mark_unread('Shift' not in modified_name)
        state  = 'Unread' if 'Shift' not in modified_name else 'Read'        
        self.dialog.say_strings(f'marked as {state}')        
    @online_only
    def mark_unread(self,unread=True):
        if unread:
            body = {'addLabelIds': ['UNREAD']}        
        else:
            body = {'removeLabelIds': ['UNREAD']}
        request=self.dialog.service.users().messages().modify(userId='me', id=self.summary['id'], body=body)
        execute_with_retry(request)
        self.summary['isUnread'] = 'Yes' if unread else 'No'
    @online_only
    @KeyHandler.hotkey("Control+V")
    def command_move_message(self,modified_name):
        if not 'Shift' in modified_name:
        #unshifted control+v is paste but copy/paste is not currently supported for mail items 
            beep() 
            return 
        #move is Shift+Control+V same as Outlook
        folder = self.dialog.select_folder(title='Move to folder')
        if folder:
            self.move_to_folder(folder)
        else:
            self.dialog.say_strings('No folder selected')
    @online_only
    @KeyHandler.hotkey("Back")
    def command_archive(self,modified_name):
        self.move_to_folder('Saved')
    def move_to_folder(self,new_folder):
        if self.dialog.folder==new_folder:
            self.dialog.say_strings(f'already working in {new_folder} folder')
        elif new_folder=='TRASH':
            #treat move to TRASH as a delete request 
            self.dialog.command_delete("Delete")
        else:
            message_id= self.summary['id']
            if self.dialog.folder=='TRASH':
                self.dialog.untrash(message_id)
                #do not remove TRASH label as will have already been done by the untrash, but do need to remove any user defined folder which will have been left in the message by trash command 
                labels_dict= {label['id']:label for label in self.dialog.labels}
                names=self.dialog.folder_names()
                old_folder=folder_from_label_ids(self.summary.get('labelIds',[]),labels_dict,names) 
            else:
                old_folder=self.dialog.folder
            if self.dialog.change_labels(message_id,new_folder=new_folder,old_folder=old_folder):
                self.dialog.remove(self)
                self.dialog.say_current()
    @KeyHandler.hotkey("Control+A")
    def command_select_all(self,modified_name,silent=False):
        beep()
    @KeyHandler.hotkey("Control+C")
    def command_copy(self,modified_name):
        beep()
    @KeyHandler.hotkey("Return")
    def command_view(self,modified_name):
        self.view_message(prefer_text_format=('Shift' in modified_name))
    @online_only
    @KeyHandler.hotkey("F5")
    def command_refresh_message(self,modified_name):
        self.view_message(refresh_cache=True,prefer_text_format=('Shift' in modified_name))
    def view_message(self,prefer_text_format=None,refresh_cache=False):
        summary=self.summary
        addresses={group:summary.get(group,'') for group in ['From','To','Cc','Bcc']}
        self.dialog.message_form('view',addresses,summary.get('Subject','No Subject'),summary,prefer_text_format=prefer_text_format,refresh_cache=refresh_cache) 
        if self.summary['isUnread']=='Yes':
            self.mark_unread(unread=False)
        #abbreviated say_current seems neater as avoids the whole snippet, etc
        #but probably better to say something sensible in message_form before returning which is current approach 
        #self.dialog.say_strings(self.values[:2])        
class TaggedElementInterface(ElementInterface):
    #should add abstract methods for tags and make tags on the concrete element use property getter/setter idiom 
    pass
def tagged_subclass(base_class):
    class TaggedClass(TaggedElementInterface,base_class):
        def __init__(self,*args,tags=[],**kwargs):
            self.tags=tags
            super().__init__(*args,**kwargs)
        def __value_representation__(self):
            representation = self.value
            #work backwards through tags on this element to avoid corrupting representation offsets during the loop
            #use tag.index rather than tag.start so as to give correct order of tags when multiple tags are at the same offset 
            tags_list = sorted(self.tags, key=lambda tag: tag.index, reverse=True)
            for tag in [t for t in tags_list if self.dialog.dialog.is_voiceable(t)]:
                representation = representation[:tag.start] + f", {tag.description}: " + representation[tag.start:]
            return representation 
        def get_active_tag_indices(self):
            active_list=[tag.index for tag in self.tags if overlap(tag,self.caret,1)]
            return set(active_list) 
        def new_tag_descriptions(self):
            tags = [tag for tag in self.tags if (tag.index in (self.dialog.active_tags-self.dialog.previous_tags))]        
            return [f"{tag.description}: {tag.description_extension} " if tag.description_extension else tag.description for tag in tags] 
        @KeyHandler.hotkey("Control+Return")
        def command_goto_link(self,modified_name):
            link_tag=self.nearest_tag(self.caret)
            if not link_tag: 
                beep() #no links associated with this line
            else:
                url = link_tag.attributes['url']
                self.dialog.url_handler.handle(url)
        def nearest_tag(self, offset):
            link_tags = [t for t in self.tags if t.name=='link']
            nearest = None
            min_distance = len(self.value)
            for tag in link_tags:
                tag_start = tag.start
                tag_end = tag.start + tag.length
                # Calculate the distance to the nearest point in the tag range
                if tag_start <= offset <= tag_end:
                # Offset is within the tag, distance is zero
                    distance = 0
                else:
                    # Offset is outside the tag, calculate nearest edge distance
                    distance = min(abs(tag_start - offset), abs(tag_end - offset))
                if distance < min_distance:
                    min_distance = distance
                    nearest = tag
            return nearest
        def say_mro(self):
            print(str(self.__class__.__mro__))
    TaggedClass.__name__=f"TaggedBase{base_class.__name__}"
    return TaggedClass
#for readability and inspection it is important that the names of dynamically constructed tagged element classes  are of the form f"TaggedBase{base_class.__name__}"
TaggedBaseElement = tagged_subclass(Element)
TaggedBaseTextElement=tagged_subclass(TextElement)
class TaggedElement(TaggedBaseElement):
    """
    This is where any methods of TaggedElement that are not methods  of TaggedBase or of Element get added  
    Slightly convoluted indirect structure is because it is clearer than monkey patching the methods into TaggedBaseElement via a dict of functions argument to the tagged_subclass function 
    """
    @KeyHandler.hotkey("Control+Return")
    def command_goto_link(self,modified_name):
        super().command_goto_link(modified_name)
class TaggedTextElement(TaggedBaseTextElement):
    """
    This is where any methods of TaggedTextElement that are not methods  of TaggedBase or of TextElement get added  
    Slightly convoluted indirect structure is because it is clearer than monkey patching the methods into TaggedBaseElement via a dict of functions argument to the tagged_subclass function 
    """
    def on_word_completion(self):
        completed_word = super().on_word_completion()
        self.add_url_tags()
        return completed_word
    def add_url_tags(self):
        # Define the URL regex pattern
        url_pattern = r"(https?://)?(www\.)?([a-zA-Z0-9-]+\.)+[a-zA-Z]{2,6}(/[\w.-]*)*"
        regex = re.compile(url_pattern)
        for match in regex.finditer(self.value):
            url = match.group(0)
            start = match.start()
            length = match.end() - start
            # Check if a tag with the same start and name='link' already exists and patch the length and description if this is a longer url at the same place as an already valid url 
            for tag in self.tags:
                if (tag.start==start) and (tag.name=='link'):
                    tag.attributes['url']=url
                    tag.description=url 
                    tag.length=length 
                    tag_exists=True
                    break
                elif tag.start>start:
                    self.tags.insert(tags.index(tag),Tag('link',start,length,attributes={'url':url}            ))
            if not tag_exists:
                # Create a new Tag for the URL
                new_tag = Tag(name='link', start=start, length=length, attributes={'url': url},description=url)
                # Find the correct insertion position
                insert_index = next(
                    (i for i, tag in enumerate(self.tags) if tag.start > start),
                    len(self.tags)
                )
                #note that this results in non-unique tag indices so sorting may not be quite right in some cases. Could do something with floats, splitting the difference between two indices to get a unique middle index 
                new_tag.index = insert_index
                # Insert the new tag at the found index
                self.tags.insert(insert_index, new_tag)
class Tag:
    def __init__(self, name, start, length,attributes={},description=None,index=None,description_extension=None):
        self.name = name
        self.description = description if (description is not None) else name
        self.description_extension = description_extension 
        self.start = start
        self.length=length
        self.attributes=attributes
        self.index=index 
        self.unlabelled_graphic=False
    def __eq__(self, other):
        return (
            self.name == other.name and
            self.start == other.start and
            self.length == other.length and
            self.attributes == other.attributes
        )
class BodyDialog(EditDialog):
    def __init__(self,*args,**kwargs):
        super().__init__(*args,**kwargs)
        self.active_tags=set()
        self.previous_tags=set()
        self.url_handler=UrlHandler()
    def content_interface(self):
        #generate_content yields either TextElement objects  for editable text lines, Element objects when only viewing plain text body or (non_editable) TaggedElement objects either when viewing html body or when creating reply/forward to html message for which original content editing is not yet supported  
        return (TaggedElementInterface,TextElementInterface, ElementInterface)
    def generate_content(self,filler,element_type=None):
        """
        filler is [action,prefer_text_format,summary]
        """
        action,prefer_text_format,summary=filler
        if action =='new':
            self.dialog.format='html'
            yield TaggedTextElement(self,'blank line','')
        else:
            if (prefer_text_format or (self.dialog.content['text/html'] is None))  and  (self.dialog.content['text/plain'] is not None):
                self.dialog.format='text'
                EditableElementClass = TextElement                
            else:  
                self.dialog.format='html'
                EditableElementClass = TaggedTextElement
            if action!='view':
                for l in header_lines(summary):
                    yield EditableElementClass(self,'blank line',l)
            yield from self.lines_from_body(read_only=(action=='view'))
    def lines_from_body(self,read_only=False):
        if self.dialog.format=='text':
            ElementClass = Element if read_only else TextElement
            text=self.dialog.content['text/plain'] #.replace('\r\n\r\n','\n').replace('\r\n','\n')
            for line in [l for l in text.splitlines() if len(clean_unspeakables(l).strip())]:
                yield ElementClass(self,'blank line',line) 
        elif self.dialog.format=='html':
            #currently always use read only TaggedElement rather than TaggedTextElement for received content as do not support editing complex html structures safely
            tagged_lines=self.dialog.bowl_of_soup.get_tagged_lines()
            for tagged_line in [l for l in tagged_lines if len(clean_unspeakables(l.text).strip())]:
                yield TaggedElement(self,'blank line',tagged_line.text,tags=tagged_line.tags)  
    def get_result(self):
        if self.dialog.format=='text':
            return '\n'.join([e.value for e in self])
        else:
            #only take the lines that are editable and let html_reply inject them into the body in the right way
            reply_lines=[e.value  for e in self if isinstance(e,TextElement)] 
            return html_reply(self.dialog.bowl_of_soup.soup,reply_lines)        
    def char_sink(self,char):
        if hasattr(self.current(),'char_sink'):
            return  self.current().char_sink(char)        
        else:
            """
            If char is not handled by the element because the latter is read only then simply beep rather than doing the usual Dialog.char_sink()
            """
            beep()
    def ok(self,modified_name):
        """
        re-allocates this method (and hence the Return key) if this is a message view form back to it's original purpose as OK return from the dialog by overriding the assignment in EditDialog to the normal editing function of the enter key (i.e. inserting a line break).
        """
        if self.dialog.action=='view':
            return self.dialog.ok(modified_name)
            beep()
        else:
            return super().ok(modified_name)
    def move(self,*args,**kwargs):
        self.previous_tags=self.active_tags
        rc =  super().move(*args,**kwargs)
        return rc
    def set_active_tags(self):
        self.active_tags = getattr(self.current(),'get_active_tag_indices',lambda:set())()
    def say_after_move(self,s,interrupt=True):
        self.set_active_tags()
        new_descriptions = getattr(self.current(),'new_tag_descriptions',lambda:[])()
        new_s = ', '.join(new_descriptions +[s])
        #print('speaking',new_s)
        return super().say_after_move(new_s,interrupt=interrupt)
    @KeyHandler.hotkey("Control+V")
    def command_paste_or_move_message(self,modified_name):
        if 'Shift' in modified_name:
            self.dialog.command_paste_or_move_message(modified_name)
        else:
            super().command_paste(modified_name)
class MailDialog(GoogleDialog):
    def __init__(self,*args,**kwargs):
        self.profile={}
        super().__init__(*args,**kwargs)
        self.sort_column = self.sort_column_from_labels(self.folder)
        self.deleted_stack=[]
    def get_api_name_and_version(self):
        return ('gmail','v1')
    def content_interface(self): 
        return MailElementInterface  
    def sort_by_column(self,column):
        super().sort_by_column(column)
        for label in self.labels:
            if label['name']==self.folder:
                label['sortColumn']=column
                break      
    def sort_key(self,element):
        column = element.dialog.sort_column 
        value = element.values[column]
        if column==0:
            #strip any delimiter off front of email name
            value=value.lower().lstrip("""'"<([{""")
        elif column==1:
            #strip prefix off subject line
            value =value.lower().replace('re:','').replace('fw:','')
        elif column==2:
            #for date the default order is most recent at the top of the dialog which is achieved by using the time delta between this datetime and the max value of datetime 
            value = datetime.now().astimezone(pytz.timezone('Europe/London') )+timedelta(days=10000)-sortable_date(element.summary.get('Date'))
            #value = datetime.max -sortable_date(element.summary.get('Date'))                
        else:
            value = value.lower()
        return value
    def labels_and_folder(self):
        self._labels=[]
        self._summaries={}
        label_path = get_gmail_labels_cache_path_with_fallback()
        if os.path.exists(label_path):
            with open(label_path,'r',encoding='utf-8') as f:
                self.labels=json.load(f)
        self.folder='INBOX'
        self.sort_column=2
    def preliminaries(self):
        self.labels_and_folder()
    def generate_content(self,filler, element_type=ValuesElement):
        #the profile contains the email address of the mailbox owner which should be used in from field display in MessageDialog 
        profile= self.get_profile()
        self.profile= profile if profile else {} 
        summaries={}
        if isinstance(filler,str):
            self.folder = filler
        #never get summaries for a search from the cache
        if not isinstance(filler, SearchParameters):
            summaries_path = get_gmail_folder_cache_path_with_fallback(self.folder)
            message_ids = []
            if os.path.exists(summaries_path):
                with open(summaries_path, encoding='utf-8') as f:
                    summaries=json.load(f)
                    #convert legacy cache entry from list to dict 
                    if isinstance(summaries,list):
                        summaries={s['id']:s for s in summaries}
        self.summaries=summaries
        if is_connected(): #need to handle case when connection fails during this clause so should be able to remove this test completely 
            labels=[self.folder]
            search_string=None
            if isinstance(filler,SearchParameters):
                labels=filler.labels
                search_string=filler.search_string 
            message_ids = self.get_message_ids(labels=labels,search_string=search_string) 
            if message_ids is None:
                self.summaries=summaries 
            else:
                #print(f'got {len(message_ids)} message ids')
                self.summaries=filter_dict_by_keys(summaries,message_ids)                
                self.get_batch_summaries(self.summaries,message_ids)
                with_attachment_ids = self.get_message_ids(labels=labels,search_string=search_string,has_attachment=True)             
                if with_attachment_ids:
                    for message_id in  with_attachment_ids: 
                        if message_id in self.summaries:        
                            self.summaries[message_id]['has_attachment']='Yes' 
                #save the new summaries dict as a json file in the resource cache unless this is a search 
                if not isinstance(filler,SearchParameters):
                    with open(summaries_path,'w',encoding='utf-8') as f:
                        json.dump(self.summaries,f)
        #now take the cached (possibly updated) summaries to build the elements in the dialog 
        for summary in self.summaries.values():
            values=[summary.get(self.get_headers()[0].strip('<>'),''),summary.get('Subject',''),reformat(summary.get('Date')),summary.get('snippet','')]
            #for summaries returned from a search of all folders as specified by labels==[], pre-pend the subject field with the name of the containing folder 
            # note that if the item was archived then it will have no folder lable but the folder item in the summary will be set in add_summary to 'archived (no folder)'
            if isinstance(filler,SearchParameters) and (filler.labels==[]):
                values[1] = ': '.join([summary['folder'],values[1]])
            elem= MailElement(self,'mail item',summary.get('Subject',''), values=values,summary=summary)
            yield elem 
        self.sort_column=self.sort_column_from_labels(self.folder)
        self.sort(key=self.sort_key)
    def sort_column_from_labels(self,name):
        labels = [l for l in self.labels if l['name']==name]
        if not len(labels):
            label={}
        else:
            label = labels[0]
                #if self.labels is empty or no sortColumn stored then use Sent column
        sc =  label.get('sortColumn',2)
        #print(f'return from  sort_column_from_labels:  {sc}')
        return sc
    def asynchronous_get_summaries(self):
        get_summaries_thread=threading.Thread(target=self.summaries_from_api)
        get_summaries_thread.start()
    def get_attachment_via_cache(self, message_id, attachment_id,filename):
        """Attempts to retrieve from cache, fetches from service if not found."""
        cached,cache_path = is_cached(message_id, filename)  
        if not cached:
            content = self.get_attachment(message_id, attachment_id)
            if content is None:
                return None  
            ensure_dir_exists(os.path.dirname(cache_path))
            with open(cache_path, 'wb') as file:
                file.write(content)
        return cache_path 
    def get_message_via_cache(self, message_id,refresh_cache=False):
        """
        Attempts to retrieve message from cache unless refresh_cache, fetches from service if not found or if refresh_cache.
        """
        if not message_id:
            return None
        service = self.service
        message=None
        cached,cache_path = is_cached(message_id, None)
        if cached and not refresh_cache:
            with open(cache_path,'r') as file:
                message= json.load(file)
        if not message:
            # Fetch email message and attachments
            message = self.get_message(message_id)
            if message:
                ensure_dir_exists(os.path.dirname(cache_path))
                with open(cache_path, 'w', encoding='utf-8') as file:
                    json.dump(message, file)
        return message
    @online_only
    def get_message(self, message_id,data_format=None):
        request = self.service.users().messages().get(userId='me', id=message_id, format='full') 
        message=execute_with_retry(request)
        if self.connection_error(message):
            return None 
        else:
            return message
    def get_attachment(self, message_id, attachment_id):
        """
        Fetches a specific attachment by its ID and returns the binary data.
        """
        request = self.service.users().messages().attachments().get(userId='me',messageId=message_id,id=attachment_id)  
        attachment= execute_with_retry(request)
        if self.connection_error(attachment):
            data=None
        else:
            data = base64.urlsafe_b64decode(attachment['data'].encode('UTF-8'))
        return data
    def get_batch_summaries(self,summaries_dict,message_ids):
        labels_dict= {label['id']:label for label in self.labels}
        callback = functools.partial(add_summary,summaries_dict,labels_dict,self.folder_names())
        batch_request = self.service.new_batch_http_request(callback=callback)
        batch_result=None
        count=0
        #re-acquire summary even if already present in summaries as the Unread status may have changed 
        for message_id in message_ids:
            summary_request = self.service.users().messages().get(userId='me', id=message_id, format='metadata',metadataHeaders=['From', 'To', 'Cc', 'Bcc', 'Subject', 'Date','In-Reply-To','References','Reply-To'])  
            batch_request.add(summary_request) 
            count+=1
            #progress beep after finding 50 new message ids to summarize 
            if count%50==0:
                beep(250,350)
            #limit batches to 20 requests to avoid 429 concurrent rate limit error 
            if count%20==0:
                #print('executing batch of 20 get message requests')
                batch_result  = execute_with_retry(batch_request)
                if self.connection_error(batch_result):
                    print(f'abnormal return after executed  total of {count} get message requests')
                    return 
                batch_request = self.service.new_batch_http_request(callback=callback)        
        if count%20!=0:
            batch_result  = execute_with_retry(batch_request)
        #print(f'executed  total of {count} get message requests')
        if self.connection_error(batch_result):
            pass #error will have been logged in connection_error method so nothing else to do
    def get_labels(self):
        request = self.service.users().labels().list(userId="me")     
        response = execute_with_retry(request)
        if not self.connection_error(response):
            labels = response.get("labels", [])
            for label in labels:
                #initialize the sort column to Sent 
                label['sortColumn'] = 2 
        return labels  
    @online_only 
    def get_profile(self):
        profile_request  = self.service.users().getProfile(userId='me') 
        profile=execute_with_retry(profile_request)
        if not self.connection_error(profile):
            return  profile 
    @online_only
    def get_message_ids(self, labels=['INBOX'],max_results=500,min_date=None,has_attachment=None, unread=None,search_string=None): 
        #min_date has format '2023/01/01'
        parameters = []
        if isinstance(labels,str):
            labels=[labels]
        for label in labels:
            parameters.append(f'label:{label}') 
        if min_date:
            parameters.append(f'after:{min_date}')
        if has_attachment:
            parameters.append('has:attachment')
        if search_string:
            parameters.append(f'(subject:"{search_string}" OR ({search_string}) OR from:"{search_string}" OR to:"{search_string}" OR cc:"{search_string}" OR bcc:"{search_string}")')  
            #print(parameters)
        query = ' '.join(parameters)
        #print(f'message ids query = {query}')
        request = self.service.users().messages().list(userId='me', maxResults=max_results,q=query) 
        response = execute_with_retry(request)
        if not self.connection_error(response):
            return [m['id'] for m in response.get('messages', [])]
    @property
    def summaries(self):
        with self.app.google_api_lock:
            return self._summaries
    @summaries.setter
    def summaries(self, value):
        with self.app.google_api_lock:
            self._summaries = value
        return
    def asynchronous_get_labels(self):
        get_labels_thread=threading.Thread(target=self.labels_from_api)
        get_labels_thread.start()
    def labels_from_api(self):
        """
        the setter method for self.labels  ensures that the assignment to self.labels by a separate thread running this method is safe
        """
        labels_path = get_gmail_labels_cache_path_with_fallback()
        labels = self.get_labels(self.service)
        if not labels:
            return
        else:
            with open(labels_path,'w',encoding='utf-8') as f:
                json.dump(labels,f)
            self.labels=labels
    @property
    def labels(self):
        with self.app.google_api_lock:
            return self._labels
    @labels.setter
    def labels(self, value):
        with self.app.google_api_lock:
            self._labels = value
        return
    def get_headers(self):
        if hasattr(self,'folder') and self.folder=='SENT':
            column_0 = 'To'
        else:
            column_0='From'
        return [column_0,'Subject','Sent','Preview']
    @KeyHandler.hotkey("Control+N")
    def command_new(self,modified_name):
        self.message_form('new',addresses={'From':self.profile.get('emailAddress','me')}) 
    @KeyHandler.hotkey("Control+F")
    def command_find_or_forward(self,modified_name):
        if 'Shift' in modified_name:
            self.reply_or_forward('forward')
        else:
            super().command_find(modified_name)
    @KeyHandler.hotkey("Control+R")
    def command_reply(self,modified_name):
        reply_type='reply' if not 'Shift' in modified_name else 'reply all'
        self.reply_or_forward(reply_type)
    def reply_or_forward(self,reply_type):
        subject = prefixed_subject(reply_type,self.current().summary .get('Subject',''))
        owner=self.profile.get('emailAddress','me') 
        addresses=reply_addresses(reply_type,owner,self.current().summary )
        self.message_form(action=reply_type,addresses=addresses, subject=subject,summary=self.current().summary )
    def message_form(self, action,addresses={},subject='',summary={},prefer_text_format=None,refresh_cache=False):
        name = 'dynamic name from subject' 
        message_id=summary.get('id')
        message =self.get_message_via_cache(message_id,refresh_cache=refresh_cache)
        content=content_from_message(message)        
        fields=message_filler(action,addresses=addresses,subject=subject,summary=summary,prefer_text_format=prefer_text_format,content=content,include_attachments=not ('reply' in action))
        form= MessageDialog(self,name,filler=fields,summary=summary,action=action,content=content,refresh_cache=refresh_cache,empty_value='edit' if action!='view' else 'read only') 
        form.set_from_name('To' if action in ['new','forward'] else 'Body',silent=True)
        results= form.run(speak=('%name','%current')) 
        if action=='view':
            self.say_strings(self.folder)
        else:
            if self.message(title='Save draft?',buttons=['Yes','No'])=='Yes':  
                self.say_strings('implement save to drafts if dirty by trapping Escape then probably first calling make_api_message')        
            else:
                self.say_strings(self.folder)
    @online_only
    @KeyHandler.hotkey("Back")
    def backspace(self,modified_name):
        self.delete(modified_name) #delete logic handles backspace as well as delete 
    @KeyHandler.hotkey("Delete")
    def command_delete(self,modified_name):
        if not 'Shift' in modified_name:
            if (self.folder=='TRASH') or not len(self):
                beep()
                return
            message_id=self.current().summary['id']
            if self.trash(message_id):
                self.deleted_stack.append(message_id)
                self.remove(self.current())
                self.say_current()
        else:
            if not len(self.deleted_stack):
                beep()
                return
            message_id=self.deleted_stack.pop()
            if self.untrash(message_id):
                if self.change_labels(message_id,old_folder='TRASH',new_folder=self.folder):
                    self.command_refresh_folder('Control+F5')
                    for i  in range(len(self)):
                        if self[i].summary['id']==message_id:
                            self.move(i,0,False)
                            break 
    def trash(self,message_id):
        request=self.service.users().messages().trash(userId='me', id=message_id) 
        response = execute_with_retry(request) 
        if not self.connection_error(response):
            return True    
    def untrash(self,message_id):
        request=self.service.users().messages().untrash(userId='me', id=message_id) 
        response = execute_with_retry(request) 
        if not self.connection_error(response):
            return True 
    def change_labels(self,message_id,new_folder=None,old_folder=None):
        body = {'addLabelIds': [label_id(self.labels,new_folder)]}
        if old_folder:
            body['removeLabelIds'] = [label_id(self.labels,old_folder)]        
        request=self.service.users().messages().modify(userId='me', id=message_id, body=body)
        response=execute_with_retry(request)
        if not self.connection_error(response):             
            return True 
    @KeyHandler.hotkey("Control+Y")
    def command_goto_folder(self,modified_name):
        folder = self.select_folder(title='Go to folder')
        if folder is None:   
            self.say_strings('cannot get folder list - may be offline with no labels.json in cache')            
        elif folder is False: #folder is False after user cancelled goto 
            self.say_strings('No folder selected')
        else:
            #update deleted stack here so that when change folder the deleted stack is reset to empty 
            self.deleted_stack=[]
            self.refill(filler=folder)        
    def select_folder(self,title):
        if not self.labels:
            return
        folder = self.message(title=title,buttons=self.folder_names(),select=self.folder)
        return folder
    def folder_names(self):
        folders = [l for l in self.labels if l.get('labelListVisibility','')!='labelHide']
        names=[f['name'] for f in folders if (f['name']!='UNREAD') and not f['name'].startswith('[')]
        names.append('TRASH')
        names.sort()    
        return names
    @online_only
    @KeyHandler.hotkey("Control+E")
    def command_search(self,modified_name):
        fields = [
            [SingleTextElement,'Containing',""],
            [BooleanElement,'All Folders','No'],
            [ButtonElement,'OK'],
            ]
        results = self.input_values(title=' search for items containing:',fields=fields,empty_value='empty string')
        if (not results) or (results['Containing']=="") :
            self.say_strings('cancelled search')
            return
        labels=[self.folder] if   (results['All Folders']=='No') else []
        filler= SearchParameters(labels=labels,search_string=results['Containing'])     
        self.refill(filler)      
    @KeyHandler.hotkey("Escape")
    def command_dismiss_search(self,modified_name):
        self.refill(filler=self.folder)    
    @online_only
    @KeyHandler.hotkey("Control+F5")
    def command_refresh_folder(self,modified_name):
        self.refill(filler=self.folder)    
        self.say_strings('refreshed items list')
class MessageDialog(FormDialog):
    def __init__(self,*args,summary={}, content={},action='new',refresh_cache=False,**kwargs):
        self.summary=summary 
        self.dirty=False
        self.content=content 
        self.action=action
        self.refresh_cache=refresh_cache
        self.format='html'
        #self.content=        {"text/plain": None, "text/html": None, "inline_attachments": [], "attachment_ids": []}
        self.bowl_of_soup=SoupBowl('')  
        self.say_tags=True
        self.say_unlabelled_graphics=False
        super().__init__(*args,**kwargs)
    @property
    def name(self):
        subject = self.fields().get('Subject','No Subject field')
        subject = (subject if len(subject) else 'untitled message') 
        return f"{subject}, {self.format}"
    def     __str__(self):
        return ': '.join([super().__str__(), self.format])
    def generate_content(self,filler,element_type=None):
        if self.content['text/html']:
            #the full list of tags  and the full_text string are accessible as attributes  of the  SoupBowl() object available for diagnostic purposes but maybe delete in due course. tag.start in this list is relative to the start of the full text so not friendly once text is split into lines so should add a start_tuple attribute containing (line, offset)
            #replace carriage return+new line with just newline and then replace carriage return with newline to cater firstly for windows systems and then for older Mac systems which just use carriage return.
            #could alternatively do '\n'.join(x.splitlines()) as splitlines copes with all combos of carriage return and line feed)
            html_content=self.content['text/html'].replace('\r\n','\n').replace('\r','\n')
            self.bowl_of_soup  = SoupBowl(html_content) 
        yield from super().generate_content(filler,element_type)            
    def get_result(self):
        """
        super().get_result returns a dictionary of the result of each of the controls in the dialog
        if an element such as the 'Body' field has a get_result method, that is used to provide the value for that key , otherwise the value is the element's value attribute 
        This overriding method also stuffs in the '__format__' item derived from the MessageDialog().format attribute 
        """
        results = super().get_result()
        results['__format__'] = 'html' if self.format=='html' else 'plain'     
        if self.action not in ['new','view']:
            references = self.summary.get('References', self.summary.get('id',None))
            reply_headers={'In-Reply-To':self.summary.get('id',None),'References':references}        
            results['replyHeaders']=reply_headers                    
        return results 
    def on_close(self,result):
        if not self.dirty:
            return result 
        else:
            user_result = self.message(title='Save message to drafts?',buttons=['Yes','No','Cancel'])
            if user_result in ['Yes','No']:
                if  user_result == 'Yes':
                    self.command_save(modified_name='')
                return result 
            else:
                self.say_current()
                return None 
    @online_only
    @KeyHandler.hotkey("Alt+S")
    def command_send(self,modified_name):
        if self.send_message(self.fields()):
            return True
    @online_only
    def send_message(self,populator):
        """
        accepts fields for an email message in a populator dict containing all the fields of this message_form() dialog which may be a reply/forward or a new message.
        The populator also includes an '__format__' item   and a 'reply_headers' dict if relevant. (
        The populator is currently constructed in MessageDialog.get_result which overrides the default FormDialog get_result method. It is actually built through a call to the fields() method, which  is a synonym for get_result defined in the Dialog base class     
        The  'body' key of the populator is a plain text or html string as indicated by the '__format__' key. This string is generated in the get_result method of the 'Body' field  through the standard get_result() logic.
        The populator is passed unmodified to make_api_message  
        The returned message from make_api_message is then sent to the gmail api 
        """
        owner=self.home_dialog.profile.get('emailAddress','chrisjmairs@gmail.com')         
        message = make_api_message(self,populator,owner)
        #the body argument of the service.users().messages().send() method is the entire message not to be confused with the html body element its payload  contains (if it is an html message)  
        if self.parent.service:
            request = self.parent.service.users().messages().send(userId='me', body=message)
            sent_message=execute_with_retry(request)       
            if not self.parent.connection_error(sent_message):
                return True
    @KeyHandler.hotkey("Control+F")
    def command_forward(self,modified_name):
        """
        If not shifted or not viewing a received message then suppress as attempt to do Command_Find on the dialog is super confusing due to also being consumed by BodyDialog 
        Otherwise, this command has the same effect as this key on the parent MailDialog which operates on the current element in that dialog (i.e. the one that launched this message dialog 
        This is achieved by adjusting this dialog's action attribute and invoking refill() with a rebuilt list of filler fields  
        """ 
        if ('Shift' not in modified_name) or (self.action!='view'):
            beep()
        else:
            self.reload_as_reply_or_forward('forward')
    def reload_as_reply_or_forward(self,action):
        self.action=action
        self.empty_value='edit'
        owner=self.parent.profile.get('emailAddress','me') 
        addresses=reply_addresses(action,owner,self.summary)
        subject=prefixed_subject(action,self.summary.get('Subject','No Subject field'))
        fields=message_filler(action,addresses=addresses,subject=subject,summary=self.summary,prefer_text_format=None,content=self.content,include_attachments=not ('reply' in action))
        self.refill(fields)
    @KeyHandler.hotkey("Control+R")
    def command_reply(self,modified_name):
        """
        has the same effect as this key on the parent MailDialog which operates on the current element in that dialog (i.e. the one that launched this message dialog 
        This is achieved by adjusting this dialog's action attribute and invoking refill() with a rebuilt list of filler fields          
        """ 
        if not self.action=='view':
            beep()
        else:
            self.reload_as_reply_or_forward('reply' if not 'Shift' in modified_name else 'reply all')
    @KeyHandler.hotkey("Alt+F")
    def command_toggle_format(self,modified_name):
        prefer_text_format  = True if self.format=='html' else False
        body_index = self.index_from_name('Body')
        new_format='text' if self.format=='html' else 'html'
        content_key = {'html':'text/html','text':'text/plain'}[new_format]
        if self.content[content_key] is None: 
            self.say_strings(f"Cannot switch view: no {new_format} content available")
        else:    
            if not self.action=='view':
                if self.message(title='Switching format will lose any edits you have already made.  Proceed anyway?',select='No')=='No':
                    self.say_strings('cancelled format switch')
                    return
            self[body_index].refill([self.action,prefer_text_format,self.summary])
            self.say_strings(f"switched to {self.format} view")
    @KeyHandler.hotkey("Alt+T")
    def command_toggle_say_tags(self,modified_name):
        if 'Shift' in modified_name:
            if self.say_tags:
                self.say_unlabelled_graphics = not self.say_unlabelled_graphics
                self.say_strings(f"speak unlabelled graphics {'enabled' if self.say_unlabelled_graphics else 'disabled'}")
            else:
                self.say_strings('enable all tag voicing before manipulating unlabelled graphic tag voicing')
        else:
            self.say_tags = not self.say_tags
            self.say_strings(f"speak tags {'enabled' if self.say_tags else 'disabled'}")        
    def is_voiceable(self,tag):
        if self.say_tags: 
            if tag.unlabelled_graphic:
                return self.say_unlabelled_graphics
            else:
                return True
        else:
            return False        
    @KeyHandler.hotkey("Control+V")
    def command_paste_or_move_message(self,modified_name):
        if 'Shift' in modified_name:
            #move the message represented by the element whose Return hotkey launched this MessageDialog 
            self.parent.current().command_move_message(modified_name)
            return False #close this MessageDialog 
        else:
            super().command_paste(modified_name)
    @KeyHandler.hotkey("Control+F")
    def command_find_or_forward(self,modified_name):
        if 'Shift' in modified_name:
            #forward the message in the same way as if keystroke pressed when on the home dialog for mail
            self.parent.current().command_move_message(modified_name)
            return False #close this MessageDialog 
        else:
            super().command_forward(modified_name)
    @KeyHandler.hotkey("Alt+A")
    def command_attachments(self,modified_name):
        attachments=attachments_from_content(self.content)
        if not len(attachments):
            self.say_strings('No attachments available')
            return
        selection = self.message(title='attachments', buttons=[a[0] for a in attachments]) 
        if not selection:
            self.say_current()
        else:
            attachment= [a for a in attachments if a[0]==selection.strip()][0]
            if attachment[2]=='inline':
                self.say_strings('implement handling of inline attachment by saving as a file then proceeding to open_file logic below')
            else:
                file_path =self.parent.get_attachment_via_cache(self.summary['id'],attachment_id=attachment[1],filename=attachment[0])
                if file_path:
                    open_file(self.parent,file_path,safe=False)
                else:
                    self.say_strings(f'failed to retrieve attachment {attachment[0]}')