## Code to scrape today's arxiv and email yourself unread papers
## Script requires existing author and topic lists, stored in the same folder
## These should be called 'topics.txt' and 'authors.txt'

import feedparser
import requests
from bs4 import BeautifulSoup
import sys
import os
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import argparse

# now, compare with my lists
def read_list_from_file(filepath):
    with open(filepath, 'r', encoding='utf-8') as f:
        lines = []
        for line in f:
            clean_line = line.strip()
            # Remove backslashes before quotes, if any
            clean_line = clean_line.replace('\\"', '"').replace("\\'", "'")
            if clean_line:
                lines.append(clean_line)
    return lines

def read_paper_list_from_txt(filepath):
    """
    Reads a plain .txt file and reconstructs a list of paper dictionaries.
    """
    separator = " ~|~ "
    papers = []
    if os.path.exists(filepath):
        with open(filepath, 'r', encoding='utf-8') as f:
            for line in f:
                parts = line.strip().split(separator)
                if len(parts) == 5:
                    arxiv_id, title, authors_str, abstract_url, abstract = parts
                    authors = [a.strip() for a in authors_str.split(";")]
                    paper = {
                        'arxiv_id': arxiv_id,
                        'title': title,
                        'authors': authors,
                        'abstract_url': abstract_url,
                        'abstract': abstract
                    }
                    papers.append(paper)
    else:
        print('Creating new old_papers.txt')
    return papers

def write_paper_list_to_txt(paper_list, filepath):
    """
    Writes each paper dict to a line in a plain .txt file using a custom separator.
    """
    separator = " ~|~ "  # unlikely to occur in normal text
    with open(filepath, 'w', encoding='utf-8') as f:
        for paper in paper_list:
            line = separator.join([
                paper['arxiv_id'],
                paper['title'].replace('\n', ' ').strip(),
                "; ".join(paper['authors']),
                paper['abstract_url'],
                paper['abstract'].replace('\n', ' ').strip()
            ])
            f.write(line + '\n')

def fetch_latest_astro_ph_with_abstracts():
    url = "https://arxiv.org/list/astro-ph/new"
    resp = requests.get(url)
    resp.raise_for_status()
    soup = BeautifulSoup(resp.text, 'html.parser')

    dl = soup.find('dl')
    if not dl:
        print("Could not find listing (<dl>) on the page.")
        return []

    dt_tags = dl.find_all('dt')
    dd_tags = dl.find_all('dd')

    if len(dt_tags) != len(dd_tags):
        print("Warning: Mismatch between number of <dt> and <dd> elements")

    papers = []

    for dt, dd in zip(dt_tags, dd_tags):
        # arXiv ID and abstract URL
        link_tag = dt.find('a', title='Abstract')
        arxiv_id = link_tag.text.strip() if link_tag else None
        abstract_url = f"https://arxiv.org{link_tag['href']}" if link_tag else None

        # Title
        title_tag = dd.find('div', class_='list-title')
        title = title_tag.text.replace('Title:', '').strip() if title_tag else None

        # Authors
        authors_tag = dd.find('div', class_='list-authors')
        authors = [a.text.strip() for a in authors_tag.find_all('a')] if authors_tag else []

        # Abstract (requires following the abstract URL)
        abstract = None
        if abstract_url:
            abs_resp = requests.get(abstract_url)
            abs_soup = BeautifulSoup(abs_resp.text, 'html.parser')
            abstract_tag = abs_soup.find('blockquote', class_='abstract')
            if abstract_tag:
                abstract_text = abstract_tag.get_text(strip=True)
                abstract = abstract_text.replace('Abstract:', '').strip()

        papers.append({
            'arxiv_id': arxiv_id,
            'title': title,
            'authors': authors,
            'abstract_url': abstract_url,
            'abstract': abstract
        })
    return papers

def send_paper_email(papers, from_addr, to_addr, smtp_server, smtp_port, password):
    # Compose email content
    message_body = ""
    for paper in papers:
        authors = ", ".join(paper['authors'])
        clean_abstract = paper['abstract'].replace('\n', ' ').replace('\r', ' ').strip()
        clean_title = paper['title'].replace('\n', ' ').replace('\r', ' ').strip()
        clean_link = paper['abstract_url']
        message_body += f"Title: {clean_title}<br>"
        message_body += f"Authors: {authors}<br>"
        message_body += f'Link: <a href="{clean_link}">Arxiv link</a><br>'
        message_body += f"Abstract: {clean_abstract}<br>"
        message_body += "-" * 80 + "<p>"

    # Create MIME message
    msg = MIMEMultipart()
    msg['From'] = from_addr
    msg['To'] = to_addr
    msg['Subject'] = "arXiv Paper Summaries"

    # Attach the message body as plain text
    msg.attach(MIMEText(message_body, 'html'))

    # Connect to SMTP server and send email
    with smtplib.SMTP(smtp_server, smtp_port) as server:
        server.starttls()  # Secure the connection
        server.login(from_addr, password)
        server.send_message(msg)
        print("Email sent successfully!")


def send_warning_email(from_addr, to_addr, smtp_server, smtp_port, password):
    # Compose email content
    message_body = ""

    # Create MIME message
    msg = MIMEMultipart()
    msg['From'] = from_addr
    msg['To'] = to_addr
    msg['Subject'] = "papergirl not working today - please attend"

    # Attach the message body as plain text
    msg.attach(MIMEText(message_body, 'html'))

    # Connect to SMTP server and send email
    with smtplib.SMTP(smtp_server, smtp_port) as server:
        server.starttls()  # Secure the connection
        server.login(from_addr, password)
        server.send_message(msg)
        print("Warning email sent successfully!")

def main():

    parser = argparse.ArgumentParser(description="Process two inputs.")
    parser.add_argument('--authors', default='authors.txt', help="location of file that lists authors of interest")
    parser.add_argument('--topics', default='topics.txt', help="location of file that lists topics of interest")

    args = parser.parse_args()

    # Set the folders and prefix
    if (args.authors):
        author_loc = args.authors
    else:
        author_loc = './authors.txt'
    if (args.topics):
        topic_loc = args.topics
    else:
        topic_loc = './topics.txt'

    author_list = read_list_from_file('./authors.txt')
    topic_list = read_list_from_file('./topics.txt')
    previously_collected = read_paper_list_from_txt('old_papers.txt')

    ## Actually fetch all papers from today's arxiv
    papers = fetch_latest_astro_ph_with_abstracts()

    ## Check today's papers for my author and topic lists
    num_papers = len(papers)
    collected = []
    send_warning = False
    if (num_papers == 0):
        send_warning = True
        exit

    i = 0
    for paper in papers:
        collect_it = False

        # check if it's already been collected
        if paper in previously_collected:
            continue

        # check if the authors are people we are looking for
        for author in paper['authors']:
            last_name = author.split()[-1]
            for item in author_list:
                if last_name in item:
                    collect_it = True

        # check if the title or abstract has any of our key words
        summary = paper['abstract'].lower()
        if any(word.lower() in summary for word in topic_list):
            collect_it = True
        # now if we need to, add it to the list
        # also add it to the historical list so we don't collect it twice
        if (collect_it == True):
            collected.append(paper)
            previously_collected.insert(0,paper)
        i += 1

    # Now write the first 100 previously collected papers to file so we can keep a record
    write_paper_list_to_txt(previously_collected,'old_papers.txt')

    ## Send this information to myself

    # now send myself an email with this info

    # retrieve the app password
    app_password = os.getenv("PAPERGIRL_PASSWORD")
    app_from = os.getenv("PAPERGIRL_FROM")
    app_to = os.getenv("PAPERGIRL_TO")

    call_stop = False
    if (app_password == None): call_stop = True
    if (app_from == None): call_stop = True
    if (app_to == None): call_stop = True

    if (call_stop):
        sys.exit('Stored app details are missing, save appropriately and try again.')

    if (send_warning == True):
        send_warning_email(
        from_addr=app_from,
        to_addr=app_to,
        smtp_server="smtp.gmail.com",
        smtp_port=587,
        password=app_password  # Use app password, not your Gmail password
        )
    else:
        send_paper_email(
            from_addr=app_from,
            papers=collected,
            to_addr=app_to,
            smtp_server="smtp.gmail.com",
            smtp_port=587,
            password=app_password  # Use app password, not your Gmail password
        )

if __name__ == "__main__":
    main()