import bilbystats as bs
import os
import pandas as pd

def inverse_bulletins(html, savename, savedir = '/Users/samd/Documents/General_code/Scraping/bulletin_pages/', model = 'anthropic/claude-sonnet-4', prompt = None):
    if not prompt:
        prompt = '''Here is the html of a bulettin page with articles coming in:''' + html + '''Extract the urls and their meta information of the articles using Python and BeautifulSoup. For each relevant url include the following:
- url (link to the article)
- title (article/page title)
- date (publication date)
- description (a description of that article if it is available - otherwise None)

Requirements:
- Use robust selectors that handle missing elements gracefully
- Return None for fields that cannot be found
- Clean and normalize extracted text (strip whitespace, remove extra newlines)
- Return results as a list of dictionaries with keys: 'url', 'title', 'data', 'description'

Call the function used to extract the fields: extract_articles. 

Additionally include another python function which returns the url for the next page of articles. 
Call this function: next_page_url

Additionally include another python function which returns the total number of bulletin pages.
Call this function: total_pages

Provide only the Python code without explanations. Begin directly with the code - no backticks.'''

    code = bs.llm_api(prompt, '', model, max_tokens = 10000)
    code = bs.process_python_string(code)
    
    with open(savedir + savename + '.py', "w") as file:
        file.write(code)
        
    return code

def next_page_code(url, savename, savedir = '/Users/samd/Documents/General_code/Scraping/next_bulletin_page/', model = 'gpt-4o-mini'):
    prompt = 'Write a python function which takes in an integer n and returns the url of the nth page of the url provided. Call the function get_nth_page_url. Just return the python code.'
    code = bs.llm_api(url, prompt, model, temperature = 0)
    code = bs.process_python_string(code)
    
    with open(savedir + savename + '.py', "w") as file:
        file.write(code)
        
    return code

def get_bulletin_pages(url, savename, savedir, model = 'anthropic/claude-sonnet-4', overwrite_modules = False, use_proxy = False, total_pages = None):
    print('Scraping original url')
    if use_proxy:
        proxy_url = bs.get_proxy_url(url)
    else:
        proxy_url = url
        
    html = bs.scrape(proxy_url)

    if not os.path.exists(savedir + savename + '.py') or overwrite_modules:
        print('Inverse scraping the landing bulletin page')
        bs.inverse_bulletins(html, savename, savedir = savedir, model = model)
    
    bulletin_module = bs.import_module(savename, savedir = savedir)
    if not total_pages:
        total_pages = bulletin_module.total_pages(html)
        
    next_page_url = bulletin_module.next_page_url(html)

    if not os.path.exists(savedir + savename + '_np.py') or overwrite_modules:
        print('Obtaining the code to generate the next page URL')
        bs.next_page_code(next_page_url, savename + '_np', savedir = savedir, model = model)

    np_module = bs.import_module(savename + '_np', savedir = savedir)
    
    if np_module.get_nth_page_url(2) == next_page_url:
        offset = 0
    elif np_module.get_nth_page_url(1) == next_page_url:
        offset = 1
    elif np_module.get_nth_page_url(3) == next_page_url:
        offset = -1
    else:
        raise Exception("Next page prediction failed")

    pages2scrape = [url]
    pages2scrape += [np_module.get_nth_page_url(i) for i in range(2 - offset, total_pages + 1)]
    
    if use_proxy == True:
        pages2scrape = [bs.get_proxy_url(k) for k in pages2scrape]

    return pages2scrape

async def get_bulletin_board(url, savename, savedir, model = 'anthropic/claude-sonnet-4', overwrite_modules = False, datasavedir = None, use_proxy = False, total_pages = None):
    
    pages2scrape = get_bulletin_pages(url, savename, savedir, model = model, overwrite_modules = overwrite_modules, use_proxy = use_proxy, total_pages = total_pages)
    bulletin_module = bs.import_module(savename, savedir = savedir)

    print('Scraping bulletin links')
    scraped_bulletin_pages = await bs.scrape_urls(pages2scrape)
    
    print('Extracting articles listed')
    fun = lambda html: bulletin_module.extract_articles(html)
    article_list = bs.parallelize(scraped_bulletin_pages, fun)
    
    all_articles = []
    for article in article_list:
        all_articles += article
        
    articles_df = pd.DataFrame(all_articles)
    
    print('Scraping individual pages')
    urls2scrape = articles_df['url'].to_list()
    if use_proxy:
        urls2scrape = [bs.get_proxy_url(k) for k in urls2scrape]

    scraped_indi_pages = await bs.scrape_urls(urls2scrape)
    
    print('Extracting individual articles text')
    fun = lambda html: bs.get_meta_data(html=html)['text']
    article_texts = bs.parallelize(scraped_indi_pages, fun)
    articles_df['text'] = article_texts
    
    if datasavedir:
        articles_df.to_parquet(datasavedir + savename + '.parquet')
    
    return articles_df

async def get_relevance(query, df, covariate, model_name = 'gpt-4o-mini'):
    prompt = '''Decide whether the provided text is relevant to ''' + query + '''.
Expected JSON output format:
{
    "reasoning": <cot reasoning>,
    "relevant": <true or false>
}
- Output ONLY the JSON object, nothing else
- Do NOT use markdown code blocks or backticks
- Start your response directly with the opening brace {
'''
    schema = {"reasoning": str, "relevant": str}
    response_format = bs.make_response_format(schema, strict = False)

    relevance = await bs.batch_llm_api_async_mr(df[covariate].to_list(), prompt, columns = ['reasoning', 'relevant'], prefix = 'Text: ', response_format = response_format, model_name = model_name, temperature = 0)
    
    relevant_entries = bs.filter_df(relevance, 'relevant', ['true'], [], reset_index = False)
    relevant_df = df.iloc[relevant_entries.index]
    relevant_df.reset_index(drop=True, inplace=True)
    relevant_df
    
    return relevant_df, relevance
    