# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_core.ipynb.

# %% auto 0
__all__ = ['cid_to_char', 'process_text', 'extract_text']

# %% ../nbs/00_core.ipynb 2
from .imports import *

# %% ../nbs/00_core.ipynb 3
def cid_to_char(cidx: str):
    try:
        return chr(int(re.findall(r'\(cid\:(\d+)\)',cidx)[0]) + 29)
    except:
        return cidx

def process_text(text: str):
    text = text.strip()
    text = cid_to_char(text)
    text = re.sub(u'\xa0', u' ', text)
    text = re.sub(" +", " ", text)
    text = re.sub(r"\uf0b7", " ", text)
    text = re.sub(r"\(cid:\d{0,3}\)", " ", text)
    text = re.sub(r'•', " ", text)
    text = re.sub(r'●', " ", text)
    text = re.sub(r'▪', " ", text)
    text = re.sub(r'➢', " ", text)
    text = re.sub(r'\u2013', " ", text)
    return text.strip()

def extract_text(data_path: Union[str, Path, List[Union[str, Path]]]):
    pdfs = resolve_data_path(data_path)
    pdf_dict = {}
    for file in pdfs:
        if Path(file).suffix == '.pdf':
            pdf = PdfReader(str(file))
            pdf_text = []
            for page in pdf.pages:
                text = page.extract_text().splitlines()
                for txt in text:
                    txt = process_text(txt)
                    if not txt.isspace() and len(txt) > 2:
                        if len(txt) > 512:
                            nw = 10
                            # split txt into smaller chunks of size 20 words
                            txt = txt.split()
                            txt = [' '.join(txt[i:i+nw]) for i in range(0, len(txt), nw)]
                        pdf_text.append(txt)
                        
                        # if len(page_text) == 0:
                        #     page_text.append(txt)
                        # else:
                        #     txt2 = page_text[-1]
                        #     if txt2[-1] not in ['.'] and len(txt2+txt) < 256:
                        #         page_text[-1] = txt2 + ' ' + txt
                        #     else:
                        #         page_text.append(txt)
                # pdf_text += page_text
            pdf_dict[str(file)] = flatten_list(pdf_text)
    return pdf_dict
