lightudq.utils API documentation

# lightudq.utils

View Source

     1import os
     2
     3import pymupdf
     4
     5
     6def read_markdown_to_text(file_path: str) -> str:
     7    with open(file_path, encoding="utf-8") as f:
     8        return f.read()
     9
    10
    11def read_pdf_to_text(file_path: str) -> str:
    12    doc = pymupdf.open(file_path)
    13    text = ""
    14    for page in doc:  # iterate the document pages
    15        text += page.get_text()  # get plain text (is in UTF-8)
    16    return text
    17
    18
    19def read_document(file_path: str) -> str:
    20    """Read document content based on file extension"""
    21    _, file_extension = os.path.splitext(file_path)
    22    file_extension = file_extension.lower()
    23
    24    if file_extension == ".md":
    25        return read_markdown_to_text(file_path)
    26    elif file_extension == ".txt":
    27        with open(file_path) as file:
    28            return file.read()
    29    elif file_extension == ".pdf":
    30        return read_pdf_to_text(file_path)
    31    else:
    32        print(f"Unsupported file format: {file_extension}")
    33        return ""

def read_markdown_to_text(file_path: str) -> str: View Source

    7def read_markdown_to_text(file_path: str) -> str:
    8    with open(file_path, encoding="utf-8") as f:
    9        return f.read()

def read_pdf_to_text(file_path: str) -> str: View Source

    12def read_pdf_to_text(file_path: str) -> str:
    13    doc = pymupdf.open(file_path)
    14    text = ""
    15    for page in doc:  # iterate the document pages
    16        text += page.get_text()  # get plain text (is in UTF-8)
    17    return text

def read_document(file_path: str) -> str: View Source

    20def read_document(file_path: str) -> str:
    21    """Read document content based on file extension"""
    22    _, file_extension = os.path.splitext(file_path)
    23    file_extension = file_extension.lower()
    24
    25    if file_extension == ".md":
    26        return read_markdown_to_text(file_path)
    27    elif file_extension == ".txt":
    28        with open(file_path) as file:
    29            return file.read()
    30    elif file_extension == ".pdf":
    31        return read_pdf_to_text(file_path)
    32    else:
    33        print(f"Unsupported file format: {file_extension}")
    34        return ""

Read document content based on file extension