# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/utils.ipynb.

# %% auto 0
__all__ = ['default_keys', 'export_env', 'gather_with_concurrency', 'rename_filepath_to_match_datatype', 'detect_encoding',
           'read_html_file', 'remove_query_params_from_url', 'update_env', 'upsert_folder', 'get_all_files_and_folders',
           'ImageBlockConverter', 'md', 'convert_html_to_markdown', 'download_zip', 'download_pptx',
           'convert_str_to_snake_case', 'convert_str_remove_accents', 'convert_str_keep_alphanumeric',
           'convert_str_file_name', 'convert_str_to_date']

# %% ../nbs/utils.ipynb 3
import os
from typing import List, Tuple, Union

import re
import pathlib
import unicodedata
import json
import chardet
from urllib.parse import urljoin, urlparse

from bs4 import BeautifulSoup
from markdownify import MarkdownConverter

import datetime as dt

import asyncio

import pptx2md
from PIL import Image

import zipfile
import io


import datetime as dt
from dateutil.parser import parse as dtu_parse

from dotenv import set_key, load_dotenv

# %% ../nbs/utils.ipynb 6
default_keys = [
    "DOCKER_BUILDKIT",
    "ENABLE_DYNAMIC_INSTALL",
    "LESSOPEN",
    "GIT_COMMITTER_NAME",
    "PYTHONIOENCODING",
    "GITHUB_CODESPACE_TOKEN",
    "USER",
    "RVM_PATH",
    "NVS_ROOT",
    "HOSTNAME",
    "DOTNET_USE_POLLING_FILE_WATCHER",
    "CONDA_SCRIPT",
    "PIPX_HOME",
    "SHLVL",
    "GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN",
    "GCP_KEY",
    "HUGO_ROOT",
    "HOME",
    "OLDPWD",
    "ORYX_ENV_TYPE",
    "NVM_BIN",
    "CODESPACES",
    "DOTNET_RUNNING_IN_CONTAINER",
    "NVM_SYMLINK_CURRENT",
    "DYNAMIC_INSTALL_ROOT_DIR",
    "PIPX_BIN_DIR",
    "NVM_INC",
    "rvm_stored_umask",
    "ORYX_DIR",
    "GRADLE_HOME",
    "rvm_user_install_flag",
    "MAVEN_HOME",
    "GOROOT",
    "NODE_ROOT",
    "GITHUB_GRAPHQL_URL",
    "GITHUB_USER",
    "NVM_DIR",
    "PYTHON_PATH",
    "DOTNET_SKIP_FIRST_TIME_EXPERIENCE",
    "ContainerVersion",
    "NVS_HOME",
    "GITHUB_API_URL",
    "rvm_bin_path",
    "SDKMAN_CANDIDATES_API",
    "_",
    "RUBY_VERSION",
    "PROMPT_DIRTRIM",
    "IRBRC",
    "CLOUDENV_ENVIRONMENT_ID",
    "DOTNET_ROOT",
    "NVS_DIR",
    "PHP_ROOT",
    "PATH",
    "JAVA_ROOT",
    "VSCODE_AGENT_FOLDER",
    "SDKMAN_CANDIDATES_DIR",
    "HUGO_DIR",
    "NPM_GLOBAL",
    "SHELL_LOGGED_IN",
    "MY_RUBY_HOME",
    "LANG",
    "SDKMAN_DIR",
    "RUBY_ROOT",
    "LS_COLORS",
    "SDKMAN_PLATFORM",
    "GITHUB_REPOSITORY",
    "SHELL",
    "GOPATH",
    "rvm_prefix",
    "rvm_loaded_flag",
    "GEM_HOME",
    "ORYX_PREFER_USER_INSTALLED_SDKS",
    "LESSCLOSE",
    "ORYX_SDK_STORAGE_BASE_URL",
    "CONDA_DIR",
    "rvm_version",
    "DEBIAN_FLAVOR",
    "GIT_COMMITTER_EMAIL",
    "GEM_PATH",
    "JAVA_HOME",
    "NVS_USE_XZ",
    "INTERNAL_VSCS_TARGET_URL",
    "PWD",
    "NVM_CD_FLAGS",
    "GITHUB_SERVER_URL",
    "PHP_PATH",
    "PYTHON_ROOT",
    "RAILS_DEVELOPMENT_HOSTS",
    "NVS_OS",
    "CODESPACE_NAME",
    "RUBY_HOME",
    "MAVEN_ROOT",
    "rvm_path",
    "NUGET_XMLDOC_MODE",
    "VSCODE_HANDLES_SIGPIPE",
    "VSCODE_AMD_ENTRYPOINT",
    "VSCODE_HANDLES_UNCAUGHT_ERRORS",
    "VSCODE_NLS_CONFIG",
    "BROWSER",
    "VSCODE_CWD",
    "ELECTRON_RUN_AS_NODE",
    "VSCODE_IPC_HOOK_CLI",
    "VSCODE_L10N_BUNDLE_LOCATION",
    "DEBUG",
    "PYDEVD_IPYTHON_COMPATIBLE_DEBUGGING",
    "PYTHONUNBUFFERED",
    "PYDEVD_USE_FRAME_EVAL",
    "TERM",
    "CLICOLOR",
    "FORCE_COLOR",
    "CLICOLOR_FORCE",
    "PAGER",
    "GIT_PAGER",
    "MPLBACKEND",
]


def export_env(default_keys, output_file_path):
    keep_keys = [key for key in os.environ.keys() if key not in default_keys]

    with open(output_file_path, "w+") as f:
        f.writelines(
            [
                f"{key} = '{value}'\n"
                for key, value in os.environ.items()
                if key in keep_keys
            ]
        )
        f.write(f"_OUTPUT_DATE = '{dt.datetime.now().strftime('%Y-%m-%d %H:%M')}'")

# %% ../nbs/utils.ipynb 9
async def gather_with_concurrency(
    *coros,  # list of coroutines to await
    n=60,  # number of open coroutines
):
    """limits the number of open coroutines at a time."""

    semaphore = asyncio.Semaphore(n)

    async def sem_coro(coro):
        async with semaphore:
            return await coro

    return await asyncio.gather(*(sem_coro(c) for c in coros))

# %% ../nbs/utils.ipynb 11
def rename_filepath_to_match_datatype(data, file_path):

    is_path_ext = os.path.splitext(file_path)[-1].lower()

    old_suffix = pathlib.Path(file_path).suffix if is_path_ext else None

    new_suffix = ""

    print(type(data))

    if isinstance(data, str) or isinstance(data, bytes) or isinstance(data, bytearray):
        new_suffix = ".txt"
    if isinstance(data, dict):
        new_suffix = ".json"

    file_path = file_path + new_suffix

    if old_suffix:
        file_path = file_path.replace(old_suffix, "")

    return file_path

# %% ../nbs/utils.ipynb 13
def detect_encoding(file_path, debug_prn: bool = False):
    detector = chardet.universaldetector.UniversalDetector()
    with open(file_path, "rb") as f:
        for line in f:
            detector.feed(line)
            if detector.done:
                break
    detector.close()

    encoding = detector.result

    return encoding

# %% ../nbs/utils.ipynb 15
def read_html_file(
    file_path, is_convert_to_soup: bool = True
) -> Union[str, BeautifulSoup]:
    if not os.path.exists(file_path):
        raise FileNotFoundError(file_path)

    page_encoding = detect_encoding(file_path)

    with open(file_path, encoding=page_encoding["encoding"]) as fp:
        if is_convert_to_soup:
            return BeautifulSoup(fp, "lxml")

        return fp.read()

# %% ../nbs/utils.ipynb 17
def remove_query_params_from_url(url):
    u = urlparse(url)
    return urljoin(url, urlparse(url).path)

# %% ../nbs/utils.ipynb 19
def update_env(env_path: str, key: str, value: str, debug_prn: bool = False) -> dict:
    """
    updates a .env file with a key value pair
    then reloads the env_file
    """

    if not os.path.exists(env_path):
        with open(env_path, "w", encoding="utf-8") as f:
            f.write("")

    quote_mode = "always"

    if isinstance(value, dict):
        quote_mode = "never"
        value = json.dumps(value)

    if debug_prn:
        from pprint import pprint

        pprint(
            {
                "env_path": env_path,
                "key": key,
                "value": value,
                "type": type(value),
                "quote_mode": quote_mode,
            }
        )

    set_key(env_path, key, value, quote_mode=quote_mode)

    set_key(env_path, "env_last_modified", f"updated - {dt.date.today()}")

    load_dotenv(env_path, override=True)

    return {key: os.getenv(key)}

# %% ../nbs/utils.ipynb 20
def upsert_folder(folder_path: str, debug_prn: bool = False):
    folder_path = os.path.dirname(folder_path)

    if debug_prn:
        print(
            {
                "upsert_folder": os.path.abspath(folder_path),
                "is_exist": os.path.exists(folder_path),
            }
        )

    if not os.path.exists(folder_path):
        os.makedirs(folder_path)

# %% ../nbs/utils.ipynb 21
def get_all_files_and_folders(
    directory, file_type=None  # to only retrieve a specific file type
) -> Union[Tuple, List]:
    """walk a directory and retrieve a list of files and a list of directory
    returns Tuple of file_ls , dir_ls OR file_ls if file_type supplied
    """
    if not os.path.exists(directory):
        raise FileNotFoundError(directory)

    file_ls = []
    dir_ls = []
    for root, dirs, files in os.walk(directory):
        for name in files:
            if file_type:
                if not name.lower().endswith(file_type.lower()):
                    continue
            file_ls.append(os.path.join(root, name))

        if file_type:
            continue

        for name in dirs:
            dir_ls.append(os.path.join(root, name))

    if file_type:
        return file_ls

    return file_ls, dir_ls

# %% ../nbs/utils.ipynb 24
class ImageBlockConverter(MarkdownConverter):
    """
    Create a custom MarkdownConverter that adds two newlines after an image
    """

    def convert_img(self, el, text, convert_as_inline, is_resize: bool = True):
        """
        custom image downloader for ImabeBlockConverter
        will handle resize
        """

        if is_resize:
            style_obj = {
                (obj.split(":")[0].strip()): obj.split(":")[1].strip()
                for obj in el.get("style").split(";")
                if ":" in obj
            }

            file_path = os.path.join(
                os.path.dirname(self.options["file_path"]), el["src"]
            )

            image = Image.open(file_path)

            width = style_obj["width"].replace("px", "")
            width = int(float(width))

            height = style_obj["height"].replace("px", "")
            height = int(float(height))

            new_image = image.resize((width, height))
            new_image.save(file_path)

        return super().convert_img(el, text, convert_as_inline)


def md(html, **options):
    """Create shorthand method for handling conversion"""
    return ImageBlockConverter(**options).convert(html)

# %% ../nbs/utils.ipynb 25
def convert_html_to_markdown(file_path):
    """converts html file to markdown in place"""

    with open(file_path, encoding="utf-8") as f:
        html = f.read()

    markdown_content = md(
        str(html),
        keep_inline_images_in=["td", "span"],
        file_path=file_path,
        is_resize=True,
    )

    md_path = file_path.replace(".html", ".md")

    with open(md_path, "w+", encoding="utf-8") as f:
        f.write(markdown_content)

    return

# %% ../nbs/utils.ipynb 26
def download_zip(zip_bytes_content, output_folder, is_convert_to_markdown: bool = True):
    """save bytes content to a zip file then convert html to markdown"""

    zip = zipfile.ZipFile(io.BytesIO(zip_bytes_content), "r")
    zip.extractall(output_folder)

    file_ls = os.listdir(output_folder)

    # rename the html file to index.html
    for file_name in file_ls:
        if file_name.endswith(".html"):
            output_index = os.path.join(output_folder, "index.html")
            os.replace(os.path.join(output_folder, file_name), output_index)

            if is_convert_to_markdown:
                convert_html_to_markdown(os.path.join(output_folder, "index.html"))

    return f"successfully downloaded zip to {output_folder}"

# %% ../nbs/utils.ipynb 29
def download_pptx(
    pptx_bytes_content, output_folder, is_convert_to_markdown: bool = True
):
    """save bytes content to a pptx file then converts to markdown"""

    upsert_folder(output_folder)

    output_ppt_index = os.path.join(output_folder, "index.pptx")

    with open(output_ppt_index, "wb+") as binary_file:
        # Write bytes to file
        binary_file.write(pptx_bytes_content)

    if is_convert_to_markdown:
        pptx2md.convert(
            output_ppt_index,
            output=os.path.join(output_folder, "index.md"),
            image_dir=os.path.join(output_folder, "images"),
        )

    return f"successfully downloaded content to {output_folder}"

# %% ../nbs/utils.ipynb 33
def convert_str_to_snake_case(text_str):
    """converts 'snake_case_str' to 'snakeCaseStr'"""

    return text_str.replace(" ", "_").lower()

# %% ../nbs/utils.ipynb 34
def convert_str_remove_accents(text_str: str) -> str:
    return "".join(
        c
        for c in unicodedata.normalize("NFD", text_str)
        if unicodedata.category(c) != "Mn"
    )

# %% ../nbs/utils.ipynb 36
def convert_str_keep_alphanumeric(text_str) -> str:
    pattern = "[^0-9a-zA-Z_\s]+"

    return re.sub(pattern, "", text_str)

# %% ../nbs/utils.ipynb 37
def convert_str_file_name(text_str: str) -> str:
    """convert strings to clean file name or url"""

    return convert_str_keep_alphanumeric(
        convert_str_to_snake_case(convert_str_remove_accents(text_str))
    )

# %% ../nbs/utils.ipynb 40
def convert_str_to_date(datefield: str) -> dt.datetime:
    """converts string date to datetime object"""
    return dtu_parse(datefield) if datefield else None
