import argparse
import sys
try:
    from .logger import Logger
    from .http_server import HTTP_Server
    from .version import __version__
    from .utils import sidebar_summary2dict, update_config, check_sidebar_diff
    from .html_renderer import Renderer
except Exception:
    from logger import Logger
    from http_server import HTTP_Server
    from version import __version__
    from utils import sidebar_summary2dict, update_config, check_sidebar_diff
    from html_renderer import Renderer
import os, sys
import json, yaml
import subprocess
import shutil
import re
from collections import OrderedDict
import multiprocessing
import threading
import math
import copy
from queue import Queue, Empty
from datetime import datetime
import tempfile
import gettext

class RebuildException(Exception):
    pass

g_sitemap_content = {}
def add_robots_txt(site_config, out_dir, log):
    if not "robots" in site_config:
        site_config["robots"] = {}
    out_path = os.path.join(out_dir, "robots.txt")
    log.i("generate robots.txt")
    robots_items = site_config["robots"]
    robots_txt = ""
    if not "User-agent" in robots_items:
        robots_items["User-agent"] = "*"
    for k, v in robots_items.items():
        robots_txt += "{}: {}\n".format(k, v)
    robots_txt += "Sitemap: {}://{}/sitemap.xml\n".format(site_config["site_protocol"], site_config["site_domain"])
    with open(out_path, "w", encoding="utf-8") as f:
        f.write(robots_txt)

def generate_sitemap(update_htmls, out_path, site_domain, site_protocol, log):
    log.i("generate sitemap.xml")
    sitemap_content = '<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">\n'
    for doc_url in update_htmls:
        htmls = update_htmls[doc_url]
        for url, html in htmls.items():
            url = "{}://{}{}".format(site_protocol, site_domain, url)
            file_path = html['file_path']
            last_edit_time = datetime.fromtimestamp(os.stat(file_path).st_mtime)
            last_edit_time = str(last_edit_time).replace(" ", "T")
            change_freq = "weekly"
            priority = 1.0
            sitemap_item = '''    <url>
            <loc>{}</loc>
            <lastmod>{}</lastmod>
            <changefreq>{}</changefreq>
            <priority>{}</priority>
        </url>
    '''.format(url, last_edit_time, change_freq, priority)
            g_sitemap_content[url] = sitemap_item
    for url in g_sitemap_content:
        sitemap_content += g_sitemap_content[url]
    sitemap_content += '</urlset>\r\n'
    with open(out_path, "w", encoding='utf-8') as f:
        f.write(sitemap_content)


def split_list(obj, n):
    dist = math.ceil(len(obj)/n)
    for i in range(0, len(obj), dist):
        yield obj[i:i+dist]

def get_content_type_by_path(file_path):
    ext = os.path.splitext(file_path)[1][1:].lower()
    content_type = "text/plain"
    if ext == "svg":
        content_type = "image/svg+xml"
    elif ext == "html":
        content_type = "text/html"
    elif ext == "jpeg" or ext == "jpg" or ext == "png":
        content_type = "image/{}".format(ext)
    elif ext == "css":
        content_type = "text/css"
    elif ext == "js":
        content_type = "application/javascript"
    elif ext == "xml":
        content_type = "text/xml"
    elif ext == "json":
        content_type = "application/json"

    return content_type

def parse_site_config(doc_src_path):
    site_config_path = os.path.join(doc_src_path, "site_config.json")
    def check_site_config(config):
        configs = ["site_name", "site_slogon", "site_root_url", "site_domain", "site_protocol", "route", "executable", "plugins"]
        for c in configs:
            if not c in config:
                return False, "need {} keys, see example docs".format(configs)
        if not site_config['site_root_url'].endswith("/"):
            site_config['site_root_url'] = "{}/".format(site_config['site_root_url'])
        return True, ""
    site_config = load_config(doc_src_path, doc_src_path, config_name="site_config")    
    ok, msg = check_site_config(site_config)
    if not ok:
        return False, "check site_config.json fail: {}".format(msg)
    return True, site_config

def copy_dir(src, dst):
    if os.path.exists(dst):
        shutil.rmtree(dst)
    shutil.copytree(src, dst)
    return True

def copy_file(src, dst):
    dir = os.path.dirname(dst)
    if not os.path.exists(dir):
        os.makedirs(dir, exist_ok=True)
    try:
        shutil.copyfile(src, dst)
    except Exception:
        return False
    return True

def get_files(dir_path, warn=None):
    result = []
    files = os.listdir(dir_path)
    if not files:
        return []
    # check index.* and readme.*, only allow one
    if warn:
        flag = False
        for name in files:
            name0 = os.path.splitext(name)[0].lower()
            if name0 == "index" or name0 == "readme":
                if flag:
                    warn("dir {} include index file and readme file, please only use one!!!".format(dir_path))
                    break
                flag = True
    for name in files:
        path = os.path.join(dir_path, name)
        if os.path.isdir(path):
            f_list = get_files(path, warn)
            result.extend(f_list)
        else:
            result.append(path.replace("\\", "/"))
    return result

def write_to_file(files_content, in_path, out_path):
    '''
        @files_content      { "/home/neucrack/site/docs/get_started/zh/README.md": "<h1>index page</h1>"
        @in_path      "/home/neucrack/site/docs/get_started/zh"
        @out_path     "/home/neucrack/site/out/get_started/zh"
    '''
    for file, html in files_content.items():
        f_path = file.replace(in_path, out_path)
        d_path = os.path.dirname(f_path)
        if not os.path.exists(d_path):
            os.makedirs(d_path, exist_ok=True)
        # TODO: check file update, if not, skip
        if html: # html, change name
            if os.path.basename(f_path).lower() == "readme.md": # change readme.md to index.html
                f_path = os.path.join(os.path.dirname(f_path), "index.html")
            else:
                f_path = "{}.html".format(os.path.splitext(f_path)[0])
            with open(f_path, "w", encoding="utf-8") as f:
                f.write(html)
        else:    # normal files, just copy
            with open(f_path, "wb") as f:
                with open(file, "rb") as s:
                    f.write(s.read())
    return True, ""

def load_config(doc_dir, config_template_dir, config_name="config"):
    '''
        @doc_dir doc diretory, abspath
        @config_dir config template files dir, abspath
    '''
    config = {}
    config_path = os.path.join(doc_dir, config_name + ".json")
    if os.path.exists(config_path):
        with open(config_path, encoding="utf-8") as f:
            try:
                config_load = json.load(f)
            except Exception as e:
                raise Exception('\n\ncan not parse json file "{}"\njson format error: {}'.format(config_path, e))
    else:
        config_path = os.path.join(doc_dir, config_name + ".yaml")
        if not os.path.exists(config_path):
            config_path = os.path.join(doc_dir, config_name + ".yml")
        if not os.path.exists(config_path):
            raise Exception("can not open file: {}".format(config_path))
        with open(config_path, encoding="utf-8") as f:
            try:
                config_load = yaml.load(f.read(), Loader=yaml.Loader)
            except Exception as e:
                raise Exception('\ncan not parse yaml file "{}"\nyaml format error: {}'.format(config_path, e))
    config.update(config_load)

    if "import" in config:
        # update parent config
        config_name = config["import"]
        if config_name.endswith(".json") or config_name.endswith(".yaml"):
            config_name = config_name[:-5]
        config_parent = load_config(config_template_dir, config_template_dir, config_name = config_name)
        config = update_config(config_parent, config, ignore=["import"])
    return config


def load_doc_config(doc_dir, config_template_dir):
    config = load_config(doc_dir, config_template_dir)
    return config

def get_sidebar(doc_dir, config_template_dir):
    return load_config(doc_dir, config_template_dir, config_name="sidebar")

def get_navbar(doc_dir, config_template_dir):
    return load_config(doc_dir, config_template_dir)["navbar"]

def get_plugins_config(doc_dir, config_template_dir):
    return load_config(doc_dir, config_template_dir)["plugins"]

def get_footer(doc_dir, config_template_dir):
    return load_config(doc_dir, config_template_dir)["footer"]

def get_url_by_file_rel(file_path, doc_url = "", rel = False):
    '''
        @rel relative url
    '''
    url = os.path.splitext(file_path)[0]
    tmp = os.path.split(url)
    if tmp[1].lower() == "readme":
        url = "{}/index".format(tmp[0])
        if url.startswith("/"):
            url = url[1:]
    url = url + ".html"
    if rel:
        return url
    if(doc_url.endswith("/")):
        url = "{}{}".format(doc_url, url)
    else:
        url = "{}/{}".format(doc_url, url)
    return url

def get_sidebar_list(sidebar, doc_path, doc_url, log, redirect_err_file = False, redirct_url=f"no_translate.html", ref_doc_url="", add_file_item = True):
    '''
        @return {
            "file_path": {
                "curr": (url, label),
                "previous": (url, label),
                "next": (url, label),
            }
        }
    '''
    def get_items(config, doc_url, level=0):
        '''
            @return {
                "file_path": {
                    "curr": (url, label),
                }
            }
        '''
        is_dir = "items" in config
        items = OrderedDict()
        if "label" in config and "file" in config and config["file"] != None and config["file"] != "null":
            file_abs = os.path.join(doc_path, config["file"]).replace("\\", "/")
            url = get_url_by_file_rel(config["file"], doc_url)
            if not os.path.exists(file_abs):
                if redirect_err_file:
                    url_rel = get_url_by_file_rel(config["file"], rel = True)
                    url = f'{redirct_url}?ref={ref_doc_url}{url_rel}&from={url}'
                else:
                    log.w("file {} not found, but set in {} sidebar config file, maybe letter case wrong?".format(file_abs, doc_path))
            items[file_abs] = {
                "curr": (url, config["label"])
            }
            if add_file_item:
                items[file_abs]["file"] = config["file"]
        if is_dir:
            for item in config["items"]:
                _items = get_sidebar_list(item, doc_path, doc_url, log, redirect_err_file = redirect_err_file, redirct_url=redirct_url, ref_doc_url=ref_doc_url)
                items.update(_items)
        return items

    dict_items = get_items(sidebar, doc_url)
    items = list(dict_items.items())
    length = len(items)
    for i, (path, item) in enumerate(items):
        p = None
        n = None
        if i > 0:
            p = items[i - 1][1]["curr"]
        if i < length - 1:
            n = items[i + 1][1]["curr"]
        if not "previous" in item or not item["previous"]:
            item["previous"] = p
        if not "next" in item or not item["next"]:
            item["next"] = n
        dict_items[path]= item
    return dict_items

def generate_sidebar_html(htmls, sidebar, doc_path, doc_url, sidebar_title_html, redirect_err_file=False, redirct_url="", ref_doc_url=""):
    '''
        @htmls  {
                "file1_path": {
                                "title": "",
                                "desc": "",
                                "keywords": [],
                                "body": html
                                }
                }
        @return {
                "file1_path": {
                                "title": "",
                                "desc": "",
                                "keywords": [],
                                "body": html,
                                "sidebar": (sidebar_title, sidebar_items_html)
                                }
                }
    '''
    def generate_items(config, doc_path_relative, doc_url, level=0):
        html = ""
        li = False
        is_dir = "items" in config
        active = False
        li_item_html = ""
        collapsed = False if ("collapsed" in config and config["collapsed"] == False) else True
        if "label" in config:
            if "file" in config and config["file"] != None and config["file"] != "null":
                file_abs = os.path.join(doc_path, config["file"]).replace("\\", "/")
                url = get_url_by_file_rel(config["file"], doc_url)
                if not os.path.exists(file_abs):
                    if redirect_err_file:
                        url_rel = get_url_by_file_rel(config["file"], rel = True)
                        url = f'{redirct_url}?ref={ref_doc_url}{url_rel}&from={url}'
                if config["file"].startswith("./"):
                    active = doc_path_relative.lower() == config["file"][2:].lower()
                else:
                    active = doc_path_relative.lower() == config["file"].lower()
                li_item_html = '<li class="{} with_link"><a href="{}"><span class="label">{}</span><span class="{}"></span></a>'.format(
                    "active" if active else "not_active",
                    url, config["label"],
                    "sub_indicator" if is_dir else ""
                )
            elif "url" in config and config["url"] != None and config["url"] != "null":
                li_item_html = '<li class="{} with_link"><a href="{}" {}><span class="label">{}</span><span class="{}"></span></a>'.format(
                    "not_active",
                    config["url"],
                    'target="{}"'.format(config["target"]) if "target" in config else "",
                    config["label"],
                    "sub_indicator" if is_dir else ""
                )
            elif not is_dir and level == 1:
                # first level label with no url or file and items, add sidebar_category class
                li_item_html = '<li class="not_active no_link sidebar_category"><span class="label">{}</span>'.format(
                    config["label"]
                )
            else:
                li_item_html = '<li class="not_active no_link"><a><span class="label">{}</span><span class="{}"></span></a>'.format(
                    config["label"], "sub_indicator" if is_dir else ""
                )
            li = True
        if is_dir:
            dir_html = ""
            _active = False
            for item in config["items"]:
                item_html, _active_sub = generate_items(item, doc_path_relative, doc_url, level + 1)
                _active |= _active_sub
                dir_html += item_html
            active |= _active
            if _active:
                li_item_html = li_item_html.replace("not_active", 'active_parent')
            elif not active:
                if not collapsed:
                    li_item_html = li_item_html.replace("sub_indicator", "sub_indicator")
                else:
                    li_item_html = li_item_html.replace("sub_indicator", "sub_indicator sub_indicator_collapsed")
            html += li_item_html
            html += '<ul class="{}">\n{}</ul>\n'.format("show" if (active or not collapsed) else "", dir_html)
        else:
            html += li_item_html
        if li:
            html += "</li>\n"
        return html, active


    for file, html in htmls.items():
        if not html:
            continue
        doc_path_relative = file.replace(doc_path, "")[1:].replace("\\", "/")
        items, _ = generate_items(sidebar, doc_path_relative, doc_url)
        # sidebar_html = '''
        #     <div id="sidebar_wrapper">
        #         <div id="sidebar">
        #             <div id="sidebar_title">
        #                 {}
        #             </div>
        #             {}
        #         </div>
        #         <div id="sidebar_splitter">
        #         </div>
        #     </div>'''.format(sidebar_title_html, items)
        html["sidebar"] = (sidebar_title_html, items)
        htmls[file] = html
    return htmls

def generate_navbar_html(htmls, navbar, doc_path, doc_url, plugins_objs):
    '''
        @doc_path  doc path, contain config.json and sidebar.json
        @doc_url   doc url, config in "route" of site_config.json
        @htmls  {
                "file1_path": {
                                "title": "",
                                "desc": "",
                                "keywords": [],
                                "body": html,
                                "sidebar": ""
                                }
                }
        @return {
                "file1_path": {
                                "title": "",
                                "desc": "",
                                "keywords": [],
                                "body": html,
                                "sidebar": "",
                                "navbar": (logo_url, logo_alt, home_url, navbar_title, navbar_main, navbar_options, navbar_plugins)
                                }
                }
    '''
    def generate_items(config, doc_url, page_url, level=0, parent_item_type="link"):
        active_item = None
        have_label = "label" in config
        have_url = "url" in config and config["url"] != None and config["url"] != "null"
        li_html = ""
        active = False
        # item_type: link, list, selection
        item_type = config["type"] if "type" in config else ("selection" if "items" in config else "link")
        if have_label and have_url:
            if not config["url"].startswith("http"):
                if not config["url"].startswith("/"):
                    config["url"] = "/{}".format(config["url"])
            _doc_url = doc_url+"/" if not doc_url.endswith("/") else doc_url
            _config_url = config["url"] + "/" if (not config["url"].endswith(".html") and not config["url"].endswith("/")) else config["url"]
            # print(parent_item_type, _doc_url, _config_url, page_url)
            if _doc_url == "/":
                if page_url == "/index.html" and _config_url == "/":       # / / /index.html
                    active = True
                elif _config_url == page_url:      # / /store.html /store.html
                    active = True
                elif parent_item_type == "selection" and _doc_url == _config_url: # / / /store.html
                    active = True
            else:
                if _config_url != "/" and page_url.startswith(_config_url):
                    active = True
                else:
                    active = _doc_url == _config_url
            if active:
                active_item = config
        sub_items_ul_html = ""
        if "items" in config:
            sub_items_html = ""
            for item in config["items"]:
                item_html, _active_item = generate_items(item, doc_url, page_url, level = level + 1, parent_item_type = item_type)
                if _active_item:
                    active_item = _active_item
                sub_items_html += item_html
            sub_items_ul_html = "<ul>{}</ul>".format(sub_items_html)
        if item_type == "list":
            li_html = '<li class="sub_items {}"><a {}>{}</a>{}\n'.format(
                            "active_parent" if active_item else "",
                            'href="{}"'.format(config["url"]) if have_url else "",
                            "{}".format(config["label"]) if have_label else "",
                            sub_items_ul_html
                        )
        elif item_type == "selection":
            li_html = '<li class="sub_items {}"><a {} href="{}">{}{}</a>{}'.format(
                "active" if active else '',
                'target="{}"'.format(config["target"]) if "target" in config else "",
                config["url"] if have_url else "", config["label"], active_item["label"] if active_item else "",
                sub_items_ul_html
            )
        else: # link
            li_html = '<li class="{}"><a {} href="{}">{}</a>'.format(
                "active" if active else '',
                'target="{}"'.format(config["target"]) if "target" in config else "",
                config["url"] if have_url else "", config["label"]
            )
        html = '{}</li>\n'.format(li_html)
        return html, active_item
    
    def generate_lef_right_items(config, doc_url, page_url):
        left = '<ul id="nav_left">\n'
        right = '<ul id="nav_right">\n'
        for item in config["items"]:
            html, _ = generate_items(item, doc_url, page_url, level = 0)
            if "position" in item and item["position"] == "right":
                right += html
            else:
                left  += html
        left += "</ul>\n"
        right += "</ul>\n"
        return left, right

    for file, html in htmls.items():
        if not html:
            continue
        # get file file url
        url = get_url_by_file_rel(file.replace(doc_path, "")[1:], doc_url)
        navbar_main, navbar_options = generate_lef_right_items(navbar, doc_url, url)
        home_url = navbar["home_url"]
        navbar_title = navbar["title"]
        if "src" in navbar["logo"] and navbar["logo"]["src"]:
            logo_url = navbar["logo"]["src"]
            logo_alt = navbar["logo"]["alt"]
        else:
            logo_url = None
            logo_alt = None

        # add navbar items from plugins
        navbar_plugins = ""
        for plugin in plugins_objs:
            _items = plugin.on_add_navbar_items()
            if not _items:
                continue
            items_html = '<ul class="nav_plugins">'
            for item in _items:
                items_html += "<li>{}</li>".format(item)
            items_html += "</ul>"
            navbar_plugins += items_html
        html["navbar"] = (logo_url, logo_alt, home_url, navbar_title, navbar_main, navbar_options, navbar_plugins)
        htmls[file] = html
    return htmls

def generate_footer_html(htmls, footer, doc_path, doc_url, plugins_objs):
    '''
        @doc_path  doc path, contain config.json and sidebar.json
        @doc_url   doc url, config in "route" of site_config.json
        @htmls  {
                "file1_path": {
                                "title": "",
                                "desc": "",
                                "keywords": [],
                                "body": html,
                                "sidebar": "",
                                "navbar": ""
                                }
                }
        @return {
                "file1_path": {
                                "title": "",
                                "desc": "",
                                "keywords": [],
                                "body": html,
                                "sidebar": "",
                                "navbar": "",
                                "footer": (footer_top, footer_bottom)
                                }
                }
    '''
    def generate_items(config, doc_url, level):
        have_url = "url" in config and config["url"] != None and config["url"] != "null"
        have_label = "label" in config
        li_html = ""
        active = False
        if have_url:
            if (not config["url"].startswith("http") and
                not config["url"].startswith("mailto")):
                if not config["url"].startswith("/"):
                    config["url"] = "/{}".format(config["url"])
            have_url = True
        sub_items_ul_html = ""
        if "items" in config:
            sub_items_html = ""
            for item in config["items"]:
                item_html = generate_items(item, doc_url, level + 1)
                sub_items_html += item_html
            sub_items_ul_html = "<ul>{}</ul>".format(sub_items_html)
        if not have_url:
            li_html = '<li><a>{}</a>{}\n'.format(
                        config["label"] if have_label else "",
                        sub_items_ul_html)
        else:
            li_html = '<li><a {} href="{}">{}</a>{}'.format(
                'target="{}"'.format(config["target"]) if "target" in config else "",
                config["url"], config["label"], sub_items_ul_html
            )
        html = '{}</li>\n'.format(li_html)
        return html
    
    def generate_footer_items(config, doc_url):
        top = '<ul>\n'
        bottom = '<ul>\n'
        for item in config["top"]:
            html = generate_items(item, doc_url, 0)
            top += html
        if "bottom" in config:
            for item in config["bottom"]:
                html = generate_items(item, doc_url, 0)
                bottom += html
        top += "</ul>\n"
        bottom += "</ul>\n"
        return top, bottom

    for file, html in htmls.items():
        if not html:
            continue
        footer_top, footer_bottom = generate_footer_items(footer, doc_url)
        html["footer"] = (footer_top, footer_bottom)
        htmls[file] = html
    return htmls

def construct_html(html_template, html_templates_i18n_dirs, htmls, header_items_in, js_items_in, site_config, sidebar_list, doc_config, doc_src_path, plugins_objs):
    '''
        @htmls  {
            "title": "",
            "desc": "",
            "keywords": ["", ],
            "tags": ["", ],
            "body": "",
            "toc": "", # may not exists
            "sidebar": (sidebar_title, sidebar_items_html),
            "navbar": (logo_url, logo_alt, home_url, navbar_title, navbar_main, navbar_options, navbar_plugins),
            "metadata": {},
            "footer": (footer_top, footer_bottom),
            "show_source": ("Edit this page", source_url), # may not exists
            "date": "2021-3-14", # may not exists
            "author": "", # may not exists
        }
    '''
    template_root = os.path.join(doc_src_path, site_config["layout_root_dir"]) if "layout_root_dir" in site_config else os.path.join(doc_src_path, "layout")
    theme_layout_root = os.path.dirname(html_template)
    locale = doc_config["locale"].replace("-", "_") if "locale" in doc_config else None
    lang = locale.replace("_", "-") if locale else None
    renderer0 = Renderer(os.path.basename(html_template), [theme_layout_root], html_templates_i18n_dirs, locale=locale)
    files = {}
    items = list(htmls.items())
    for i, (file, html) in enumerate(items):
        if not html:
            files[file] = None
        else:
            renderer = renderer0
            metadata = copy.deepcopy(html["metadata"])
            if "title" in metadata:
                metadata.pop("title")
            if "keywords" in metadata:
                metadata.pop("keywords")
            if "desc" in metadata:
                metadata.pop("desc")
            if "tags" in metadata:
                metadata.pop("tags")
            if "id" in metadata:
                metadata.pop("id")
            if "layout" in html["metadata"]:
                layout = os.path.join(template_root, html["metadata"]["layout"])
                if os.path.exists(layout):
                    renderer = Renderer(html["metadata"]["layout"], [template_root, theme_layout_root], html_templates_i18n_dirs, locale=locale)
            id, classes = get_html_start_id_class(html, doc_config["id"] if "id" in doc_config else None, doc_config['class'] if 'class' in doc_config else None)
            if "sidebar" in html:
                previous_article = None
                next_article = None
                if file in sidebar_list:
                    if sidebar_list[file]["previous"]:
                        previous_article = {
                                "url": sidebar_list[file]["previous"][0],
                                "title": sidebar_list[file]["previous"][1]
                            }
                    if sidebar_list[file]["next"]:
                        next_article = {
                                "url": sidebar_list[file]["next"][0],
                                "title": sidebar_list[file]["next"][1]
                            }
                vars = {
                    "lang": lang,
                    "metadata": metadata,
                    "page_id" : id,
                    "page_classes" : classes,
                    "keywords" : html["keywords"],
                    "description" : html["desc"],
                    "header_items" : header_items_in,
                    "title" : html["title"],
                    "site_name" : site_config["site_name"],
                    # navbar
                    # (logo_url, logo_alt, home_url, navbar_title, navbar_main, navbar_options, navbar_plugins),
                    "logo_url" : html["navbar"][0],
                    "logo_alt" : html["navbar"][1],
                    "home_url" : html["navbar"][2],
                    "navbar_title" : html["navbar"][3],
                    "navbar_main" : html["navbar"][4],
                    "navbar_options" : html["navbar"][5],
                    "navbar_plugins" : html["navbar"][6],
                    # sidebar info
                    "sidebar_title" : html["sidebar"][0],
                    "sidebar_items_html" : html["sidebar"][1],
                    # docs body info
                    "article_title" : html["title"],
                    "tags" : html["tags"],
                    "author" : html["author"] if "author" in html else None,
                    "date" : html["date"] if "date" in html else None,
                    "show_source" : html["show_source"][0] if "show_source" in html else None,
                    "source_url" : html["show_source"][1] if "show_source" in html else None,
                    "body" : html["body"],
                    "previous" : previous_article,
                    "next" : next_article,
                    "toc" : html["toc"] if "toc" in html else None,
                    # footer
                    "footer_top" : html["footer"][0],
                    "footer_bottom" : html["footer"][1],
                    "footer_js_items" : js_items_in
                }
                for plugin in plugins_objs:
                    vars = plugin.__getattribute__("on_render_vars")(vars)
                rendered_html = renderer.render(**vars)
            else:
                vars = {
                    "lang": lang,
                    "metadata": metadata,
                    "page_id" : id,
                    "page_classes" : classes,
                    "keywords" : html["keywords"],
                    "description" : html["desc"],
                    "header_items" : header_items_in,
                    "title" : html["title"],
                    "site_name" : site_config["site_name"],
                    # navbar
                    # (logo_url, logo_alt, home_url, navbar_title, navbar_main, navbar_options, navbar_plugins),
                    "logo_url" : html["navbar"][0],
                    "logo_alt" : html["navbar"][1],
                    "home_url" : html["navbar"][2],
                    "navbar_title" : html["navbar"][3],
                    "navbar_main" : html["navbar"][4],
                    "navbar_options" : html["navbar"][5],
                    "navbar_plugins" : html["navbar"][6],
                    # docs body info
                    "body" : html["body"],
                    # footer
                    "footer_top" : html["footer"][0],
                    "footer_bottom" : html["footer"][1],
                    "footer_js_items" : js_items_in
                }
                for plugin in plugins_objs:
                    vars = plugin.__getattribute__("on_render_vars")(vars)
                rendered_html = renderer.render(**vars)
            files[file] = rendered_html
    return files

def update_html_abs_path(file_htmls, root_path):
    def re_del(c):
        content = c[0]
        if content.startswith("src"):
            if content[5] == "/" and content[6] != "/":
                content = "{}{}{}".format(content[:5], root_path[:-1], content[5:])
        elif content.startswith("href"):
            if content[6] == "/" and content[7] != "/": # href="/static/..."
                content = "{}{}{}".format(content[:6], root_path[:-1], content[6:])
        elif content.startswith("url"):
            if content[4] != "/" and content[5] == "/" and content[6] != "/": # url("/static/...")
                content = "{}{}{}".format(content[:5], root_path[:-1], content[5:])
            elif content[4] == "/" and content[5] != "/": # url(/static/...)
                content = "{}{}{}".format(content[:4], root_path[:-1], content[4:])
        return content

    for path in file_htmls:
        if not file_htmls[path]:
            continue
        file_htmls[path] = re.sub(r'href=".*?"', re_del, file_htmls[path])
        file_htmls[path] = re.sub(r'src=".*?"', re_del, file_htmls[path])
        file_htmls[path] = re.sub(r'url\(.*?\)', re_del, file_htmls[path])
    return file_htmls

def add_url_item(htmls, url, dir, site_root_url):
    '''
        will remove empty html items, only return valid html items
        @htmls {
            "file_path":{
                "title": ""
                "body": ""
                "raw": ""
            }
        }
        
        @return {
            "url":{
                "title": ""
                "body": ""
                "raw": ""
                "file_path": ""
            }
        }
    '''
    htmls_valid = {}
    for file_path in htmls:
        if not htmls[file_path]:
            continue
        if not url:
            url_path = site_root_url[:-1]
        else:
            url_path = "{}{}".format(site_root_url, url)
        file_path_rel = file_path.replace(dir, "")
        if file_path_rel.startswith("/"):
            file_path_rel = file_path_rel[1:]
        url_path = get_url_by_file_rel(file_path_rel, url_path)
        htmls_valid[url_path] = htmls[file_path]
        htmls_valid[url_path]["file_path"] = file_path
    return htmls_valid

def get_html_start_id_class(html, id, classes):
    if id:
        id = id.split(" ")[0]
    else:
        id = None
    if classes:
        classes = classes.split(",")
    else:
        classes = []
    if "id" in html["metadata"]:
        id = html["metadata"]["id"].split(" ")[0]
    if "class" in html["metadata"]:
        classes.extend(html["metadata"]["class"].split(","))
    if "" in classes:
        classes.remove("")
    return id, classes

def htmls_add_source(htmls, repo_addr, label, doc_src_path):
    '''
        @htmls {
            "file_path": {
                "metadata": {
                    "show_source": "Edit this page"
                }
            }
        }
        @repo_addr https://github.com/teedoc/teedoc.github.io/blob/main

        @return {
            "file_path": {
                "metadata": {
                    "show_source": "Edit this page"
                },
                "show_source": ("Edit this page", source_url)
            }
        }
    '''
    if not repo_addr:
        return htmls
    for file, v in htmls.items():
        # not show source
        if "show_source" in v["metadata"]:
            show_source = v["metadata"]["show_source"].strip().lower()
            if show_source == "false":
                continue
            elif show_source != "true":
                label = v["metadata"]["show_source"]
        source_url = repo_addr
        if source_url.endswith("/"):
            source_url = source_url[:-1]
        source_url += file.replace(doc_src_path, "")
        htmls[file]["show_source"] = (label, source_url)

    return htmls

def generate_return(plugins_objs, ok):
    for p in plugins_objs:
        p.on_new_process_del()
    return ok

def generate(html_template, html_templates_i18n_dirs, files, url, dir, doc_config, plugin_func, routes,
             site_config, doc_src_path, log, out_dir, plugins_objs, header_items, js_items,
             sidebar, sidebar_list, allow_no_navbar, site_root_url, navbar, footer, queue, pipe_rx, pipe_tx,
             redirect_err_file, redirct_url, ref_doc_url):
    if not pipe_tx is None:
        def on_err():
            pipe_tx.send(True)

        def is_err():
            if pipe_rx.poll():
                pipe_rx.recv()
                return True
            return False
    else:
        is_err_flag = False
        def on_err():
            global is_err_flag
            is_err_flag = True

        def is_err():
            return is_err_flag

    try:
        # call init in new process
        for p in plugins_objs:
            p.on_new_process_init()

        if url.startswith("/"):
            rel_url = url[1:]
        else:
            rel_url = url
        out_path = os.path.join(out_dir, rel_url)
        in_path  = os.path.join(doc_src_path, dir)
        if in_path.endswith("/"):
            in_path = in_path[:-1]
        if out_path.endswith("/"):
            out_path = out_path[:-1]
        # call plugins to parse files
        result_htmls = {}
        for plugin in plugins_objs:
            # parse file content
            result = plugin.__getattribute__(plugin_func)(files)
            if result:
                if not result['ok']:
                    log.e("plugin <{}> {} error: {}".format(plugin.name, plugin_func, result['msg']))
                    on_err()
                    return generate_return(plugins_objs, False)
                else:
                    for key in result['htmls']:
                        if result['htmls'][key]:
                            result_htmls[key] = result['htmls'][key] # will cover the before
            if is_err():
                return generate_return(plugins_objs, False)
        # copy not parsed files
        for path in files:
            if not path in result_htmls:
                copy_file(path, path.replace(in_path, out_path))
        # no file parsed, just return
        if not result_htmls:
            log.d("parse files empty: {}".format(files))
            # on_err()
            return generate_return(plugins_objs, True)
            
        htmls = result_htmls
        # generate sidebar to html
        if sidebar:
            htmls = generate_sidebar_html(htmls, sidebar, dir, url, sidebar["title"] if "title" in sidebar else "",
                                        redirect_err_file=redirect_err_file, redirct_url=redirct_url, ref_doc_url=ref_doc_url)
        if is_err():
            return generate_return(plugins_objs, False)
        # generate navbar to html
        if navbar:
            htmls = generate_navbar_html(htmls, navbar, dir, url, plugins_objs)
        if footer:
            htmls = generate_footer_html(htmls, footer, dir, url, plugins_objs)
        if is_err():
            return generate_return(plugins_objs, False)
        # show source code url
        if "source" in site_config:
            label = None
            if not "show_source" in doc_config:
                label = "Edit this page"
            elif doc_config["show_source"]:
                label = doc_config["show_source"]
            if label:
                htmls = htmls_add_source(htmls, site_config["source"], label, doc_src_path)

        # consturct html page
        htmls_str = construct_html(html_template, html_templates_i18n_dirs, htmls, header_items, js_items, site_config, sidebar_list, doc_config, doc_src_path, plugins_objs)
        if is_err():
            return generate_return(plugins_objs, False)
        # check abspath
        if site_root_url != "/":
            htmls_str = update_html_abs_path(htmls_str, site_root_url)
        if is_err():
            return generate_return(plugins_objs, False)
        # write to file
        ok, msg = write_to_file(htmls_str, in_path, out_path)
        if not ok:
            log.e("write files error: {}".format(msg))
            on_err()
            return generate_return(plugins_objs, False)
        if is_err():
            return generate_return(plugins_objs, False)
        # add url, add "url" keyword for htmls, will remove empty html items
        htmls = add_url_item(htmls, rel_url, dir, site_root_url)
        if len(htmls) > 0:
            queue.put((url, htmls))
    except Exception as e:
        log.e("generate html fail: {}".format(e))
        on_err()
        raise e
        return generate_return(plugins_objs, False)
    log.i("generate ok")
    return generate_return(plugins_objs, True)

def parse(type_name, plugin_func, routes, site_config, doc_src_path, config_template_dir, log, out_dir, plugins_objs, sidebar, allow_no_navbar, update_files, max_threads_num, preview_mode,
            translate = False, ref_doc_url="", ref_doc_dir = "", translate_src_sidebar_list = None):
    '''
        @return {
            "doc_url", {
                "page_url": {
                    "title": "",
                    "desc": "",
                    "keywords": "",
                    "tasg": [],
                    "body": "",
                    "toc": "",
                    "raw": "",
                    "sidebar": "",
                    "navbar": "",
                    "metadata": {},
                    "footer": ""
                }
            }
        }
    '''
    manager = multiprocessing.Manager()
    queue = manager.Queue()
    site_root_url = site_config["site_root_url"]
    # parse all docs in route
    for url, _dir in routes.items():
        if not url.startswith("/"):
            url = "{}{}".format("/", url)
        dir = os.path.abspath(os.path.join(doc_src_path, _dir)).replace("\\", "/")
        if not os.path.exists(dir):
            log.w("dir {} not exists!!!".format(dir))
            continue
        # get files
        if update_files:
            all_files = []
            for modify_file in update_files:
                if modify_file.startswith(dir):
                    all_files.append(modify_file)
            if len(all_files) == 0:
                continue
            log.i("update file:", all_files)
        else:
            all_files = get_files(dir, warn = log.w)
        # load doc config
        log.i("parse {}: {}, url:{}".format(type_name, dir, url))
        doc_config = load_doc_config(dir, config_template_dir)
        if not "locale" in doc_config:
            locale = "en"
            log.w(f'locale of <{_dir}> not set, value can be "zh" "zh_CN" "en" "en_US" etc.')
        else:
            locale = doc_config["locale"]
            log.i("locale: {}".format(doc_config["locale"]))
        # inform plugin parse doc start
        try:
            plugins_new_config = doc_config['plugins']
        except Exception as e:
            plugins_new_config = {}
        for plugin in plugins_objs:
            if plugin.name in plugins_new_config:
                new_config = plugins_new_config[plugin.name]["config"]
            else:
                new_config = {}
            plugin.on_parse_start(type_name, doc_config, new_config)
        # get header footer items, and template dir
        # get html header item from plugins
        header_items = []
        footer_js_items = []
        #     get html template from plugins
        html_template = None
        #     get html template i18n dir
        html_templates_i18n_dirs = []
        for plugin in plugins_objs:
            items = plugin.on_add_html_header_items(type_name)
            _js_items = plugin.on_add_html_footer_js_items(type_name)
            if type(items) != list or type(_js_items) != list:
                log.e("plugin <{}> error, on_add_html_header_items should return list type".format(plugin.name))
                return False, None
            if items:
                header_items.extend(items)
            if _js_items:
                footer_js_items.extend(_js_items)
            temp = plugin.on_html_template(type_name)
            if temp and os.path.exists(temp):
                html_template = temp
            temp = plugin.on_html_template_i18n_dir(type_name)
            if temp and os.path.exists(temp):
                html_templates_i18n_dirs.append(temp)
        if not html_template:
            log.e("no html templates for {}, please install theme plugin".format(type_name))
            return False
        log.i("html_templates_i18n_dirs: {}".format(html_templates_i18n_dirs))

        # preview_mode js file
        if preview_mode:
            footer_js_items.append('<script type="text/javascript" src="{}static/js/live.js"></script>'.format(site_config['site_root_url']))

        # get sidebar config
        sidebar_dict = {}
        if sidebar is True:
            try:
                sidebar_dict = get_sidebar(dir, config_template_dir)
            except Exception as e:
                log.e("parse sidebar.json fail: {}".format(e))
                return False, None
        elif sidebar:
            sidebar_dict = sidebar
        try:
            navbar = doc_config['navbar']
        except Exception as e:
            if not allow_no_navbar:
                log.e("parse config.json navbar fail: {}".format(e))
                return False, None
            navbar = None
        try:
            footer = doc_config['footer']
        except Exception as e:
            footer = None
        redirect_err_file = translate,
        redirct_url=f"{url}no_translate.html"
        if sidebar_dict:
            sidebar_list = get_sidebar_list(sidebar_dict, dir, url, log,
                                redirect_err_file = redirect_err_file,
                                redirct_url=redirct_url,
                                ref_doc_url = ref_doc_url)
            if translate:
                check_sidebar_diff(translate_src_sidebar_list, sidebar_list, ref_doc_url, url, ref_doc_dir, dir, doc_src_path, log)
        else:
            sidebar_list = {}
        if max_threads_num > 1 and len(all_files) > 10:
            all_files = split_list(all_files, max_threads_num)
            ts = []
            pipe_rx, pipe_tx = multiprocessing.Pipe()
            for files in all_files:
                p = multiprocessing.Process(target=generate,
                                            args=(html_template, html_templates_i18n_dirs, files, url, dir, doc_config, plugin_func,
                                            routes, site_config, doc_src_path, log, out_dir, plugins_objs, 
                                            header_items, footer_js_items, sidebar_dict, sidebar_list, allow_no_navbar,
                                            site_root_url, navbar, footer, queue, pipe_rx, pipe_tx,
                                            redirect_err_file, redirct_url, ref_doc_url))
                p.start()
                ts.append(p)
            for p in ts:
                p.join()
                # log.i("{} generate ok".format(p.name))
        else:
            ok = generate(html_template, html_templates_i18n_dirs, all_files, url, dir, doc_config, plugin_func,
                          routes, site_config, doc_src_path, log, out_dir, plugins_objs, header_items,
                          footer_js_items, sidebar_dict, sidebar_list, allow_no_navbar, site_root_url, navbar, footer, queue, None, None,
                          redirect_err_file, redirct_url, ref_doc_url)
            if not ok:
                return False, None
        # create no_translate.html
        if translate:
            temp = os.path.join(dir, "no_translate.html")
            if not os.path.exists(temp) and not os.path.exists(os.path.join(dir, "no_translate.md")):
                root_dir = os.path.abspath(os.path.dirname(__file__))
                try:
                    lang = gettext.translation('messages', localedir=os.path.join(root_dir, 'locales'), languages=[locale])
                    lang.install()
                    _ = lang.gettext
                    no_translate_title = _("no_translate_title")
                    no_translate_hint = _("no_translate_hint")
                    visit_hint = _("visit_hint")
                except Exception as e:
                    no_translate_hint = "This page not translated yet"
                    visit_hint = "Please visit"
                    no_translate_title = "no translation"
                with open(os.path.join(root_dir, "templates", "no_translate.md"), "r") as f:
                    content = f.read().replace('<div id="no_translate_hint"></div>', f'<div id="no_translate_hint">{no_translate_hint}</div>').replace(
                        '<span id="visit_hint"></span>', f'<span id="visit_hint">{visit_hint}</span>').replace(
                            "no_translate_title", no_translate_title
                        )
                dir = tempfile.gettempdir()
                all_files = [os.path.join(dir, "no_translate.md")]
                with open(all_files[0], "w") as f:
                    f.write(content)
                ok = generate(html_template, html_templates_i18n_dirs, all_files, url, dir, doc_config, plugin_func,
                          routes, site_config, doc_src_path, log, out_dir, plugins_objs, header_items,
                          footer_js_items, sidebar_dict, sidebar_list, allow_no_navbar, site_root_url, navbar, footer, queue, None, None,
                          redirect_err_file, redirct_url, ref_doc_url)
                if not ok:
                    return False, None

    htmls = {}
    for i in range(queue.qsize()):
        url, _htmls = queue.get()
        if not url in htmls:
            htmls[url] = {}
        htmls[url].update(_htmls)
    for plugin in plugins_objs:

        plugin.on_parse_end()
    return True, htmls

def build(doc_src_path, config_template_dir, plugins_objs, site_config, out_dir, log, update_files=None, preview_mode = False, max_threads_num = 1):
    '''
        "route": {
            "docs": {
                "/get_started/zh": "docs/get_started/zh",
                "/get_started/en": "docs/get_started/en",
                "/develop/zh": "docs/develop/zh",
                "/develop/en": "docs/develop/en"
            },
            "pages": {
                "/": "pages/index/zh",
                "/en": "pages/index/en"
            },
            "/blog": "blog"
        }
    '''
    # parse all docs
    routes = site_config["route"]["docs"]
    ok, htmls_files = parse("doc", "on_parse_files", routes, site_config, doc_src_path, config_template_dir, log, out_dir, plugins_objs,
                 sidebar=True, allow_no_navbar=False, update_files=update_files, max_threads_num=max_threads_num, preview_mode=preview_mode)
    if not ok:
        return False

    # parse all pages
    routes = site_config["route"]["pages"]
    ok, htmls_pages = parse("page", "on_parse_pages", routes, site_config, doc_src_path, config_template_dir, log, out_dir, plugins_objs,
                 sidebar=False, allow_no_navbar=True, update_files=update_files, max_threads_num=max_threads_num, preview_mode=preview_mode)
    if not ok:
        return False
    # parse all blogs
    htmls_blog = None
    if "blog" in site_config["route"]:
        routes = site_config["route"]["blog"]
        ok, htmls_blog = parse("blog", "on_parse_blog", routes, site_config, doc_src_path, config_template_dir, log, out_dir, plugins_objs,
                    sidebar={"items":[]}, allow_no_navbar=True, update_files=update_files, max_threads_num=max_threads_num, preview_mode=preview_mode)
        if not ok:
            return False
    
    # parse all translate docs
    if "translate" in site_config:
        if "docs" in site_config["translate"]:
            docs_translates = site_config["translate"]["docs"]
            routes = {}
            for src in docs_translates:
                for dst in docs_translates[src]:
                    # check_sidebar_diff(os.path.join(doc_src_path, site_config["route"]["docs"][src]), os.path.join(doc_src_path, dst["src"]), log)
                    routes[dst["url"]] = dst["src"]
                src_dir = os.path.join(doc_src_path, site_config["route"]["docs"][src]).replace("\\", "/")
                sidebar_dict = get_sidebar(src_dir, config_template_dir) # must be success
                sidebar_list = get_sidebar_list(sidebar_dict, src_dir, src, log)
                #    pase mannually translated files, and change links of sidebar items that no mannually translated file
                ok, htmls_files = parse("doc", "on_parse_files", routes, site_config, doc_src_path, config_template_dir, log, out_dir, plugins_objs,
                            sidebar=True, allow_no_navbar=False, update_files=update_files, max_threads_num=max_threads_num, preview_mode=preview_mode,
                            translate=True, ref_doc_url=src, ref_doc_dir=src_dir, translate_src_sidebar_list = sidebar_list)
                #    create
                if not ok:
                    return False
        if "pages" in site_config["translate"]:
            docs_translates = site_config["translate"]["pages"]
            routes = {}
            for src in docs_translates:
                for dst in docs_translates[src]:
                    # check_sidebar_diff(os.path.join(doc_src_path, site_config["route"]["docs"][src]), os.path.join(doc_src_path, dst["src"]), log)
                    routes[dst["url"]] = dst["src"]
                #    pase mannually translated files, and change links of sidebar items that no mannually translated file
                ok, htmls_files = parse("page", "on_parse_pages", routes, site_config, doc_src_path, config_template_dir, log, out_dir, plugins_objs,
                            sidebar=False, allow_no_navbar=True, update_files=update_files, max_threads_num=max_threads_num, preview_mode=preview_mode,
                            translate=True, ref_doc_url=src, ref_doc_dir=src_dir)
                #    create
                if not ok:
                    return False

    # generate sitemap.xml
    if not update_files: # only generate when build all
        sitemap_out_path = os.path.join(out_dir, "sitemap.xml")
        generate_sitemap(htmls_files, sitemap_out_path, site_config["site_domain"], site_config["site_protocol"], log)

    # send all htmls to plugins
    for plugin in plugins_objs:
        ok = plugin.on_htmls(htmls_files = htmls_files, htmls_pages = htmls_pages, htmls_blog = htmls_blog)
        if not ok:
            return False

    # copy assets
    log.i("copy assets files")
    assets = site_config["route"]["assets"]
    for target_dir, from_dir in assets.items(): 
        in_path  = os.path.join(doc_src_path, from_dir)
        if target_dir.startswith("/"):
            target_dir = target_dir[1:]
        out_path = os.path.join(out_dir, target_dir).replace("\\", "/")
        if update_files:
            for file in update_files:
                if file.startswith(in_path):
                    log.i("update assets file: {}".format(file))
                    in_path = file.replace(in_path+"/", "")
                    out_path = os.path.join(out_path, in_path)
                    log.i("copy", file, out_path)
                    if not copy_file(file, out_path):
                        log.w("copy {} to {} fail".format(file, out_path))
        else:
            if not copy_dir(in_path, out_path):
                return False
    # copy files from pulgins
    log.i("copy assets files of plugins")
    for plugin in plugins_objs:
        files = plugin.on_copy_files()
        for dst,src in files.items():
            if dst.startswith("/"):
                dst = dst[1:]
            dst = os.path.join(out_dir, dst)
            if not os.path.isabs(src):
                log.e("plugin <{}> on_copy_files error, file path {} must be abspath".format(plugin.name, src))
            if not copy_file(src, dst):
                log.e("copy plugin <{}> file {} to {} error".format(plugin.name, src, dst))
                return False
    if not update_files:
        # preview mode js
        if preview_mode:
            js_out_dir = os.path.join(out_dir, "static/js")
            curr_dir_path = os.path.dirname(os.path.abspath(__file__))
            copy_file(os.path.join(curr_dir_path, "static", "js", "live.js"), os.path.join(js_out_dir, "live.js"))
    return True

def files_watch(doc_src_path, log, delay_time, queue):
    from watchdog.observers import Observer
    from watchdog.events import RegexMatchingEventHandler
    import time
 
    class FileEventHandler(RegexMatchingEventHandler):
        def __init__(self, doc_src_path):
            ignore = "{}/out/.*".format(doc_src_path)
            RegexMatchingEventHandler.__init__(self, ignore_regexes=[r".*out.*", r".*/.git/.*"])
            self.update_files = []
            self.doc_src_path = doc_src_path
            self.lock = threading.Lock()
        
        def get_update_files(self):
            self.lock.acquire()
            self.update_files = list(set(self.update_files))  # remove dumplicated files
            files = self.update_files
            if files:
                self.update_files = []
            self.lock.release()
            return files
        
        def _append_files(self, files):
            self.lock.acquire()
            self.update_files.extend(files)
            self.lock.release()

        def on_moved(self, event):
            if event.is_directory:
                # print("directory created:{0}".format(event.src_path))
                pass
            else:
                print("file moved:{0}".format(event.dest_path))
                files = [os.path.abspath(os.path.join(self.doc_src_path, event.dest_path)).replace("\\", "/")]
                self._append_files(files)
        
        def on_created(self, event):
            if event.is_directory:
                log.d("directory created:{0}".format(event.src_path))
                pass
            else:
                log.d("file created:{0}".format(event.src_path))
                files = [os.path.abspath(os.path.join(self.doc_src_path, event.src_path)).replace("\\", "/")]
                self._append_files(files)
        
        def on_deleted(self, event):
            pass

        def on_modified(self, event):
            if event.is_directory:
                # print("directory modified:{0}".format(event.src_path))
                pass
            else:
                log.d("file modified:{0}".format(event.src_path))
                files = [os.path.abspath(os.path.join(self.doc_src_path, event.src_path)).replace("\\", "/")]
                self._append_files(files)
 
    observer = Observer()
    handler = FileEventHandler(doc_src_path)
    observer.schedule(handler, doc_src_path, True)
    observer.start()
    try:
        while True:
            time.sleep(delay_time)
            update_files = handler.get_update_files()
            if update_files:
                log.i("file changes detected:", update_files)
                queue.put(update_files)
    except KeyboardInterrupt:
        observer.stop()
        observer.join()



def main():
    parser = argparse.ArgumentParser(prog="teedoc", description="teedoc, a doc generator, generate html from markdown and jupyter notebook\nrun 'teedoc install && teedoc serve'")
    parser.add_argument("-d", "--dir", default=".", help="doc source root path" )
    parser.add_argument("-f", "--file", type=str, default="", help="file path for json2yaml or yaml2json command")
    parser.add_argument("-p", "--preview", action="store_true", default=False, help="preview mode, provide live preview support for build command, serve command always True" )
    parser.add_argument("-t", "--delay", type=int, default=-1, help="automatically rebuild and refresh page delay time")
    parser.add_argument("-v", "--version", action="version", version="%(prog)s v{}".format(__version__))
    parser.add_argument("-i", "--index-url", type=str, default="", help="for install command, base URL of the Python Package Index (default https://pypi.org/simple). This should point to a repository compliant with PEP 503 (the simple repository API) or a local directory laid out in the same format.\ne.g. Chinese can use https://pypi.tuna.tsinghua.edu.cn/simple")
    parser.add_argument("-log", "--log-level", type=str, default="i", choices=["d", "i", "w", "e"], help="log level")
    parser.add_argument("--thread", type=int, default=0, help="how many threads use to building, default 0 will use max CPU supported")
    parser.add_argument("--host", type=str, default="0.0.0.0", help="host address for serve command")
    parser.add_argument("--port", type=int, default=2333, help="port for serve command")
    parser.add_argument("command", choices=["install", "init", "build", "serve", "json2yaml", "yaml2json", "summary2yaml", "summary2json"])
    args = parser.parse_args()

    log = Logger(level=args.log_level)
    # convert json or yaml file
    if args.command == "json2yaml":
        if not os.path.exists(args.file):
            log.e("file {} not found".format(args.file))
            return 1
        with open(args.file, encoding="utf-8") as f:
            obj = json.load(f)
            yaml_str = yaml.dump(obj, allow_unicode=True, indent=4, sort_keys=False)
            yaml_path = "{}.yaml".format(os.path.splitext(args.file)[0])
            with open(yaml_path, "w", encoding="utf-8") as f2:
                f2.write(yaml_str)
            log.i("convert yaml from json complete, file at: {}".format(yaml_path))
        return 0
    elif args.command == "yaml2json":
        if not os.path.exists(args.file):
            log.e("file {} not found".format(args.file))
            return 1
        with open(args.file, encoding="utf-8") as f:
            obj = yaml.load(f.read(), Loader=yaml.Loader)
            json_path = "{}.json".format(os.path.splitext(args.file)[0])
            with open(json_path, "w", encoding="utf-8") as f2:
                json.dump(obj, f2, ensure_ascii=False, indent=4)
            log.i("convert json from yaml complete, file at: {}".format(json_path))
        return 0
    elif args.command == "summary2json":
        if not os.path.exists(args.file):
            log.e("file {} not found".format(args.file))
            return 1
        with open(args.file, encoding="utf-8") as f:
            obj = sidebar_summary2dict(f.read())
            json_path = os.path.join(os.path.dirname(args.file), "sidebar.json")
            with open(json_path, "w", encoding="utf-8") as f2:
                json.dump(obj, f2, ensure_ascii=False, indent=4)
            log.i("convert json from gitbook summary complete, file at: {}".format(json_path))
        return 0
    elif args.command == "summary2yaml":
        if not os.path.exists(args.file):
            log.e("file {} not found".format(args.file))
            return 1
        with open(args.file, encoding="utf-8") as f:
            obj = sidebar_summary2dict(f.read())
            yaml_str = yaml.dump(obj, allow_unicode=True, indent=4, sort_keys=False)
            yaml_path = os.path.join(os.path.dirname(args.file), "sidebar.yaml")
            with open(yaml_path, "w", encoding="utf-8") as f2:
                f2.write(yaml_str)
            log.i("convert yaml from gitbook summary complete, file at: {}".format(yaml_path))
        return 0
    elif args.command == "init":
        log.i("init doc now")
        if not os.path.exists(args.dir):
            os.makedirs(args.dir, exist_ok=True)
        if os.listdir(args.dir):
            log.e("directory {} not empty, please init in empty directory".format(args.dir))
            return 1
        template_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "template")
        for name in os.listdir(template_path):
            path = os.path.join(template_path, name)
            to_path = os.path.join(args.dir, name)
            if os.path.isfile(path):
                shutil.copyfile(path, to_path)
            else:
                shutil.copytree(path, to_path)
        log.i("init doc complete, doc root: {}".format(args.dir))
        return 0
    t = None
    t2 = None
    while 1: # for rebuild all files
        plugins_objs = []
        try:
            # doc source code root path
            doc_src_path = os.path.abspath(args.dir).replace("\\", "/")
            # parse site config
            ok, site_config = parse_site_config(doc_src_path)
            if not ok:
                log.e(site_config)
                return 1
            if "config_template_dir" in site_config:
                config_template_dir = os.path.abspath(os.path.join(doc_src_path, site_config["config_template_dir"]))
            else:
                config_template_dir = doc_src_path
            # out_dir
            if site_config["site_root_url"] != "/":
                serve_dir = os.path.join(doc_src_path, "out").replace("\\", "/")
                out_dir = os.path.join(serve_dir, site_config["site_root_url"][1:]).replace("\\", "/")
            else:
                serve_dir = os.path.join(doc_src_path, "out").replace("\\", "/")
                out_dir = serve_dir
            # thread num
            if args.thread > 0:
                max_threads_num = args.thread
            else:
                max_threads_num = multiprocessing.cpu_count()
            log.i("max thread number: {}".format(max_threads_num))
            if args.command in ["build", "serve"]:
                # init plugins
                plugins = list(site_config['plugins'].keys())
                log.i("plugins: {}".format(plugins))
                for plugin, info in site_config['plugins'].items():
                    try:
                        plugin_config = info['config']
                    except Exception:
                        plugin_config = {}
                    # import plugin from local source code
                    path = info["from"]
                    if not os.path.isabs(path):
                        path = os.path.abspath(os.path.join(doc_src_path, path))
                    if os.path.exists(path):
                        sys.path.insert(0, path)
                    plugin_import_name = plugin.replace("-", "_")
                    module = __import__(plugin_import_name)
                    plugin_obj = module.Plugin(doc_src_path=doc_src_path, config=plugin_config, site_config=site_config, logger=log)
                    plugin_obj.module_path = os.path.abspath(os.path.dirname(module.__file__))
                    plugins_objs.append(plugin_obj)
            # execute command
            if args.command == "install":
                log.i("install, source doc root path: {}".format(doc_src_path))
                log.i("plugins: {}".format(list(site_config["plugins"].keys())))
                curr_path = os.getcwd()
                for plugin, info in site_config['plugins'].items():
                    path = info['from']
                    # install from pypi.org
                    if not path or path.lower() == "pypi":
                        if "version" in info:
                            plugin = f"{plugin}=={info['version']}"
                        log.i("install plugin <{}> from pypi.org".format(plugin))
                        if args.index_url:
                            cmd = [site_config["executable"]["pip"], "install", "--upgrade", plugin, "-i", args.index_url]
                        else:
                            cmd = [site_config["executable"]["pip"], "install", "--upgrade", plugin]
                        p = subprocess.Popen(cmd, shell=False)
                        p.communicate()
                        if p.returncode != 0:
                            log.e("install <{}> fail".format(plugin))
                            return 1
                        log.i("install <{}> complete".format(plugin))
                    # install from git like: git+https://github.com/Neutree/COMTool.git#egg=comtool
                    elif path.startswith("svn") or path.startswith("git"):
                        log.i("install plugin <{}> from {}".format(plugin, path))
                        cmd = [site_config["executable"]["pip"], "install", "-e", path]
                        log.i("install <{}> by pip: {}".format(plugin, " ".join(cmd)))
                        p = subprocess.Popen(cmd, shell=False)
                        p.communicate()
                        if p.returncode != 0:
                            log.e("install <{}> fail".format(plugin))
                            return 1
                        log.i("install <{}> complete".format(plugin))
                    # install from local file system
                    else:
                        if not os.path.isabs(path):
                            path = os.path.abspath(os.path.join(doc_src_path, path))
                        if not os.path.exists(path):
                            log.e("{} not found".format(path))
                            return 1
                        os.chdir(path)
                        cmd = [site_config["executable"]["pip"], "install", "."]
                        log.i("plugin path: {}".format(path))
                        log.i("install <{}> by pip: {}".format(plugin, " ".join(cmd)))
                        p = subprocess.Popen(cmd, shell=False)
                        p.communicate()
                        if p.returncode != 0:
                            log.e("install <{}> fail".format(plugin))
                            return 1
                        log.i("install <{}> complete".format(plugin))
                os.chdir(curr_path)
                log.i("all plugins install complete")
            elif args.command == "build":
                # parse files
                if not build(doc_src_path, config_template_dir, plugins_objs, site_config=site_config, out_dir=out_dir, log=log, preview_mode=args.preview, max_threads_num=max_threads_num):
                    return 1
                add_robots_txt(site_config, out_dir, log)
                log.i("build ok")
            elif args.command == "serve":
                from http.server import SimpleHTTPRequestHandler
                from http import HTTPStatus

                if not build(doc_src_path, config_template_dir, plugins_objs, site_config=site_config, out_dir=out_dir, log=log, preview_mode=True, max_threads_num=max_threads_num):
                    return 1
                add_robots_txt(site_config, out_dir, log)
                log.i("build ok")

                host = (args.host, args.port)
                
                class On_Resquest(SimpleHTTPRequestHandler):

                    def __init__(self, *args, **kwargs):
                        super().__init__(*args, directory=serve_dir, **kwargs)

                    def do_GET(self):
                        file_path = self.path[1:].split("?")[0]
                        if not file_path:
                            file_path = "index.html"
                        file_path = os.path.join(serve_dir, file_path)
                        if not os.path.exists(file_path) or not os.path.isfile(file_path):
                            file_path = os.path.join(file_path, "index.html")
                        if not os.path.exists(file_path):
                            file_path = os.path.join(out_dir, "404.html")
                            self.send_response(404)
                        else:
                            self.send_response(200)
                        if not os.path.exists(file_path):
                            content = b"page not found"
                            content_type = "text/html"
                        else:
                            with open(file_path, "rb") as f:
                                content = f.read()
                            content_type = get_content_type_by_path(file_path)
                        self.send_header('Content-type', content_type)
                        self.end_headers()
                        self.wfile.write(content)
                        # print(self.address_string())
                        # print(self.request)

                    def do_HEAD(self):
                        f = self.send_head()
                        if f:
                            f.close()
                    
                    def log_request(self, code='-', size='-'):
                        if isinstance(code, HTTPStatus):
                            code = code.value
                        if code == 304 or code == 200 or code == 301:
                            return
                        self.log_message('"%s" %s %s',
                                        self.requestline, str(code), str(size))
                                    
                    def log_message(self, format, *args):
                        sys.stderr.write("%s - - [%s] %s\n" %
                                        (self.address_string(),
                                        self.log_date_time_string(),
                                        format%args))
                if not t:
                    queue = Queue(maxsize=50)
                    delay_time = (int(site_config["rebuild_changes_delay"]) if "rebuild_changes_delay" in site_config else 3) if int(args.delay) < 0 else int(args.delay)
                    t = threading.Thread(target=files_watch, args=(doc_src_path, log, delay_time, queue))
                    t.setDaemon(True)
                    t.start()
                    def server_loop(host, log):
                        server = HTTP_Server(host, On_Resquest)
                        log.i("root dir: {}".format(serve_dir))
                        log.i("Starting server at {}:{} ....".format(host[0], host[1]))
                        if host[0] == "0.0.0.0":
                            log.i("You can visit http://127.0.0.1:2333{}".format(site_config["site_root_url"]))
                        server.serve_forever()
                    if not t2:
                        t2 = threading.Thread(target=server_loop, args=(host, log))
                        t2.setDaemon(True)
                        t2.start()
                while 1:
                    try:
                        files_changed = queue.get(timeout=1)
                    except Empty:
                        continue
                    # detect config.json or site_config.json change, if changed, update all docs file along with the json file
                    files = []
                    for path in files_changed:
                        # if path.replace(doc_src_path, "")
                        ext = os.path.splitext(path)
                        if ".sw" in ext:
                            log.w("ingnore {} temp file".format(path))
                            continue
                        dir = os.path.dirname(path)
                        if path[:-5].endswith("site_config"): # site_config changed, rebuild all
                            raise RebuildException()
                        elif config_template_dir == dir:      # config template changed, just rebuild all
                            raise RebuildException()
                        elif path[:-5].endswith("config") or path[:-5].endswith("sidebar"):    # doc or pages config or sidebar changed, rebuild the changed doc
                            files.extend(get_files(dir))
                        else:                                 # normal file, nonly rebuild this file
                            files.append(path)
                    if files:
                        if not build(doc_src_path, config_template_dir, plugins_objs, site_config=site_config, out_dir=out_dir, log=log, update_files = files, preview_mode=True, max_threads_num=max_threads_num):
                            return 1
                        log.i("rebuild ok")
                t.join()
                t2.join()
            else:
                log.e("command error")
                return 1
        except RebuildException:
            continue
        break
    return 0



if __name__ == "__main__":
    python_version = sys.version_info
    if python_version.major < 3 or python_version.minor < 7:
        print('''only support python 3.7 or higher, now python version: {}.{}, please upgrade
or use miniconda to create a virtual env by:
                                            "conda create -n 3.9 python=3.9"
                                            "conda activate 3.9"'''.format(python_version.major, python_version.minor))
        sys.exit(1)
    ret = main()
    sys.exit(ret)
