# =============================================================================
# GenerateTipueIndex
#
# Written by Bryan K. Ressler
# (c) 2018 Institute for Disease Modeling
#
# Usage:
#  pythong gti.py [-i <sourceDir>] [-o <indexFileJs>] [-u <urlPrefix>] [-v]
#
# Where:
#   sourceDir   The source directory containing HTML files
#   indexFileJs The output file path
#   urlPrefix   The prefix for URLs in the output file
#   -v          Verbose output, for diagnostic purposes
#
# Example
#  python gti.py -i ../Documentation/malaria/_build/html/ \
#    -o ../Documentation/malaria/_build/html/_static/tipuesearch/tipuesearch_content.js
#    -u 'https://www.idmod.org/malaria'
#
# Required libraries
#   pip install beautifulsoup
#   pip install argparse
# =============================================================================

# =============================================================================
import urllib
import re
from bs4 import BeautifulSoup
import argparse
import glob
import sys
import os.path
import json
from datetime import datetime


# -----------------------------------------------------------------------------
def process_file(file):
    """Read and process the HTML of the given HTML file. Returns an object for
    the output record."""
    try:
        f = open(file, encoding="utf-8")
        html = f.readlines()
    except BaseException as ex:
        print("Exception processing file {0}: {1}", file, str(ex))
        return None
    html = " ".join(html)
    soup = BeautifulSoup(html, "html.parser")

    # kill all script and style elements
    for script in soup(["script", "style"]):
        script.extract()  # rip it out

    # find the title
    title = soup.head.find('title').get_text()

    # get text
    doc_div = soup.body.find_all("div", class_="document")
    if len(doc_div) > 0:
        # print(str.format("len(doc_civ) = {}", len(doc_div)))
        text = doc_div[0].get_text(separator=' ')
        text = text.replace("\n", "")
        text = re.sub('\s+', " ", text)
        text = text.strip()
    else:
        print(str.format("Skipping {} because no div.document was found.",
                         file), file=sys.stderr)
        text = ""

    # prepare output record
    result = {
        "title": title,
        "tags": "",  # no tags
        "url": str.format("{0}/{1}", args.url, os.path.basename(file)),
        "text": str(text)
    }
    return result


# =============================================================================
# Main program
# =============================================================================

# -----------------------------------------------------------------------------
# Parse command line arguments
# -----------------------------------------------------------------------------
parser = argparse.ArgumentParser(
    description='Generate Tipue Search indexes from HTML files.')
parser.add_argument('-i', '--input', help='Input directory name',
                    default='./', required=False)
parser.add_argument('-o', '--output',
                    help='Output Tipue Search index file, typically named tipuesearch_content.js',
                    default='./tipuesearch_content.js', required=False)
parser.add_argument('-u', '--url', help='Prefix for output URLs',
                    default='', required=False)
parser.add_argument('-v', '--verbose', help='Emit diagnostic messages',
                    default=False, required=False, action="store_true")
args = parser.parse_args()

# -----------------------------------------------------------------------------
# Get the list of HTML files
# -----------------------------------------------------------------------------
files = glob.glob(
    (args.input if args.input[-1] == '/' else args.input + '/') + '*.html')
count = len(files)
if count == 0:
    print(str.format('Error: no HTML files found at {}', args.input), file=sys.stderr)
    sys.exit()

if args.verbose:
    print(str.format("{} html files found.", len(files)))

# -----------------------------------------------------------------------------
# Read and extract the text from each HTML file, and append a record for it
# to our output file.
# -----------------------------------------------------------------------------
output = {
    "pages": []
}
for file in files:
    if args.verbose:
        print(str.format('Processing {}', file))
    record = process_file(file)
    if record is not None:
        output["pages"].append(record)
# print(record)
# print(output)
out_str = str.format(
    "var tipuesearch_info = 'Generated by gti.py on {0}';\nvar tipuesearch = {1};\n",
    datetime.now(), json.dumps(output))
# print(out_str)
f = open(args.output, "w")
f.write(out_str)
f.close()
print(str.format("Output content written to {}", args.output))
