import copy
import copy
from report_ranger.errors import InputError
import os
from pathlib import Path
import re
from jinja2 import Template
import jinja2
from jinja2.runtime import Macro
from jinja2.runtime import Macro
import yaml
import json
from report_ranger.imports.imports import import_csv, import_xlsx
import logging
from report_ranger.imports import section
from report_ranger.helpers import make_list
from report_ranger.imports import vulnerability
import traceback
import datetime
try:
    import tomllib
    hastomlib = True
except ModuleNotFoundError:
    # Error handling
    hastomlib = False
    pass

from report_ranger.utils.jinja_helpers import log_jinja2_error

log = logging.getLogger(__name__)

yamlmatch = re.compile(r'^[\.\=\-]{3}$')
jsonmatch = re.compile(r'^\;\;\;$')


def process_included_header(headers, includedheaders):
    """ Process the included headers using an overlay routine:
    - Don't overwrite values
    - Add new values if they're not there
    - Append to lists
    """
    if isinstance(includedheaders, dict):
        if not isinstance(headers, dict):
            return headers
        for header in includedheaders:
            if header in headers:
                header = process_included_header(
                    headers[header], includedheaders[header])
            else:
                headers[header] = includedheaders[header]
        return headers
    elif isinstance(includedheaders, list):
        if not isinstance(headers, dict):
            return headers
        headers.extend(includedheaders)
        return headers
    else:
        return headers


def process_template(headers, markdown, env=None, name='', filename=''):
    try:
        template = Template(markdown)  # Start jinja2
        env.push(headers)  # Add our new headers in
        output = template.render(env.get_env())
        env.pop()  # Take away the headers

    except jinja2.exceptions.TemplateSyntaxError as error:
        log.error(f"Error in processing {name} at {filename}")
        log.error(f"Message: {error.message}. Line number: {error.lineno}.")
        log_jinja2_error(markdown, error)
        raise Exception(f"Error reading {name}: {error.message} at lineno {error.lineno}")

    return output

def _get_resultant_path(file:str, filemapper:dict={}, pathlist:list=[]):
    # Try CWD
    filepath = Path(file)
    if filepath.exists():
        return filepath
    
    # Try includemapper
    if file in filemapper:
        file = filemapper[file]
        return Path(file)
    
    # Try pathlist
    if len(pathlist) > 0:
        for path in pathlist:
            filepath = Path(path) / Path(file)
            if Path(path) not in filepath.parents:
                log.warn(f"Path {filepath} seems to be going up in directories, skipping")
                raise Exception("Could not get resultant path")
            if filepath.exists():
                return filepath
    raise Exception("Could not find file")

def _process_includes(headers, env=None, includemapper={}, pathlist=[], watcher=None):
    for included in make_list(headers['include']):
        log.info(f"Including headers from file {included}")
        try:
            included = _get_resultant_path(included, includemapper, pathlist)
            includedheaders, _ = markdown_from_file(included, pathlist=pathlist, env=env, watcher=watcher)
            headers = process_included_header(headers, includedheaders)
        except Exception as e:
            log.warn(f"Could not include headers from file {included}: {e.args}")
    return headers

def _process_include_markdown(markdown, includemapper, pathlist, watcher=None):
    try:
        included = _get_resultant_path(markdown, includemapper, pathlist)
        log.debug(f"Processing included markdown")
        _, included_markdown = markdown_from_file(included, pathlist=pathlist, process_imports=False, process_includes=False, watcher=watcher)
        return included_markdown
    except Exception as e:
        log.warn(f"Could not include headers from file {included}: {e.args}")

def _check_import_directory(importdict):
    """ Run the standard checks for directory imports including vulnerabilities and sections
    """
    if 'variable' not in importdict or not isinstance(importdict['variable'], str):
        raise Exception("Import item does not have variable name")
    if not isinstance(importdict, dict):
        raise Exception(f"Import item '{importdict['variable']}' in front matter not a dict")
    if 'directory' not in importdict or not isinstance(importdict['directory'], str):
        raise Exception(f"Import item '{importdict['variable']}' does not have a directory")
    return True

def _process_imports(headers, env=None, watcher=None, includemapper:dict={}, pathlist:dict={}):
    new_headers = {}
    if not 'import' in headers:
        return {}
    if not isinstance(headers['import'], dict):
        log.warn("import front matter variable not a dict, skipping")
    else:
        if 'vulnerabilities' in headers['import']:
            vulnerabilities = make_list(headers['import']['vulnerabilities'])
            for vulnerabilitiesimport in vulnerabilities:
                if vulnerabilitiesimport['variable'] in headers:
                    log.warn(
                        f"Vulnerabilities import variable name {vulnerabilitiesimport['variable']} already in front matter, skipping")
                    continue
                
                try:
                    _check_import_directory(vulnerabilitiesimport)
                except Exception as e:
                    log.warn(f"Error importing vulnerabilities: {e.args}")

                vulndir = vulnerabilitiesimport['directory']
                log.info("Vulnerability directory included: {}".format(vulndir))

                vulnerabilitylist = vulnerability.VulnerabilityList()

                try:
                    # Get the vulnerability validation (if it exists) from the env
                    vulnerability_validation = {}
                    if 'validation' in vulnerabilitiesimport:
                        validation = vulnerabilitiesimport['validation']
                        if validation is dict and 'vulnerability' in validation:
                            vulnerability_validation = validation['vulnerability']

                    if env == None:
                        log.warn("Exception raising because env is None in _process_imports vulnerability import")
                        raise Exception("env is None")

                    vulnerabilitylist.add_from_dir(
                        vulndir, env, env.get("ra"), vulnerability_validation=vulnerability_validation, watcher=watcher)

                    if 'updated_date' in vulnerabilitiesimport:
                        updated_date = vulnerabilitiesimport['updated_date']
                        if updated_date == 'today':
                            updated_date = datetime.date.today()
                        log.debug(f"Updating vulnerabilities to {updated_date}")
                        log.debug(f"Vulnerabilities: {vulnerabilitylist}")
                        vulnerabilitylist = vulnerabilitylist.updated(updated_date)
                        log.debug(f"Updated vulnerabilities: {vulnerabilitylist}")
                    
                    log.debug(f"Setting vulnerabilities as {vulnerabilitylist}")
                    new_headers[vulnerabilitiesimport['variable']] = vulnerabilitylist

                    # Generate vulnerability markdown
                    vulnerabilitylist.generate_markdown(env)
                except FileNotFoundError as e:
                    log.warn(e.args[0])
                except Exception as e:
                    log.warn(f"Error loading vulnerabilities import {vulnerabilitiesimport['variable']}: {e.args}")
                    log.warn(traceback.format_exc())
            
        if 'sections' in headers['import']:
            sectionlist = make_list(headers['import']['sections'])
            for sectionimport in sectionlist:
                if not isinstance(sectionimport, dict):
                    log.warn(
                        "Section import item in front matter not a dict, skipping")
                    continue
                if 'directory' not in sectionimport or not isinstance(sectionimport['directory'], str):
                    log.warn(
                        "Section import item does not have a directory, skipping")
                    continue
                if 'variable' not in sectionimport or not isinstance(sectionimport['variable'], str):
                    log.warn(
                        "Section import item does not have variable name, skipping")
                    continue
                if sectionimport['variable'] in headers:
                    log.warn(
                        "Section import variable name already in front matter, skipping")
                    continue
                if 'ordinal' in sectionimport:
                    ordinal = sectionimport['ordinal']
                else:
                    ordinal = '1'

                # Build the section
                newsection = section.Section(
                    sectionimport['directory'], ordinal, watcher=watcher)

                new_headers[sectionimport['variable']] = newsection.get_chapters(env)

        if 'toml' in headers['import']:
            tomllist = make_list(headers['import']['toml'])
            if not hastomlib:
                log.warn("Cannot import TOML as we cannot import hastomlib")
            for tomlimport in tomllist:
                if not isinstance(tomlimport, str):
                    log.warn(
                        "TOML import item in front matter not a dict, skipping")
                    continue

                try:
                    with open(tomlimport, "rb") as f:
                        data = tomllib.load(f)
                        new_headers.update(data)
                        if watcher:
                            watcher.add_path(tomlimport)
                except Exception as e:
                    log.warn(f"Could not load toml file {tomlimport}: {e.args}")
        # Begin CSV imports
        if 'csv' in headers['import']:
            csvlist = make_list(headers['import']['csv'])
            for csvimport in csvlist:
                if not isinstance(csvimport, dict):
                    log.warn(
                        "csv import item in front matter not a dict, skipping")
                    continue
                if 'file' not in csvimport or not isinstance(csvimport['file'], str):
                    log.warn(
                        "csv import item does not have file location, skipping")
                    continue
                if 'variable' not in csvimport or not isinstance(csvimport['variable'], str):
                    log.warn(
                        "csv import item does not have variable name, skipping")
                    continue
                if csvimport['variable'] in headers:
                    log.warn(
                        "csv import variable name already in front matter, skipping")
                    continue
                if 'as_dict_list' in csvimport and not isinstance(csvimport['as_dict_list'], bool):
                    log.warning(
                        'csv import defines "as_dict_list", but it is not a boolean')
                    continue
                if 'index_col' in csvimport and not isinstance(csvimport['index_col'], int):
                    log.warning(
                        'csv import defines "index_col", but it is not an integer')
                    continue

                new_headers[csvimport['variable']] = import_csv(csvimport['file'], as_dict_list=csvimport.get(
                    'as_dict_list'), index_col=csvimport.get('index_col'))
                if watcher:
                    watcher.add_path(csvimport['file'])
        # Begin xlsx imports
        if 'xlsx' in headers['import']:
            xlsxlist = make_list(headers['import']['xlsx'])
            for xlsximport in xlsxlist:
                if not isinstance(xlsximport, dict):
                    log.warn(
                        "xlsx import item in front matter not a dict, skipping")
                    continue
                if 'file' not in xlsximport or not isinstance(xlsximport['file'], str):
                    log.warn(
                        "xlsx import item does not have file location, skipping")
                    continue
                if 'variable' not in xlsximport or not isinstance(xlsximport['variable'], str):
                    log.warn(
                        "xlsx import item does not have variable name, skipping")
                    continue
                if xlsximport['variable'] in headers:
                    log.warn(
                        "xlsx import variable name already in front matter, skipping")
                    continue
                if 'as_dict_list' in xlsximport and not isinstance(xlsximport['as_dict_list'], bool):
                    log.warning(
                        'xlsx import defines "as_dict_list", but it is not a boolean')
                    continue
                if 'index_col' in xlsximport and not isinstance(xlsximport['index_col'], int):
                    log.warning(
                        'xlsx import defines "index_col", but it is not an integer')
                    continue

                new_headers[xlsximport['variable']] = import_xlsx(
                    xlsximport['file'],
                    xlsximport.get('worksheet'),
                    xlsximport.get('min_row'),
                    xlsximport.get('max_row'),
                    xlsximport.get('min_col'),
                    xlsximport.get('max_col'),
                    as_dict_list=xlsximport.get('as_dict_list'),
                    index_col=xlsximport.get('index_col')
                )
                if watcher:
                    watcher.add_path(xlsximport['file'])

        if 'macros' in headers['import']:
            macrolist = make_list(headers['import']['macros'])
            for macrofile in macrolist:
                fn, ext = os.path.splitext(macrofile)
                if ext not in ['.md', '.j2', '.jinja', '.rr']:
                    log.warn(f"Macro file {macrofile} not of the right extension. The extension is {ext} but should be one of md, j2, jinja, or rr")
                    continue
                try:
                    macrofile = _get_resultant_path(macrofile, includemapper, pathlist)
                except:
                    log.warn(f"Could not find {macrofile}")
                    continue

                try:
                    mdheaders, markdown = markdown_from_file(macrofile, env=env, watcher=watcher)
                except:
                    log.warn(f"Could not read macro file {macrofile}")
                    continue
                mdheaders['cwd'] = os.path.dirname(
                    os.path.join(os.path.curdir, macrofile))

                try:
                    jenv = jinja2.Environment()
                    # Add in the env to the globals. We want the existing globals to take precedence
                    globals = env.get_env()
                    globals.update(jenv.globals)
                    jenv.globals = globals
                    # Get the macro template from the markdown
                    macro_template = jenv.from_string(markdown)
                    # Copy over the macros and set variables into the new headers
                    for f in [method for method in dir(macro_template.module) if method.startswith('_') is False]:
                        k = getattr(macro_template.module, f)
                        if isinstance(k, Macro):
                            log.debug(f"Setting macro {k.name}")
                            new_headers[k.name] = k
                        elif isinstance(k, str) or isinstance(k, int):
                            log.debug(f"Setting variable {f} as {k}")
                            new_headers[f] = k
                except jinja2.exceptions.TemplateSyntaxError as error:
                    log.error("Jinja2 error processing macro file {}: {} at lineno {} for file {}".format(
                        macrofile, error.message, error.lineno, filename=error.filename))
                    log.error("Skipping import of macro file {}".format(macrofile))
                    log_jinja2_error(markdown, error)
                    log.error(traceback.format_exc())
                    return ""
                except Exception as e:
                    log.error(
                        "Received exception when processing markdown for macro file {}: {}".format(macrofile, e.args))
                    log.error(
                        "Skipping import of macro file {}".format(macrofile))
                    log.error(traceback.format_exc())
                    return ""

    return new_headers


def markdown_from_file(file_loc, env=None, includemapper={}, process_includes=True, process_imports=True, pathlist=[], watcher=None):
    """ Read the markdown from a file.

    Arguments:
    - file_loc: The location of the file to read
    - env: The environment in its current context. The stack for this environment will be extended.
    - process_includes: Whether or not to process the include header, allowing the including of headers and markdown from other files.
    - process_imports: Whether or not to process the import header, allowing importing of sections, vulns, csv and xlsx files.
    - watcher: if a watcher is configured then add_path will automatically be called
    
    - watcher: if a watcher is configured then add_path will automatically be called
    
    Returns (headers, markdown) where headers is a dict and markdown is a long string """

    log.debug(f"Reading markdown from file {file_loc}")

    if not os.path.exists(file_loc):
        log.warn(f"Trying to read file {file_loc} but it doesn't exist")
        raise InputError("Could not read markdown file.")

    if watcher:
        watcher.add_path(file_loc)

    with open(file_loc, 'r') as vf:
        # If the first line starts with "---" that's our headers We have to make sure that it starts with "---"
        firstline = vf.readline()
        headers = dict()
        if yamlmatch.match(firstline):
            line = vf.readline()
            headerstring = ''
            # Read the headers.
            # Go until we get to the terminator to end the headers
            while not yamlmatch.match(line):
                headerstring += line
                line = vf.readline()
                if not line:
                    raise InputError(
                        f"Markdown file {file_loc} YAML headers don't end in a YAML terminator (...,---,===)")
            headers = yaml.safe_load(headerstring)  # Retrieve the YAML headers
            if headers == None:
                headers = {}
            markdown = vf.read()  # Read the rest of the file
        elif jsonmatch.match(firstline):
            line = vf.readline()
            headerstring = ''
            # Read the headers.
            # Go until we get to the terminator to end the headers
            while not jsonmatch.match(line):
                headerstring += line
                line = vf.readline()
                if not line:
                    raise InputError(
                        f"Markdown file {file_loc} JSON headers don't end in a JSON terminator (;;;)")
            headers = json.loads(headerstring)  # Retrieve the JSON headers
            markdown = vf.read()  # Read the rest of the file
        else:
            # There's no headers in this file. Put the first line back and just add it all into the markdown
            markdown = firstline + '\n' + vf.read()

        # Begin post processing. Process includes and imports.

        # Process includes in headers
        if process_includes and 'include' in headers:
            headers = _process_includes(headers, includemapper=includemapper, pathlist=pathlist, env=env)
        
        # Process include_markdown
        if process_includes and 'include_markdown' in headers:
            markdown = _process_include_markdown(headers['include_markdown'], includemapper, pathlist)

        # Process imports in headers
        if process_imports and 'import' in headers:
            if env == None:
                log.warn("Importing stopped because passed environment is None")
            else:
                log.info(f"Processing imports in file {file_loc}")
                headers.update(_process_imports(headers, env, watcher=watcher, pathlist=pathlist, includemapper=includemapper))

        return headers, markdown


def markdown_from_directory(dir_loc, env=None, watcher=None):
    log.debug("Reading markdown from directory '{}'".format(dir_loc))

    if not os.path.isdir(dir_loc):
        raise FileNotFoundError("Directory '{0}' doesn't exist. Skipping import of files from the directory.".format(
            dir_loc))

    files = []

    for f in os.listdir(dir_loc):
        mdfile = os.path.join(dir_loc, f)
        if os.path.isfile(mdfile):
            dir, filename = os.path.split(mdfile)
            fn, ext = os.path.splitext(filename)
            if ext == ".md" or ext == ".rr":
                try:
                    mdfromfile=markdown_from_file(mdfile, env, watcher=watcher)
                    mdfromfile[0]["filename"] = filename
                    mdfromfile[0]["filename_noext"] = fn
                    mdfromfile[0]["filename_full"] = mdfile
                    files.extend([mdfromfile])
                except:
                    log.warn(f"Could not read file {mdfile}")
                    continue
                

    return files
