import logging
import os
from enum import Enum
from pathlib import Path
import uuid

import lxml.etree as ET

from lxml import objectify

from eatb import ROOT
from eatb.checksum import ChecksumValidation, ChecksumAlgorithm
from eatb.metadata.mets_generator import MetsGenerator
from eatb.metadata.parsed_mets import ParsedMets

from eatb.metadata.premis_generator import PremisGenerator
from eatb.packaging import create_package
from eatb.csip_validation import ValidationResult, XmlValidation
from eatb.file_format import FormatIdentification

import hashlib
from mimetypes import MimeTypes

from subprocess import Popen, PIPE

from eatb.utils.XmlHelper import XSI_NS, q
from eatb.utils.datetime import get_file_ctime_iso_date_str, DT_ISO_FMT_SEC_PREC, current_timestamp


LOGGER = logging.getLogger(__name__)


class OAISPackageType(Enum):
    SIP = 1
    AIP = 2
    DIP = 3
    IP = 4


def create_sip(package_dir: str, package_name: str, identifier: str, generate_premis: bool=True,
               generate_package: bool=True, custom_logger=None) -> bool:

    logger = custom_logger if custom_logger else LOGGER

    # PREMIS
    premis_path = os.path.join(package_dir, 'metadata/preservation/premis.xml')
    if generate_premis:
        premisgen = PremisGenerator(package_dir)

        premisgen.createPremis()
        premisinfo = {'outcome': 'success',
                      'task_name': 'create_sip',
                      'event_type': 'Package initialisation',
                      'linked_object': None}

        logger.info("Creating PREMIS file %s" % premis_path)
        premisgen.addEvent('metadata/preservation/premis.xml', premisinfo)

    reps_path = os.path.join(package_dir, 'representations')
    for name in os.listdir(reps_path):
        rep_path = os.path.join(reps_path, name)
        if os.path.isdir(rep_path):
            if generate_premis:
                # representation PREMIS
                logger.info(
                    "Creating representation PREMIS file %s" %
                    os.path.join(rep_path, 'metadata/preservation/premis.xml'))
                premisgen = PremisGenerator(rep_path)
                premisgen.createPremis()
            # representation METS
            logger.info("Creating representation METS file: %s" % os.path.join(rep_path, "METS.xml"))
            mets_data = {'packageid': package_name,
                         'type': "SIP",
                         'schemas': os.path.join(package_dir, 'schemas'),
                         'parent': ''}
            metsgen = MetsGenerator(rep_path)

            metsgen.createMets(mets_data)
    if generate_premis:
        # PREMIS
        premisgen = PremisGenerator(package_dir)
        premisgen.createPremis()

    # METS
    mets_data = {'packageid': package_name,
                 'type': "SIP",
                 'schemas': os.path.join(package_dir, 'schemas'),
                 'parent': ''}
    mets_path = os.path.join(package_dir, "METS.xml")
    logger.info("Creating METS file %s" % mets_path)
    metsgen = MetsGenerator(package_dir)
    metsgen.createMets(mets_data)

    # packaging
    input_path = Path(package_dir)
    storage_tar_file = os.path.join(input_path.parent, package_name + '.tar')
    if generate_package:
        logger.info("Creating package: %s" % storage_tar_file)
        create_package(package_dir, package_name)

    if not os.path.exists(mets_path):
        logger.error("Error creating METS file %s" % mets_path)
    premis_success = (not generate_package or os.path.exists(premis_path))
    if not premis_success:
        logger.error("Error creating PREMIS file %s" % premis_path)
    package_success = (not generate_package or os.path.exists(storage_tar_file))
    if not package_success:
        logger.error("Error creating package %s" % storage_tar_file)

    return (
        os.path.exists(mets_path)
        and
        premis_success
        and
        package_success
    )



logger = logging.getLogger(__name__)


def create_aip(package_dir, identifier, package_name, identifier_map=None, generate_premis=True,
               generate_package=True) -> bool:

    # schema file location for Mets generation
    schemas = os.path.join(ROOT, 'resources/schemas')

    # for every REPRESENTATION without METS file:
    for repdir in os.listdir(os.path.join(package_dir, 'representations')):
        rep_path = os.path.join(package_dir, 'representations/%s' % repdir)
        # TODO: packageid?
        # parent = task_context.additional_data['parent_id']
        mets_data = {'packageid': repdir,
                     'type': "AIP",
                     'schemas': schemas,
                     'parent': ''}
        metsgen = MetsGenerator(rep_path)
        metsgen.createMets(mets_data)

        logger.info('Generated a Mets file for representation %s.' % repdir)

    #########

    # schema file location for Mets generation
    schemas = os.path.join(package_dir, 'schemas')

    mets_data = {'packageid': identifier,
                 'type': "AIP",
                 'schemas': schemas,
                 'parent': None}

    metsgen = MetsGenerator(package_dir)
    metsgen.createMets(mets_data)

    # TODO: get structMap from submission root METS, translate SIP packagename to
    # AIP UUID and add it to the AIP root METS

    # get identifier_map
    if identifier_map:
        # check if packagename is in the identifier map
        if package_name in identifier_map:
            identifier = identifier_map[package_name]
            logger.info("Provided identifier assigned: %s" % identifier)
        else:
            logger.error("Cannot find uuid for package name %s in provided identifier_map" % package_name)


        METS_NS = 'http://www.loc.gov/METS/'
        XLINK_NS = "http://www.w3.org/1999/xlink"

        # parse submission SIP mets
        parser = ET.XMLParser(resolve_entities=False, remove_blank_text=True, strip_cdata=False)
        sip_mets_path = os.path.join(package_dir, 'submission', 'METS.xml')
        # print sip_mets_path
        sip_parse = ET.parse(sip_mets_path, parser)
        sip_root = sip_parse.getroot()

        # parse package AIP METS and append children and parent structMaps
        aip_mets_path = os.path.join(package_dir, 'METS.xml')
        aip_parse = ET.parse(aip_mets_path, parser)
        aip_root = aip_parse.getroot()

        # get children structMap and replace the urn:uuid:packagename with corresponding urn:uuid:uuid
        children_map = sip_root.find("%s[@LABEL='child %s']" % (q(METS_NS, 'structMap'), 'SIP'))
        if children_map is not None:
            children_map.set('LABEL', 'child AIP')
            children_div = children_map.find("%s[@LABEL='child %s identifiers']" % (q(METS_NS, 'div'), 'SIP'))
            children_div.set('LABEL', 'child AIP identifiers')
            children = children_div.findall("%s[@LABEL='child %s']" % (q(METS_NS, 'div'), 'SIP'))
            for child in children:
                child.set('LABEL', 'child AIP')
                mptr = child.find("%s" % q(METS_NS, 'mptr'))
                urn = mptr.get(q(XLINK_NS, 'href'))
                urn = urn.split('urn:uuid:', 1)[1]
                uuid = identifier_map[urn]
                mptr.set(q(XLINK_NS, 'href'), 'urn:uuid:' + uuid)
                mptr.set(q(XLINK_NS, 'title'), 'Referencing a child AIP.')
            aip_root.append(children_map)
        # get parents structMap and replace the urn:uuid:packagename with corresponding urn:uuid:uuid
        parents_map = sip_root.find("%s[@LABEL='parent %s']" % (q(METS_NS, 'structMap'), 'SIP'))
        if parents_map is not None:
            parents_map.set('LABEL', 'parent AIP')
            parents_div = parents_map.find("%s[@LABEL='parent %s identifiers']" % (q(METS_NS, 'div'), 'SIP'))
            parents_div.set('LABEL', 'parent AIP identifiers')
            parents = parents_div.findall("%s[@LABEL='parent %s']" % (q(METS_NS, 'div'), 'SIP'))
            for parent in parents:
                parent.set('LABEL', 'parent AIP')
                mptr = parent.find("%s" % q(METS_NS, 'mptr'))
                urn = mptr.get(q(XLINK_NS, 'href'))
                urn = urn.split('urn:uuid:', 1)[1]
                uuid = identifier_map[urn]
                mptr.set(q(XLINK_NS, 'href'), 'urn:uuid:' + uuid)
                mptr.set(q(XLINK_NS, 'title'), 'Referencing a parent AIP.')
            aip_root.append(parents_map)

        str = ET.tostring(aip_root, encoding='UTF-8', pretty_print=True, xml_declaration=True)
        with open(aip_mets_path, 'w') as output_file:
            output_file.write(str)

    # packaging
    input_path = Path(package_dir)
    storage_tar_file = os.path.join(input_path.parent, identifier + '.tar')
    if generate_package:
        logger.info("Creating package: %s" % storage_tar_file)
        create_package(package_dir, identifier)

    mets_path = os.path.join(package_dir, "METS.xml")
    if not os.path.exists(mets_path):
        logger.error("Error creating METS file %s" % mets_path)
    else:
        logger.info('Generated a root Mets file: %s.' % mets_path)

    package_success = (not generate_package or os.path.exists(storage_tar_file))
    if not package_success:
        logger.error("Error creating package %s" % storage_tar_file)

    return (
        os.path.exists(mets_path)
        and
        package_success
    )


class DeliveryValidation(object):

    def getFileElements(self, deliveryDir, delivery_xml_file, schema_file):
        log = []
        log.append("Validating delivery: %s using schema: %s" % (delivery_xml_file, schema_file))
        try:
            # Parse the XML file, get the root element
            parsed_mets = ParsedMets(deliveryDir)
            parsed_mets.load_mets(delivery_xml_file)
            # If the XSD file wasn't found, extract location from the XML
            if schema_file == None:
                schema_file = parsed_mets.get_mets_schema_from_schema_location()
            # Parse the XSD file
            parsed_sfile = ET.parse(schema_file)
            # Validate the delivery XML file
            xmlVal = XmlValidation()
            validation_result = xmlVal.validate_XML(parsed_mets.mets_tree, parsed_sfile)
            if validation_result:
                return parsed_mets.get_file_elements()
        except ET.XMLSyntaxError as why:
            logger.error('Error validating delivery %s, why: %s' % (delivery_xml_file, str(why)))
        return None

    def validate_delivery(self, deliveryDir, checksum_file, schema_file, package_file):
        """
        Validate the delivery document.
        level.

        @type       deliveryDir: string
        @param      deliveryDir: Path to delivery directory
        @type       delivery_xml_file:  string
        @param      delivery_xml_file:  Path to delivery file.
        @type       package_file:  string
        @param      package_file:  Path to package file (e.g. TAR).
        @rtype:     ValidationResult
        @return:    Validation result (validity, process log, error log)
        """
        valid = False
        log = []
        err = []
        valid_checksum = False
        # TODO: define
        checksum_algorithm = ""
        # Checksum validation
        self.get_checksum()
        checksum_expected = self.get_checksum(deliveryDir, checksum_algorithm)
        csval = ChecksumValidation()
        valid_checksum = csval.validate_checksum(package_file, checksum_expected, ChecksumAlgorithm.get(checksum_algorithm))
        log.append("Checksum validity: \"%s\"" % str(valid_checksum))
        valid = (valid_checksum)
        return ValidationResult(valid, log, err)


METS_NS = 'http://www.loc.gov/METS/'
METSEXT_NS = 'ExtensionMETS'
PREMIS_NS = 'http://www.loc.gov/premis/v3'
PREMIS_NSMAP = {None: PREMIS_NS}
XLINK_NS = "http://www.w3.org/1999/xlink"
METS_NSMAP = {None: METS_NS, "xlink": "http://www.w3.org/1999/xlink", "ext": METSEXT_NS,
                  "xsi": "http://www.w3.org/2001/XMLSchema-instance"}
DELIVERY_METS_NSMAP = {None: METS_NS, "xlink": "http://www.w3.org/1999/xlink",
                           "xsi": "http://www.w3.org/2001/XMLSchema-instance"}
P = objectify.ElementMaker(
        annotate=False,
        namespace=PREMIS_NS,
        nsmap=PREMIS_NSMAP)
M = objectify.ElementMaker(
    annotate=False,
    namespace=METS_NS,
    nsmap=METS_NSMAP)


class SIPGenerator():
    fid = FormatIdentification()
    mime = MimeTypes()
    root_path = ""

    def __init__(self, root_path):
        self.root_path = root_path

    def sha256(self, fname):

        sha256_hash = hashlib.sha256()
        with open(fname, "rb") as f:
            # Read and update hash string value in blocks of 4K
            for byte_block in iter(lambda: f.read(4096), b""):
                sha256_hash.update(byte_block)
            return sha256_hash.hexdigest()

    def createAgent(self,role, type, other_type, name, note):
        if other_type:
            agent = M.agent({"ROLE":role,"TYPE":type, "OTHERTYPE": other_type}, M.name(name), M.note(note))
        else:
            agent = M.agent({"ROLE":role,"TYPE":type}, M.name(name), M.note(note))
        return agent

    def runCommand(self, program, stdin = PIPE, stdout = PIPE, stderr = PIPE):
        result, res_stdout, res_stderr = None, None, None
        try:
            # quote the executable otherwise we run into troubles
            # when the path contains spaces and additional arguments
            # are presented as well.
            # special: invoking bash as login shell here with
            # an unquoted command does not execute /etc/profile

            print('Launching: ' + ' '.join(program))
            process = Popen( program, stdin = stdin, stdout = stdout, stderr = stderr, shell = False)

            res_stdout, res_stderr = process.communicate()
            result = process.returncode
            print('Finished: ' + ' '.join(program))

        except Exception as ex:
            res_stderr = ''.join(str(ex.args))
            result = 1

        if result != 0:
            print('Command failed:' + ''.join(res_stderr))
            raise Exception('Command failed:' + ''.join(res_stderr))

        return result, res_stdout, res_stderr

    def addFile(self, file_name, mets_filegroup):
        #reload(sys)
        #sys.setdefaultencoding('utf8')
        file_url = "%s" % os.path.relpath(file_name, self.root_path)
        file_mimetype,_ = self.mime.guess_type(file_url)
        file_mimetype = file_mimetype if file_mimetype else "application/octet-stream"
        file_checksum = self.sha256(file_name)
        file_size = os.path.getsize(file_name)
        file_cdate = get_file_ctime_iso_date_str(file_name, DT_ISO_FMT_SEC_PREC)
        file_id = "ID"+uuid.uuid4().__str__()
        mets_file = M.file({"MIMETYPE":file_mimetype, "CHECKSUMTYPE":"SHA-256", "CREATED":file_cdate, "CHECKSUM":file_checksum, "USE":"Datafile", "ID":file_id, "SIZE":file_size})
        mets_filegroup.append(mets_file)
        #_,fname = os.path.split(file_name)
        mets_FLocat = M.FLocat({q(XLINK_NS, 'href'): file_url, "LOCTYPE":"URL", q(XLINK_NS, 'type'): 'simple'})
        mets_file.append(mets_FLocat)
        return file_id

    def addFiles(self, folder, mets_filegroup):
        ids = []
        for top, dirs, files in os.walk(folder):
            for fn in files:
                file_name = os.path.join(top, fn)
                file_id = self.addFile(file_name,mets_filegroup)
                ids.append(file_id)
        return ids

    # def createPremis(self, enable_jhove = False):
    #     jhove_parser = None
    #     if enable_jhove == True:
    #         jhove_parser = etree.XMLParser(remove_blank_text=True)
    #
    #     PREMIS_ATTRIBUTES = {"version" : "2.0"}
    #     premis = P.premis(PREMIS_ATTRIBUTES)
    #     premis.attrib['{%s}schemaLocation' % XSI_NS] = "http://www.loc.gov/premis/v3 ../../schemas/premis-v3-0.xsd"
    #
    #     premis_ids = []
    #     for top, dirs, files in os.walk(os.path.join(self.root_path, 'data')):
    #         for nm in files:
    #             file_name = os.path.join(top,nm)
    #             hash = self.sha256(file_name)
    #             file_url = "./%s" % os.path.relpath(file_name, self.root_path)
    #             fmt = self.fid.identify_file(file_name)#os.path.abspath(remove_protocol(file_url)))
    #             jhove = None
    #             if enable_jhove == True:
    #                 try:
    #                     result = self.runCommand(["/usr/bin/jhove", "-h", "xml", file_name] )
    #                     if result[0] == 0:
    #                         jhove = etree.XML(result[1], parser=jhove_parser)
    #                 except Exception:
    #                     #TODO: handle exception
    #                     pass
    #
    #             size = os.path.getsize(file_name)
    #             premis_id = uuid.uuid4()
    #             premis_ids.append(premis_id)
    #             premis.append(
    #                 P.object(
    #                     {q(XSI_NS, 'type'): 'file', "xmlID":premis_id},
    #                     P.objectIdentifier(
    #                         P.objectIdentifierType('LOCAL'),
    #                         P.objectIdentifierValue(premis_id)
    #                     ),
    #                     P.objectIdentifier(
    #                         P.objectIdentifierType('FILEPATH'),
    #                         P.objectIdentifierValue(file_url)
    #                     ),
    #                     P.objectCharacteristics(
    #                         P.compositionLevel(0),
    #                         P.size(size),
    #                         P.fixity(
    #                             P.messageDigestAlgorithm("SHA-256"),
    #                             P.messageDigest(hash),
    #                             P.messageDigestOriginator("hashlib")
    #                         ),
    #                         P.format(
    #                             P.formatRegistry(
    #                                 P.formatRegistryName("PRONOM"),
    #                                 P.formatRegistryKey(fmt),
    #                                 P.formatRegistryRole("identification")
    #                             )
    #                         ),
    #                         #P.objectCharacteristicsExtension(
    #                             #TODO:// generate id or reference from somewhere
    #                         #    P.mdSec({"ID":"ID426087e8-0f79-11e3-847a-34e6d700c47b"},
    #                         #        P.mdWrap({"MDTYPE":"OTHER", "OTHERMDTYPE":"JHOVE"},
    #                         #            P.xmlData(
    #                         #                jhove
    #                         #                 )
    #                         #                 )
    #                         #)
    #                     ),
    #                 )
    #             )
    #
    #     identifier_value = 'earkweb'
    #     premis.append(P.agent(
    #             P.agentIdentifier(
    #                 P.agentIdentifierType('LOCAL'),
    #                 P.agentIdentifierValue(identifier_value)
    #             ),
    #             P.agentName('E-ARK AIP to DIP Converter'),
    #             P.agentType('Software')))
    #
    #     identifier_value = 'AIP Creation'
    #     linking_agent = 'earkweb'
    #     linking_object=None
    #     premis.append(P.event(
    #             P.eventIdentifier(
    #                 P.eventIdentifierType('LOCAL'),
    #                 P.eventIdentifierValue(identifier_value)
    #             ),
    #             P.eventType,
    #             P.eventDateTime(current_timestamp()),
    #             P.linkingAgentIdentifier(
    #                 P.linkingAgentIdentifierType('LOCAL'),
    #                 P.linkingAgentIdentifierValue(linking_agent)
    #             ),
    #
    #             P.linkingAgentIdentifier(
    #                 P.linkingAgentIdentifierType('LOCAL'),
    #                 P.linkingAgentIdentifierValue(linking_object)
    #             )
    #             if linking_object is not None else None
    #         ))
    #
    #     str = etree.tostring(premis, encoding='UTF-8', pretty_print=True, xml_declaration=True)
    #     preservation_dir = os.path.join(self.root_path, './metadata/preservation')
    #     if not os.path.exists(preservation_dir):
    #         os.mkdir(preservation_dir)
    #     path_premis = os.path.join(self.root_path, './metadata/preservation/premis.xml')
    #     with open(path_premis, 'w') as output_file:
    #         output_file.write(str)
    #
    #     return premis_ids

    # def createSIPParentMets(self, mets_files):
    #     #create METS skeleton
    #     METS_ATTRIBUTES = {"OBJID" : "", "TYPE" : "", "LABEL" : "", "PROFILE" : "http://www.ra.ee/METS/v01/IP.xml", "ID" : "" }
    #     root = M.mets(METS_ATTRIBUTES)
    #     root.attrib['{%s}schemaLocation' % XSI_NS] = "http://www.loc.gov/METS/ schemas/mets.xsd ExtensionMETS schemas/ExtensionMETS.xsd http://www.w3.org/1999/xlink schemas/xlink.xsd"
    #
    #     mets_hdr = M.metsHdr({"CREATEDATE": current_timestamp(), q(METSEXT_NS,"OAISSTATUS") :"", "RECORDSTATUS" :""})
    #     root.append(mets_hdr)
    #
    #
    #     mets_hdr.append(self.createAgent("ARCHIVIST", "ORGANIZATION", "" ,"Institution", "Note"))
    #     mets_hdr.append(self.createAgent("ARCHIVIST", "OTHER", "" ,"Institution", "Note"))
    #     mets_hdr.append(self.createAgent("CREATOR", "ORGANIZATION", "", "Institution", "Note"))
    #     mets_hdr.append(self.createAgent("CREATOR", "OTHER", "SOFTWARE", "E-ARK SIP Creator", "VERSION=0.0.1"))
    #     mets_hdr.append(self.createAgent("PRESERVATION", "ORGANIZATION", "", "Institution", "Note"))
    #     mets_hdr.append(M.metsDocumentID("METS.xml"))
    #
    #     mets_dmd = M.dmdSec({"ID":""})
    #     root.append(mets_dmd)
    #
    #     mets_amdSec = M.amdSec({"ID":"ID" + uuid.uuid1().__str__()})
    #     root.append(mets_amdSec)
    #
    #     mets_techmd = M.techMD({"ID":"ID" + uuid.uuid1().__str__()})
    #     mets_amdSec.append(mets_techmd)
    #     #for id in premis_ids:
    #     #    mets_mdref = M.mdRef({"LOCTYPE":"URL", "MDTYPE":"PREMIS:OBJECT", q(XLINK_NS,"href"):"./metadata/preservation/PREMIS.xml#"+id.__str__()})
    #     #    mets_techmd.append(mets_mdref)
    #
    #     mets_fileSec = M.fileSec()
    #     root.append(mets_fileSec)
    #
    #     mets_filegroup = M.fileGrp({"ID": "ID" + uuid.uuid1().__str__()})
    #     mets_fileSec.append(mets_filegroup)
    #
    #     #content_ids = self.addFiles(os.path.join(self.root_path, 'data'), mets_filegroup)
    #     #metadata_ids = self.addFiles(os.path.join(self.root_path, 'metadata'), mets_filegroup)
    #
    #     #mets_ids = []
    #     #for file in mets_files:
    #     #    print file
    #     #    mets_id = self.addFile(file, mets_filegroup)
    #     #    mets_ids.append(mets_id)
    #
    #
    #     # checking for cross-representation metadata:
    #     #package_root = self.root_path.rsplit('/', 2)
    #     #package_metadata = os.path.join(package_root[0], 'metadata')
    #     #ext_metadata_ids = self.addFiles(package_metadata, mets_filegroup)
    #
    #     mets_structmap = M.structMap({"ID": "", "TYPE":"", "LABEL":"Simple grouping"})
    #     root.append(mets_structmap)
    #
    #     mets_structmap_div = M.div({"ADMID":"", "LABEL":"Package", "DMDID" : ""})
    #     mets_structmap.append(mets_structmap_div)
    #
    #     mets_structmap_mets_div = M.div({"LABEL":"Representation Metadata"})
    #     mets_structmap_div.append(mets_structmap_mets_div)
    #
    #     for file in mets_files:
    #         file_url = "./%s" % os.path.relpath(file, self.root_path)
    #         mptr = M.mptr({"LOCTYPE": "URL", q(XLINK_NS, 'href'): file_url })
    #         mets_structmap_mets_div.append(mptr)
    #
    #     #for id in mets_ids:
    #     #    mptr = M.mptr({"FILEID": id})
    #     #    mets_structmap_mets_div.append(mptr)
    #     #    #fptr = M.fptr({"FILEID": id})
    #     #    #mets_structmap_mets_div.append(fptr)
    #
    #     #mets_structmap_content_div = M.div({"LABEL":"Content"})
    #     #mets_structmap_div.append(mets_structmap_content_div)
    #     #for id in content_ids:
    #     #    fptr = M.fptr({"FILEID": id})
    #     #    mets_structmap_content_div.append(fptr)
    #
    #     #mets_structmap_metadata_div = M.div({"LABEL":"Metadata"})
    #     #mets_structmap_div.append(mets_structmap_metadata_div)
    #     #for id in metadata_ids:
    #     #    fptr = M.fptr({"FILEID": id})
    #     #    mets_structmap_metadata_div.append(fptr)
    #     #for id in ext_metadata_ids:
    #     #    fptr = M.fptr({"FILEID": id})
    #     #    mets_structmap_metadata_div.append(fptr)
    #
    #     #my_mets.fileSec.append(M.fileGrp({'USE': 'submission'}))
    #     #my_mets.fileSec(M.fileGrp({'USE': 'submission'}))
    #     #mets_schema_file = os.path.join(root_dir, "sandbox/sipgenerator/resources/ENA_RK_TartuLV_141127/schemas/mets.xsd")
    #     #mets_schema = etree.parse(mets_schema_file)
    #     #mets_xsd = etree.XMLSchema(mets_schema)
    #
    #     str = etree.tostring(root, encoding='UTF-8', pretty_print=True, xml_declaration=True)
    #
    #     path_mets = os.path.join(self.root_path,'METS.xml')
    #     with open(path_mets, 'w') as output_file:
    #         output_file.write(str)


    # def createSIPMets(self, premis_ids = None):
    #     if premis_ids == None:
    #         premis_ids = self.createPremis()
    #
    #     #create METS skeleton
    #     # METS_ATTRIBUTES = {"OBJID" : "", "TYPE" : "", "LABEL" : "", "PROFILE" : "http://www.ra.ee/METS/v01/IP.xml", "ID" : "" }
    #     METS_ATTRIBUTES = {"OBJID": "placeholder123",
    #                        "LABEL": "METS file describing the SIP matching the OBJID.",
    #                        "PROFILE": "http://www.ra.ee/METS/v01/IP.xml",
    #                        "TYPE": "SIP"}
    #     root = M.mets(METS_ATTRIBUTES)
    #     root.attrib['{%s}schemaLocation' % XSI_NS] = "http://www.loc.gov/METS/ schemas/mets.xsd ExtensionMETS schemas/ExtensionMETS.xsd http://www.w3.org/1999/xlink schemas/xlink.xsd"
    #
    #     # mets_hdr = M.metsHdr({"CREATEDATE": current_timestamp(), q(METSEXT_NS,"OAISSTATUS") :"", "RECORDSTATUS" :""})
    #     mets_hdr = M.metsHdr({"CREATEDATE": current_timestamp(), "RECORDSTATUS" :"NEW"})
    #     root.append(mets_hdr)
    #
    #     mets_hdr.append(self.createAgent("ARCHIVIST", "ORGANIZATION", "" ,"Institution", "Note"))
    #     mets_hdr.append(self.createAgent("ARCHIVIST", "OTHER", "" ,"Institution", "Note"))
    #     mets_hdr.append(self.createAgent("CREATOR", "ORGANIZATION", "", "Institution", "Note"))
    #     mets_hdr.append(self.createAgent("CREATOR", "OTHER", "SOFTWARE", "E-ARK SIP Creator", "VERSION=0.0.1"))
    #     mets_hdr.append(self.createAgent("PRESERVATION", "ORGANIZATION", "", "Institution", "Note"))
    #     mets_hdr.append(M.metsDocumentID("METS.xml"))
    #
    #     mets_dmd = M.dmdSec({"ID": "ID" + uuid.uuid4().__str__()})
    #     root.append(mets_dmd)
    #     # this is how to add descriptive metadata entry
    #     #file_name = "../schemas/ead.xml"
    #     #file_url = unicode(os.path.join("",file_name), "utf-8")
    #     #checksum = self.sha256(file_name)
    #     #file_size = os.path.getsize(file_name)
    #     #mets_mdref= M.mdRef({"LOCTYPE":"URL", "MDTYPE":"EAD", "MIMETYPE":"text/xml", "CREATED":current_timestamp(), q(XLINK_NS,"type"):"simple", q(XLINK_NS,"href"):file_url, "CHECKSUMTYPE":"SHA-256", "CHECKSUM":file_checksum, "SIZE":file_size})
    #     #mets_dmd.append(mets_mdref)
    #
    #     mets_amdSec = M.amdSec({"ID":"ID" + uuid.uuid4().__str__()})
    #     root.append(mets_amdSec)
    #
    #     mets_techmd = M.techMD({"ID":"ID" + uuid.uuid4().__str__()})
    #     mets_amdSec.append(mets_techmd)
    #     #for id in premis_ids:
    #     #    mets_mdref = M.mdRef({"LOCTYPE":"URL", "MDTYPE":"PREMIS:OBJECT", q(XLINK_NS,"href"):"./metadata/preservation/PREMIS.xml#"+id.__str__()})
    #     #    mets_techmd.append(mets_mdref)
    #     #techmd_ids = self.addFiles(os.path.join(self.root_path, 'metadata/preservation'))
    #
    #
    #     mets_fileSec = M.fileSec()
    #     root.append(mets_fileSec)
    #
    #     # general filegroup
    #     mets_filegroup = M.fileGrp({"ID": "ID" + uuid.uuid4().__str__(), "USE": "general filegroup"})
    #     mets_fileSec.append(mets_filegroup)
    #
    #     # data, metadata and schemas - package level
    #     content_ids = self.addFiles(os.path.join(self.root_path, 'data'), mets_filegroup)
    #     metadata_ids = self.addFiles(os.path.join(self.root_path, 'metadata'), mets_filegroup)
    #     schema_ids = self.addFiles(os.path.join(self.root_path, 'schemas'), mets_filegroup)
    #
    #     # checking for cross-representation metadata and schemas:
    #     package_root = self.root_path.rsplit('/', 2)
    #     package_metadata = os.path.join(package_root[0], 'metadata')
    #     cross_metadata_ids = self.addFiles(package_metadata, mets_filegroup)
    #     package_schemas = os.path.join(package_root[0], 'schemas')
    #     cross_schemas_ids = self.addFiles(package_schemas, mets_filegroup)
    #
    #     # mets_structmap = M.structMap({"ID": "", "TYPE":"", "LABEL":"Simple grouping"})
    #     mets_structmap = M.structMap({"LABEL": "Simple SIP structuring", "TYPE": "logical"})
    #     root.append(mets_structmap)
    #
    #     mets_structmap_div = M.div({"ADMID":"", "LABEL":"Package", "DMDID" : ""})
    #     mets_structmap.append(mets_structmap_div)
    #
    #     # files in /data
    #     mets_structmap_content_div = M.div({"LABEL":"Content"})
    #     mets_structmap_div.append(mets_structmap_content_div)
    #     for id in content_ids:
    #         fptr = M.fptr({"FILEID": id})
    #         mets_structmap_content_div.append(fptr)
    #
    #     # files in /metadata and ./../../metadata
    #     mets_structmap_metadata_div = M.div({"LABEL":"Metadata"})
    #     mets_structmap_div.append(mets_structmap_metadata_div)
    #     for id in metadata_ids:
    #         fptr = M.fptr({"FILEID": id})
    #         mets_structmap_metadata_div.append(fptr)
    #     for id in cross_metadata_ids:
    #         fptr = M.fptr({"FILEID": id})
    #         mets_structmap_metadata_div.append(fptr)
    #
    #     # files in /schemas and ./../../schemas
    #     mets_structmap_schemas_div = M.div({"LABEL": "Schemas"})
    #     mets_structmap_div.append(mets_structmap_schemas_div)
    #     for id in schema_ids:
    #         fptr = M.fptr({"FILEID": id})
    #         mets_structmap_schemas_div.append(fptr)
    #     for id in cross_schemas_ids:
    #         fptr = M.fptr({"FILEID": id})
    #         mets_structmap_schemas_div.append(fptr)
    #
    #     #my_mets.fileSec.append(M.fileGrp({'USE': 'submission'}))
    #     #my_mets.fileSec(M.fileGrp({'USE': 'submission'}))
    #     #mets_schema_file = os.path.join(root_dir, "sandbox/sipgenerator/resources/ENA_RK_TartuLV_141127/schemas/mets.xsd")
    #     #mets_schema = etree.parse(mets_schema_file)
    #     #mets_xsd = etree.XMLSchema(mets_schema)
    #
    #     str = etree.tostring(root, encoding='UTF-8', pretty_print=True, xml_declaration=True)
    #
    #     path_mets = os.path.join(self.root_path,'METS.xml')
    #     with open(path_mets, 'w') as output_file:
    #         output_file.write(str)


    #def createAIPMets(self, premis_ids=None):
    # def createAIPMets(self, packageid):
    #     '''
    #     Create the AIP METS file.
    #
    #     @param premis_ids:
    #     @return:
    #     '''
    #     #premis_ids = None
    #
    #     #if premis_ids == None:
    #     #    premis_ids = self.createPremis()
    #
    #     # TODO: add metadata files to correct metadata sections!
    #
    #     # create METS skeleton
    #     METS_ATTRIBUTES = {"OBJID": packageid,
    #                        "LABEL": "METS file describing the AIP matching the OBJID.",
    #                        "PROFILE": "http://www.ra.ee/METS/v01/IP.xml",
    #                        "TYPE": "AIP"}
    #     root = M.mets(METS_ATTRIBUTES)
    #     root.attrib['{%s}schemaLocation' % XSI_NS] = "http://www.loc.gov/METS/ schemas/mets.xsd ExtensionMETS schemas/ExtensionMETS.xsd http://www.w3.org/1999/xlink schemas/xlink.xsd"
    #
    #     # mets_hdr = M.metsHdr({"CREATEDATE": current_timestamp(), q(METSEXT_NS,"OAISSTATUS") :"", "RECORDSTATUS" :"AIP"})
    #     mets_hdr = M.metsHdr({"CREATEDATE": current_timestamp(), "RECORDSTATUS" :"NEW"})
    #     root.append(mets_hdr)
    #
    #     mets_hdr.append(self.createAgent("ARCHIVIST", "ORGANIZATION", "" ,"Institution", "Note"))
    #     mets_hdr.append(self.createAgent("ARCHIVIST", "OTHER", "" ,"Institution", "Note"))
    #     mets_hdr.append(self.createAgent("CREATOR", "ORGANIZATION", "", "Institution", "Note"))
    #     mets_hdr.append(self.createAgent("CREATOR", "OTHER", "SOFTWARE", "E-ARK SIP Creator", "VERSION=0.0.1"))
    #     mets_hdr.append(self.createAgent("PRESERVATION", "ORGANIZATION", "", "Institution", "Note"))
    #     mets_hdr.append(M.metsDocumentID("METS.xml"))
    #
    #     mets_dmd = M.dmdSec({"ID": "ID" + uuid.uuid4().__str__()})
    #     root.append(mets_dmd)
    #     # this is how to add descriptive metadata entry
    #     #file_name = "../schemas/ead.xml"
    #     #file_url = unicode(os.path.join(",file_name), "utf-8")
    #     #checksum = self.sha256(file_name)
    #     #file_size = os.path.getsize(file_name)
    #     #mets_mdref= M.mdRef({"LOCTYPE":"URL", "MDTYPE":"EAD", "MIMETYPE":"text/xml", "CREATED":current_timestamp(), q(XLINK_NS,"type"):"simple", q(XLINK_NS,"href"):file_url, "CHECKSUMTYPE":"SHA-256", "CHECKSUM":file_checksum, "SIZE":file_size})
    #     #mets_dmd.append(mets_mdref)
    #
    #     # create amdSec
    #     mets_amdSec = M.amdSec({"ID": "ID" + uuid.uuid4().__str__()})
    #     root.append(mets_amdSec)
    #
    #     # create techMD
    #     #mets_techmd = M.techMD({"ID": "ID" + uuid.uuid4().__str__()})
    #     #mets_amdSec.append(mets_techmd)
    #     #for id in premis_ids:
    #     #    mets_mdref = M.mdRef({"LOCTYPE":"URL", "MDTYPE":"PREMIS:OBJECT", q(XLINK_NS,"href"):"./metadata/preservation/PREMIS.xml#"+id.__str__()})
    #     #    mets_techmd.append(mets_mdref)
    #
    #     mets_fileSec = M.fileSec()
    #     root.append(mets_fileSec)
    #
    #     # TODO: metadata entries into correct mets section + structmap: seperate between AIP and rep level metadata!
    #     # check if there is another METS file referencing the metadata
    #
    #     # general filegroup
    #     mets_filegroup = M.fileGrp({"ID": "ID" + uuid.uuid4().__str__(), "USE": "general filegroup"})
    #     mets_fileSec.append(mets_filegroup)
    #
    #     #metadata_ids = self.addFiles(os.path.join(self.root_path, 'metadata'), mets_filegroup)
    #     #submission_meta_ids = self.addFiles(os.path.join(self.root_path, 'submission/metadata'), mets_filegroup)
    #
    #     # mets_structmap = M.structMap({"ID": "", "TYPE":"", "LABEL":"Simple grouping"})
    #     mets_structmap = M.structMap({"LABEL": "Simple AIP structuring", "TYPE": "logical"})
    #     root.append(mets_structmap)
    #
    #     mets_structmap_div = M.div({"ADMID":"", "LABEL":"Package", "DMDID" : ""})
    #     mets_structmap.append(mets_structmap_div)
    #
    #     # metadata structmap - IP root level!
    #     mets_structmap_metadata_div = M.div({"LABEL": "Metadata"})
    #     mets_structmap_div.append(mets_structmap_metadata_div)
    #     #for id in metadata_ids:
    #     #    fptr = M.fptr({"FILEID": id})
    #     #    mets_structmap_metadata_div.append(fptr)
    #
    #     # metadata structmap - submission level!
    #     #mets_structmap_metadata_sub_div = M.div({"LABEL": "Metadata Submission"})
    #     #mets_structmap_div.append(mets_structmap_metadata_sub_div)
    #     #for id in submission_meta_ids:
    #     #    fptr = M.fptr({"FILEID": id})
    #     #    mets_structmap_metadata_sub_div.append(fptr)
    #
    #     # content structmap - all representations! (is only filled if no separate METS exists for the rep)
    #     mets_structmap_content_div = M.div({"LABEL": "various files"})
    #     mets_structmap_div.append(mets_structmap_content_div)
    #
    #     # structmap for schema files
    #     mets_structmap_schema_div = M.div({"LABEL": "schema files"})
    #     mets_structmap_div.append(mets_structmap_schema_div)
    #
    #     # create structmap for representations
    #     # mets_structmap_reps = M.structMap({"ID": "", "TYPE":"", "LABEL":"representations"})
    #     mets_structmap_reps = M.structMap({"TYPE":"logical", "LABEL":"representations"})
    #     root.append(mets_structmap_reps)
    #     mets_div_reps = M.div({'LABEL': 'representations', 'TYPE': 'type'})
    #     mets_structmap_reps.append(mets_div_reps)
    #
    #     # package
    #     workdir_length = len(self.root_path)
    #     #for directory, subdirectories, filenames in os.walk(os.path.join(self.root_path, 'submission/representations')):
    #     for directory, subdirectories, filenames in os.walk(self.root_path):
    #         if directory == os.path.join(self.root_path, 'metadata/earkweb'):
    #             del directory
    #             del filenames[:]
    #             del subdirectories[:]
    #         if len(filenames) > 0:
    #             for filename in filenames:
    #                 # ignore all files on AIP root level (since they wont be packed with the AIP):
    #                 if directory == self.root_path:
    #                     del filename
    #                 else:
    #                     # TODO: list rep metadata only in the rep Mets?
    #                     rel_path_file = ('.' + directory[workdir_length:] + '/' + filename).decode('utf-8')
    #                     if filename.lower() == 'mets.xml':
    #                         # delete the subdirectories list to stop os.walk from traversing further;
    #                         # mets file should be added as <mets:mptr> to <structMap> for corresponding rep
    #                         del subdirectories[:]
    #                         rep_name = directory.rsplit('/', 2)
    #                         rep_name = os.path.join(rep_name[1], rep_name[2])
    #                         # create structMap div and append to representations structMap
    #                         mets_structmap_rep_div = M.div({"LABEL": rep_name, "TYPE": "representation mets", "ID": "ID" + uuid.uuid4().__str__()})
    #                         mets_div_reps.append(mets_structmap_rep_div)
    #                         # add mets file as <mets:mptr>
    #                         metspointer = M.mptr({"LOCTYPE": "URL",
    #                                               q(XLINK_NS,"title"): "mets file describing representation: " + rep_name + " of AIP: " + packageid,
    #                                               q(XLINK_NS,"href"): rel_path_file})
    #                                              #"ID": uuid.uuid4().__str__()})
    #                         mets_structmap_rep_div.append(metspointer)
    #                         # also add the rep mets to the filegroup, so we can have a fptr
    #                         id = self.addFile(os.path.join(directory, filename), mets_filegroup)
    #                         mets_fptr = M.fptr({"FILEID": id})
    #                         mets_structmap_rep_div.append(mets_fptr)
    #                     elif filename and directory.endswith('schemas'):
    #                         # schema files
    #                         id = self.addFile(os.path.join(directory, filename), mets_filegroup)
    #                         fptr = M.fptr({'FILEID': id})
    #                         mets_structmap_schema_div.append(fptr)
    #                     elif filename == 'earkweb.log':
    #                         # earkweb log file - currently treated as digiprovMD
    #                         mets_digiprovmd = M.digiprovMD({"ID": "ID" + uuid.uuid4().__str__()})
    #                         mets_amdSec.append(mets_digiprovmd)
    #                         checksum = self.sha256(os.path.join(directory,filename))
    #                         id = "ID" + uuid.uuid4().__str__()
    #                         mets_mdref = M.mdRef({"LOCTYPE":"URL",
    #                                              "MIMETYPE":"text/xml",
    #                                              "CREATED":current_timestamp(),
    #                                              q(XLINK_NS,"type"):"simple",
    #                                              q(XLINK_NS,"href"):rel_path_file,
    #                                              "CHECKSUMTYPE":"SHA-256",
    #                                              "CHECKSUM":checksum,
    #                                              "ID": id,
    #                                               "MDTYPE": "OTHER"})
    #                         mets_digiprovmd.append(mets_mdref)
    #                         fptr = M.fptr({"FILEID": id})
    #                         mets_structmap_metadata_div.append(fptr)
    #                     elif directory.endswith('descriptive'):
    #                         # descriptive metadata
    #                         #mets_dmd = M.dmdSec({"ID": "ID" + uuid.uuid4().__str__()})
    #                         #root.append(mets_dmd)
    #                         checksum = self.sha256(os.path.join(directory,filename))
    #                         id = "ID" + uuid.uuid4().__str__()
    #                         mets_mdref = M.mdRef({"LOCTYPE":"URL",
    #                                              "MIMETYPE":"text/xml",
    #                                              "CREATED":current_timestamp(),
    #                                              q(XLINK_NS,"type"):"simple",
    #                                              q(XLINK_NS,"href"):rel_path_file,
    #                                              "CHECKSUMTYPE":"SHA-256",
    #                                              "CHECKSUM":checksum,
    #                                              "ID": id,
    #                                               "MDTYPE": "OTHER"})
    #                         mets_dmd.append(mets_mdref)
    #                         fptr = M.fptr({"FILEID": id})
    #                         mets_structmap_metadata_div.append(fptr)
    #                     elif directory.endswith('preservation'):
    #                         # preservation metadata (premis, techMD?)
    #                         #mets_techmd = M.techMD({"ID": id})
    #                         #mets_amdSec.append(mets_techmd)
    #                         mets_digiprovmd = M.digiprovMD({"ID": "ID" + uuid.uuid4().__str__()})
    #                         mets_amdSec.append(mets_digiprovmd)
    #                         checksum = self.sha256(os.path.join(directory,filename))
    #                         id = "ID" + uuid.uuid4().__str__()
    #                         mets_mdref = M.mdRef({"LOCTYPE":"URL",
    #                                              "MIMETYPE":"text/xml",
    #                                              "CREATED":current_timestamp(),
    #                                              q(XLINK_NS,"type"):"simple",
    #                                              q(XLINK_NS,"href"):rel_path_file,
    #                                              "CHECKSUMTYPE":"SHA-256",
    #                                              "CHECKSUM":checksum,
    #                                              "ID": id,
    #                                               "MDTYPE": "OTHER"})
    #                         #mets_techmd.append(mets_mdref)
    #                         mets_digiprovmd.append(mets_mdref)
    #                         fptr = M.fptr({"FILEID": id})
    #                         mets_structmap_metadata_div.append(fptr)
    #                     elif filename and not (directory.endswith('descriptive') or
    #                                            directory.endswith('metadata') or
    #                                            directory.endswith('preservation')):
    #                         # print 'found a file: ' + os.path.join(directory, filename)
    #                         id = self.addFile(os.path.join(directory, filename), mets_filegroup)
    #                         fptr = M.fptr({"FILEID": id})
    #                         mets_structmap_content_div.append(fptr)
    #
    #
    #     str = etree.tostring(root, encoding='UTF-8', pretty_print=True, xml_declaration=True)
    #
    #     path_mets = os.path.join(self.root_path,'METS.xml')
    #     with open(path_mets, 'w') as output_file:
    #         output_file.write(str)


    def createDeliveryMets(self, input_archive, output_mets):
        #create delivery METS skeleton
        METS_ATTRIBUTES = {"OBJID" : "UUID:" + uuid.uuid4().__str__(), "TYPE" : "SIP", "LABEL" : "Delivery METS", "PROFILE" : "http://webb.eark/package/METS/IP_CS.xml", "ID" : "ID" + uuid.uuid4().__str__() }
        root = M.mets(METS_ATTRIBUTES)
        root.attrib['{%s}schemaLocation' % XSI_NS] = "http://www.loc.gov/METS/ schemas/mets.xsd"

        mets_hdr = M.metsHdr({"CREATEDATE": current_timestamp()})
        root.append(mets_hdr)

        mets_hdr.append(self.createAgent("ARCHIVIST", "ORGANIZATION", "", "E-ARK", "Demonstrator"))
        mets_hdr.append(self.createAgent("CREATOR", "ORGANIZATION", "", "E-ARK", "Demonstrator"))
        mets_hdr.append(self.createAgent("CREATOR", "OTHER", "SOFTWARE", "E-ARK Web SIP Creator", "VERSION=1.0"))
        mets_hdr.append(self.createAgent("PRESERVATION", "ORGANIZATION", "", "E-ARK", "Demonstrator"))
        _,fname = os.path.split(output_mets)
        mets_hdr.append(M.metsDocumentID(fname))

        mets_fileSec = M.fileSec()
        root.append(mets_fileSec)

        mets_filegroup = M.fileGrp({"USE" : "PACKAGES", "ID": "ID" + uuid.uuid4().__str__()})
        mets_fileSec.append(mets_filegroup)

        content_id = self.addFile(input_archive, mets_filegroup)

        mets_structmap = M.structMap({"ID": "ID%s" % uuid.uuid4(), "TYPE": "physical", "LABEL": "Profilestructmap"})
        root.append(mets_structmap)
        mets_structmap_div = M.div({"LABEL": "Package"})
        mets_structmap.append(mets_structmap_div)
        mets_structmap_content_div = M.div({"LABEL": "Content"})
        mets_structmap_div.append(mets_structmap_content_div)
        fptr = M.fptr({"FILEID": "ID%s" % uuid.uuid4()})
        mets_structmap_content_div.append(fptr)

        xml_content = ET.tostring(root, encoding='UTF-8', pretty_print=True, xml_declaration=True)
        with open(output_mets, 'wb') as output_file:
            output_file.write(xml_content)

