# If the CGI script crashes before finishing the headers, cgitb will emit invalid HTTP headers before showing the error message.
# The workaround is to put: HttpProtocolOptions Unsafe line into the apache .conf


import cgitb

import os
# This library is used by CGI scripts and command-line scripts.
# Therefore, its behaviour is different in case of error.
if os.getenv("SERVER_SOFTWARE"):
    cgitb.enable()

import re
import sys
import six
import cgi
import time
import socket
import base64
import importlib
import logging
import inspect
import rdflib

import lib_kbase

# This minimizes changes because it is used everywhere.
from lib_naming_conventions import standardized_file_path
from lib_naming_conventions import standardized_memmap_path

# In Python 3, urllib.quote has been moved to urllib.parse.quote and it does handle unicode by default.
# TODO: Use module six.
try:
    from urllib import quote as urllib_quote
    from urllib import unquote as urllib_unquote
except ImportError:
    from urllib.parse import quote as urllib_quote
    from urllib.parse import unquote as urllib_unquote

try:
    from urlparse import urlparse as survol_urlparse
    from urlparse import parse_qs
except ImportError:
    from urllib.parse import urlparse as survol_urlparse
    from urllib.parse import parse_qs

is_py3 = sys.version_info >= (3,)

if is_py3:
    import html
    import html.parser
    def survol_unescape(s):
        return html.parser.unescape(s)

    html_escape = html.escape
else:
    # TODO: html might be present, so it might be worth testing.
    import HTMLParser
    def survol_unescape(s):
        return HTMLParser.HTMLParser().unescape(s)

    html_escape = cgi.escape

# See tests/init.py which duplicates this statement.
try:
    # For Python 3.0 and later
    from urllib.request import urlopen as survol_urlopen
except ImportError:
    # Fall back to Python 2's urllib2
    from urllib2 import urlopen as survol_urlopen


################################################################################

def SetLoggingConfig(log_level):
    # Reinit: https://stackoverflow.com/questions/12158048/changing-loggings-basicconfig-which-is-already-set
    for handler in logging.root.handlers[:]:
        logging.root.removeHandler(handler)

    # This avoids the message "No handlers could be found for logger "rdflib.term""
    # rdflib is used at least by lib_kbase.py
    logging.basicConfig(
        stream=sys.stderr,
        format='%(asctime)s %(levelname)8s %(filename)s %(lineno)d %(message)s',
        level = log_level)


SetLoggingConfig(logging.WARNING)

# Avoid this message:
# 2018-09-18 21:57:54,868  WARNING rdflib.term term.py 207: http://L... does not look like a valid URI, trying to serialize this will break.
loggerRdflib = logging.getLogger("rdflib.term")
loggerRdflib.setLevel(logging.ERROR)

# This is the general purpose logger.
if is_py3:
    logger_name = "inspect"
else:
    frm = inspect.stack()[1]
    mod = inspect.getmodule(frm[0])
    logger_name = mod.__name__

gblLogger = logging.getLogger(logger_name)

if is_py3:
    import builtins
    builtins.DEBUG = gblLogger.debug
    builtins.WARNING = gblLogger.warning
    builtins.ERROR = gblLogger.error
    builtins.INFO = gblLogger.info
    builtins.CRITICAL = gblLogger.critical
else:
    import __builtin__
    __builtin__.DEBUG = gblLogger.debug
    __builtin__.WARNING = gblLogger.warning
    __builtin__.ERROR = gblLogger.error
    __builtin__.INFO = gblLogger.info
    __builtin__.CRITICAL = gblLogger.critical

################################################################################

# Returns None even if jinja2 is available but configuration does not use it.
def GetJinja2():
    try:
        import jinja2
        return jinja2
    except:
        return None

################################################################################

# Sometimes we have to display many strings of the same type, for example
# filenames or WMI monikers. These strings are mixes of words and numbers.
# They do not sort naturally well, because the numbers are lexicographically
# sorted. The Python module natsort does that.
# Simple yet flexible natural sorting in Python.
try:
    import natsort
    from natsort import natsort_keygen

    natural_sorted = natsort.natsorted

    def natural_sort_list(one_list, **args):

        natsort_key = natsort_keygen()

        try:
            orig_key = args['key']
            args['key'] = lambda in_param: natsort_key(orig_key(in_param))

        except KeyError:
            args['key'] = natsort_key
        one_list.sort(**args)

except ImportError:
    WARNING("WritePatterned Module natsorted not available.")
    natural_sorted = sorted

    def natural_sort_list(one_list, **args):
        one_list.sort(**args)

################################################################################

# This avoids needing the "six" module which is not always available.
# On some environments, it is a hassle to import it.
if is_py3:
    def six_iteritems(array):
            return array.items()
else:
    def six_iteritems(array):
        return array.iteritems()

# Add all usual Python types.
scalar_data_types = six.string_types + (six.text_type, six.binary_type, float, bool) + six.integer_types

################################################################################

NodeLiteral = rdflib.Literal
#def NodeLiteral(value):
#    return rdflib.Literal(value)

NodeUrl = rdflib.term.URIRef
#def NodeUrl(url):
#    # TODO: Apparently, it is called twice, which is not detected
#    # because MakeNodeUrl returns the same string.
#    return rdflib.term.URIRef(url)

################################################################################


# See xidCgiDelimiter = "?xid="
def EncodeEntityId(entity_type, entity_id):
    return "xid=%s.%s" % (entity_type, entity_id)

################################################################################


# unitSI = "B", "b", "B/s" for example.
# TODO: We need a way to describe a number of items, without unit.
# This is different from an integer ID which should always be displayed "as is",
# just like a string.
# We might have units such as "B/B" which are without dimensions.
def AddSIUnit(number, unitSI):
    if unitSI:
        return str(number) + " " + unitSI
    else:
        return str(number)

################################################################################


def HttpPrefix():
    """This is the protocol, the server address followed by the port:
     "http://192.168.0.14:80", "http://rchateau-hp:8000" """

    # Default values for ease of testing, so CGI scripts can be run as is from command line..
    try:
        server_addr = os.environ['SERVER_NAME']
        # This is a special value if the client library is in local mode.
        if server_addr != "LOCALHOST":
            # Hostnames always in lowercase.
            server_addr = server_addr.lower()

        # 'REMOTE_ADDR'] => "127.0.0.1"
        # 'SERVER_NAME'] => "rchateau-HP"
        # 'REMOTE_HOST'] => "rchateau-HP"

    except KeyError:
        ERROR("HttpPrefix SERVER_NAME MUST BE DEFINED")
        sys.exit(1)
    
    try:
        server_port = os.environ['SERVER_PORT']
    except KeyError:
        # Default HTTP port.
        server_port = "80"

    # BEWARE: Colons are forbidden in URIs apparently !!!
    # Due to a very strange bug which displays:
    # "http://127.0.0.1:80/PythonStyle/survol/entity.py" ...
    # does not look like a valid URI, trying to serialize this will break.
    # But if we do not add "http:" etc... SVG adds its prefix "127.0.0.1" etc...
    prfx = 'http://' + server_addr + ':' + server_port
    # sys.stderr.write("HttpPrefix server_addr=%s prfx=%s\n"%(server_addr,prfx))
    return prfx

# This is also used in lib_client to differentiate local from remote scripts.
# Objects need an URL, but if no agent is used, we need a pseudo-URL,
# represented by the string, by convention. Syntactically, this is a correct URL.
prefixLocalExecution = "/LocalExecution"


def UriRootHelper():
    try:
        # Checks of this environment variable is defined.
        os.environ["SERVER_NAME"]
        # sys.stderr.write("SERVER_NAME=%s gethostname=%s\n" % (os.environ["SERVER_NAME"], socket.gethostname()))
    except KeyError:
        # This is necessary when returning objects for example from get_instances()
        # in the client library lib_client.py. The local objects need a hostname,
        # and "localhost" fills this role.
        # However, if used with remote objects, this hostname should be replaced
        # on-the-fly by the actual host name.
        # An alternative is to specify the right hostname here.
        os.environ["SERVER_NAME"] = socket.gethostname().lower()
        # sys.stderr.write("SERVER_NAME=%s FIXED\n" % os.environ["SERVER_NAME"])
    try:
        # SCRIPT_NAME=/PythonStyle/survol/internals/print.py
        # SCRIPT_NAME=/survol/print_environment_variables.py
        scriptNam=os.environ['SCRIPT_NAME']
        idx = scriptNam.find('survol')
        if idx >= 0:
            root = scriptNam[:idx] + 'survol'
        else:
            # Should not happen.
            root = "/NON_SURVOL_URL/" + scriptNam
        # sys.stderr.write("UriRootHelper scriptNam=%s root=%s\n"%(scriptNam,root))

    except KeyError:
        # If this runs from the command line and not as a CGI script,
        # then this environment variable is not set.
        # Just like SERVER_NAME, it should test that the caller is lib_client.py.
        root = prefixLocalExecution
    urh = HttpPrefix() + root
    # sys.stderr.write("UriRootHelper urh=%s\n"%urh)
    return urh

uriRoot = UriRootHelper()
# sys.stderr.write("Setting uriRoot. __file__=%s\n"%__file__)

################################################################################

# This returns the hostname as a string. Some special processing because on Windows,
# the returned hostname seems truncated.
# See lib_uris.HostnameUri()
#
# socket.gethostname()                 socket.gethostbyaddr(socket.gethostname()) 
# fedora22                             ('advancedsearch.virginmedia.com', [], ['81.200.64.50'])
# rchateau-HP                          ('rchateau-HP', [], ['fe80::3c7a:339:64f0:2161'])
# ssh02.cluster023.gra.hosting.ovh.net ('ssh02.cluster023.gra.hosting.ovh.net', ['ssh02'], ['10.23.90.2'])
#
# Some example of the values of important CGI variables:
# rchateau-hp IP address is 192.168.0.14
#
# http://rchateau-hp:8000/survol/print_environment_variables.py
# SERVER_SOFTWARE=SimpleHTTP/0.6 Python/2.7.10
# SERVER_NAME=rchateau-HP
#
# http://rchateau-hp/Survol/survol/print_environment_variables.py
# SERVER_SOFTWARE=Apache/2.4.12 (Win64) OpenSSL/1.0.1m mod_wsgi/4.4.12 Python/2.7.10
# SERVER_NAME=rchateau-hp
# SERVER_ADDR=fe80::3c7a:339:64f0:2161
# HTTP_HOST=rchateau-hp
#
# http://127.0.0.1/Survol/survol/print_environment_variables.py
# SERVER_SOFTWARE=Apache/2.4.12 (Win64) OpenSSL/1.0.1m mod_wsgi/4.4.12 Python/2.7.10
# SERVER_NAME=127.0.0.1
# SERVER_ADDR=127.0.0.1
# HTTP_HOST=127.0.0.1
#
# http://192.168.0.14/Survol/survol/print_environment_variables.py
# SERVER_SOFTWARE=Apache/2.4.12 (Win64) OpenSSL/1.0.1m mod_wsgi/4.4.12 Python/2.7.10
# SERVER_NAME=192.168.0.14
# SERVER_ADDR=192.168.0.14
# HTTP_HOST=192.168.0.14
#
# This is 192.168.0.17 accessible via NAT and primhillcomputers.ddns.net:
#
# http://primhillcomputers.ddns.net/Survol/survol/print_environment_variables.py
# SERVER_SOFTWARE=Apache/2.4.18 (Fedora) PHP/5.6.23 mod_wsgi/4.4.8 Python/2.7.10
# SERVER_NAME=primhillcomputers.ddns.net
# SERVER_ADDR=192.168.0.17
# HTTP_HOST=primhillcomputers.ddns.net
#
# http://192.168.0.17/Survol/survol/print_environment_variables.py
# SERVER_SOFTWARE=Apache/2.4.18 (Fedora) PHP/5.6.23 mod_wsgi/4.4.8 Python/2.7.10
# SERVER_NAME=192.168.0.17
# SERVER_ADDR=192.168.0.17
# HTTP_HOST=192.168.0.17
#
# Todays, my IP address is 82.45.12.63, so let's try to access the same machine:
# http://82.45.12.63/Survol/survol/print_environment_variables.py
# SERVER_SOFTWARE=Apache/2.4.18 (Fedora) PHP/5.6.23 mod_wsgi/4.4.8 Python/2.7.10
# SERVER_NAME=82.45.12.63
# SERVER_ADDR=192.168.0.17
# HTTP_HOST=82.45.12.63
#
# It is better to rely on a distributed naming system: DNS or plain IP address.
def HostName():
    # SERVER_NAME is set by the HTTP server and might be wrong, but gives some consistency.

    # Converted to lowercase because of RFC4343: Domain Name System (DNS) Case Insensitivity Clarification
    return os.environ["SERVER_NAME"].lower()


currentHostname = HostName()


def GlobalGetHostByName(host_nam):
    try:
        the_ip = socket.gethostbyname(host_nam)
        # sys.stderr.write("GlobalGetHostByName tm=%f OK host_nam=%s the_ip=%s\n"%(time.time()-timeStart,host_nam,the_ip))
        return the_ip
    except Exception:
        # sys.stderr.write("GlobalGetHostByName tm=%f FAIL host_nam=%s\n"%(time.time()-timeStart,host_nam))
        return host_nam


# Beware: The machine might have several IP addresses.
try:
    # BEWARE: Possibly very slow.
    localIP = GlobalGetHostByName(currentHostname)
except Exception:
    # Apparently, it happens if the router is down.
    localIP = "127.0.0.1"


# This is for example used by WMI, which does not accept credentials
# for a local machine: We must therefore be sure that the machine is local or not.
def IsLocalAddress(an_host_nam):
    # Maybe entity_host="http://192.168.1.83:5988"
    host_only = EntHostToIp(an_host_nam)
    if host_only in [ None, "", "localhost", "127.0.0.1", currentHostname]:
        # sys.stderr.write("IsLocalAddress %s TRUE\n"%an_host_nam)
        return True

    try:
        ipOnly = GlobalGetHostByName(host_only)
    # socket.gaierror
    except Exception as exc:
        # Unknown machine
        # sys.stderr.write("IsLocalAddress an_host_nam=%s:%s FALSE\n" % ( an_host_nam, str(exc) ) )
        return False

    # IsLocalAddress MYHOST-HP ipOnly=192.168.0.14 localIP=127.0.0.1 currentHostname=127.0.0.1
    # sys.stderr.write("IsLocalAddress %s ipOnly=%s localIP=%s currentHostname=%s\n"%(an_host_nam,ipOnly,localIP,currentHostname))
    if ipOnly in [ "0.0.0.0", "127.0.0.1", localIP]:
        # sys.stderr.write("IsLocalAddress %s TRUE\n"%an_host_nam)
        return True

    # "MYHOST-HP" and "myhost-HP" ??
    # sys.stderr.write("IsLocalAddress %s socket.gethostname()=%s\n"%(an_host_nam,socket.gethostname()))
    if an_host_nam.lower() == socket.gethostname().lower():
        return True

    # sys.stderr.write("IsLocalAddress %s FALSE\n"%an_host_nam)
    return False

# Beware: lib_util.currentHostname="Unknown-30-b5-c2-02-0c-b5-2.home"
# socket.gethostname() = 'Unknown-30-b5-c2-02-0c-b5-2.home'
# socket.gethostbyaddr(hst) = ('Unknown-30-b5-c2-02-0c-b5-2.home', [], ['192.168.1.88'])
def SameHostOrLocal(srv, ent_host):
    if (ent_host == srv) or ((ent_host is None or ent_host in ["", "0.0.0.0"]) and (localIP == srv)):
        # We might add credentials.
        DEBUG("SameHostOrLocal ent_host=%s localIP=%s srv=%s SAME", ent_host, localIP, srv)
        return True
    else:
        DEBUG("SameHostOrLocal ent_host=%s localIP=%s srv=%s Different", ent_host, localIP, srv)
        return False

################################################################################


def TopUrl(entity_type, entity_id):
    """ This returns the top-level URL"""
    try:
        script_nam = os.environ['SCRIPT_NAME']
    except KeyError:
        script_nam = "Hello.py"
    if re.match( ".*/survol/entity.py.*", script_nam):
        if entity_type == "":
            top_url = uriRoot + "/entity.py"
        else:
            # Same as in objtypes.py
            if entity_id == "" or re.match("[a-zA-Z_]*=", entity_id):
                top_url = uriRoot + "/entity.py"
            else:
                top_url = EntityUri(entity_type, "")
    else:
        top_url = uriRoot + "/entity.py"
    return top_url

################################################################################


# This, because graphviz transforms a "\L" (backslash-L) into "<TABLE>". Example:
# http://127.0.0.1/PythonStyle/survol/entity.py?xid=com_type_lib:C%3A%5CWINDOWS%5Csystem32%5CLangWrbk.dll
# Or if the url contains a file in "App\Local"
def EncodeUri(an_str):
    # sys.stderr.write("EncodeUri str=%s\n" % str(anStr) )

    if an_str:
        strTABLE = an_str.replace("\\L", "\\\\L")
    else:
        strTABLE = ""

    # In Python 3, urllib.quote has been moved to urllib.parse.quote and it does handle unicode by default.
    if is_py3:
        return urllib_quote(strTABLE, '')
    else:

        # THIS SHOULD NORMALLY BE DONE. BUT WHAT ??
        ###strTABLE = strTABLE.replace("&","%26")
        # UnicodeDecodeError: 'ascii' codec can't decode byte 0xe9 in position 32
        return urllib_quote(strTABLE,'ascii')

################################################################################


def RequestUri():
    """This function returns the value of the environment variable REQUEST_URI.
    But, a minimal HTTP server such as CGIHTTPServer does not set REQUEST_URI.
    So its value is recreated with available values in other environment variables.
    """
    try:
        # Example: REQUEST_URI=/Survol/survol/print_environment_variables.py
        script = os.environ["REQUEST_URI"]
        #sys.stderr.write("RequestUri script=%s\n"%script)
    except KeyError:
        try:
            # For example SCRIPT_NAME=/survol/print_environment_variables.py
            script = os.environ['SCRIPT_NAME']

            # Normalizes "survol\sources_types/a_script.py" into "/survol/sources_types/a_script.py"
            # This is for the case of debugging a script from the command line: If the URL is also an event key,
            # it must be the same as the one set by the HTTP server.
            if not script.startswith("/"):
                script = "/" + script
            script = script.replace("\\", "/")

            try:
                # For example QUERY_STRING="xid=EURO%5CLONL00111310@process:16580"
                query_string = os.environ['QUERY_STRING']
                if query_string:
                    script += "?" + query_string
            except KeyError:
                script = "QUERY_STRING should be set in RequestUri()"
        except KeyError:
            script = "SCRIPT_NAME should be set in RequestUri()"
    return script

################################################################################


# This assumes that this file is at the top of "survol" package.
gblTopScripts = os.path.dirname(os.path.abspath(__file__))
sys.path.append(gblTopScripts)


def EntHostToIp(entity_host):
    """Depending on the category, entity_host can have several forms.
    The name is misleading because it returns a host name,
    which might or might not be an IP."""


    # WBEM: http://192.168.1.88:5988
    #       https://jdd:test@acme.com:5959
    #       http://192.168.1.88:5988
    # TODO: Not sure this will work with IPV6
    mtch_host_wbem = re.match("https?://([^/:]*).*", entity_host)
    if mtch_host_wbem:
        #sys.stderr.write("EntHostToIp WBEM=%s\n" % mtch_host_wbem.group(1) )
        return mtch_host_wbem.group(1)

    # WMI : \\MYHOST-HP
    mtch_host_wmi = re.match(r"\\\\([-0-9A-Za-z_\.]*)", entity_host)
    if mtch_host_wmi:
        #sys.stderr.write("EntHostToIp WBEM=%s\n" % mtch_host_wmi.group(1) )
        return mtch_host_wmi.group(1)

    # sys.stderr.write("EntHostToIp Custom=%s\n" % entity_host )
    return entity_host


# TODO: Coalesce with EntHostToIp
def EntHostToIpReally(entity_host):
    try:
        hostOnly = EntHostToIp(entity_host)
        return GlobalGetHostByName(hostOnly) # POSSIBLY VERY SLOW.
    except Exception:
        return hostOnly

################################################################################


def ParseXidLocal(xid):
    # A machine name can contain a domain name : "WORKGROUP\MYHOST-HP", the backslash cannot be at the beginning.
    # "WORKGROUP\MYHOST-HP@CIM_ComputerSystem.Name=Unknown-30-b5-c2-02-0c-b5-2"
    # "WORKGROUP\MYHOST-HP@oracle/table.Name=MY_TABLE"
    # BEWARE: This must NOT match "http://127.0.0.1:8000/survol/namespaces_wbem.py?xid=http://192.168.1.83:5988/."
    # that is "http://192.168.1.83:5988/."
    # A class name starts with a letter. There are no consecutives slashes "/".
    # TODO: Filter when consecutives slashes.
    mtch_entity = re.match(r"([-0-9A-Za-z_]*\\?[-0-9A-Za-z_\.]*@)?([a-zA-Z_][a-z0-9A-Z_/]*)\.(.*)", xid)

    if mtch_entity:
        if mtch_entity.group(1) == None:
            entity_host = ""
        else:
            entity_host = mtch_entity.group(1)[:-1]

        entity_type = mtch_entity.group(2)
        entity_id_quoted = mtch_entity.group(3)

        # Everything which comes after the dot which follows the class name.
        entity_id = urllib_unquote(entity_id_quoted)

        return entity_type, entity_id, entity_host

    return None


def ParseXidWMI(xid):
    # WMI : \\MYHOST-HP\root\cimv2:Win32_Process.Handle="0"
    # Beware ! On Windows, namespaces are separated by backslashes.
    # WMI : \\MYHOST-HP\root\cimv2:Win32_Process.Handle="0"
    # http://127.0.0.1:8000/survol/objtypes_wmi.py?xid=\\myhost-HP\root\CIMV2\Applications%3A.
    # http://127.0.0.1:8000/survol/class_wmi.py?xid=\\myhost-HP\root\CIMV2%3AWin32_PerfFormattedData_Counters_IPHTTPSGlobal.
    # http://127.0.0.1:8000/survol/entity_wmi.py?xid=\\MYHOST-HP\root\CIMV2%3AWin32_PerfFormattedData_Counters_IPHTTPSGlobal.Name%3D%22Default%22
    # TODO: BEWARE ! If the host name starts with a L, we have to "triplicate" the back-slash
    # TODO: otherwise graphviz replace "\L" par "<TABLE">

    # This matches for example 'root\cimv2:Win32_Process.Handle="0"'
    wmi_regex_local_part = r"([a-zA-Z0-9_]+)\\([^.]*)(\..*)"

    mtch_ent_wmi = re.match(r"\\\\\\?([-0-9A-Za-z_\.]*)\\" + wmi_regex_local_part, xid)
    if mtch_ent_wmi:
        grp = mtch_ent_wmi.groups()
        entity_host = grp[0]
        entity_type = grp[1] + "\\" + grp[2]
        entity_id_quoted = grp[3]
        # ( entity_host, entity_type, entity_id_quoted ) = grp
        if entity_id_quoted is None:
            entity_id = ""
            # sys.stderr.write("WMI Class Cimom=%s ns_type=%s\n" % ( entity_host, entity_type ))
        else:
            # Remove the dot which comes after the class name.
            entity_id = urllib_unquote(entity_id_quoted)[1:]
            # sys.stderr.write("WMI Object Cimom=%s ns_type=%s path=%s\n" % ( entity_host, entity_type, entity_id ))

        return entity_type, entity_id, entity_host

    # WMI : Maybe the host is missing, and implicitely the local machine.
    # http://127.0.0.1:8000/survol/class_type_all.py?xid=root\CIMV2:Win32_Process.
    mtch_ent_wmi = re.match(wmi_regex_local_part, xid)
    if mtch_ent_wmi:
        grp = mtch_ent_wmi.groups()
        entity_host = ""
        entity_type = grp[0] + "\\" + grp[1]
        entity_id_quoted = grp[2]
        if entity_id_quoted is None:
            entity_id = ""
            # sys.stderr.write("WMI Class Cimom=%s ns_type=%s\n" % ( entity_host, entity_type ))
        else:
            # Remove the dot which comes after the class name.
            entity_id = urllib_unquote(entity_id_quoted)[1:]
            # sys.stderr.write("WMI Object Cimom=%s ns_type=%s path=%s\n" % ( entity_host, entity_type, entity_id ))

        return entity_type, entity_id, entity_host

    return None


def ParseXidWBEM(xid):
    # https://jdd:test@acme.com:5959/cimv2:Win32_SoftwareFeature.Name="Havana",ProductName="Havana",Version="1.0"
    # http://192.168.1.88:5988/root/PG_Internal:PG_WBEMSLPTemplate
    # "http://127.0.0.1:8000/survol/namespaces_wbem.py?xid=http://192.168.1.83:5988/."
    # "xid=http://192.168.1.88:5988/."
    mtch_ent_wbem = re.match(r"(https?://[^/]*)/([^.]*)(\..*)?", xid)
    if mtch_ent_wbem:
        #sys.stderr.write("mtch_ent_wbem\n")
        grp = mtch_ent_wbem.groups()
        (entity_host, entity_type, entity_id_quoted) = grp
        # TODO: SAME LOGIC FOR THE TWO OTHER CASES !!!!!!!!!!!!!!
        if entity_id_quoted is None:
            entity_id = ""
            # sys.stderr.write("WBEM Class Cimom=%s ns_type=%s\n" % ( entity_host, entity_type ))
        else:
            # Remove the dot which comes after the class name.
            entity_id = urllib_unquote(entity_id_quoted)[1:]
            # sys.stderr.write("WBEM Object Cimom=%s ns_type=%s path=%s\n" % ( entity_host, entity_type, entity_id ))

        return entity_type, entity_id, entity_host

    return None


# This receives the xid value for, for example: "xid=@/:oracle_package."
# It parses this string into three components and returns the class,
# the concatenation of key=value pairs, and the host.
# BEWARE: This cannot work if the hostname contains a ":", see IPV6. MUST BE VERY FAST !!!
# TODO: Should also parse the namespace.
# ParseXid xid=CIM_ComputerSystem.Name=rchateau-HP
# ParseXid xid=CIM_ComputerSystem.Name=Unknown-30-b5-c2-02-0c-b5-2
def ParseXid(xid):
    # sys.stderr.write( "ParseXid xid=%s\n" % (xid) )

    # First, we try to match our terminology.
    # The type can be in several directories separated by slashes: "oracle/table"
    # If suffixed with "/", it means namespaces.

    entity_triplet = ParseXidLocal(xid)
    if entity_triplet:
        return entity_triplet

    # Apparently it is not a problem for the plain old entities.
    xid = urllib_unquote(xid)

    entity_triplet = ParseXidWMI(xid)
    if entity_triplet:
        return entity_triplet

    entity_triplet = ParseXidWBEM(xid)
    if entity_triplet:
        return entity_triplet

    # sys.stderr.write( "ParseXid=%s RETURNS NOTHING\n" % (xid) )
    return "", "", ""

################################################################################


# TODO: Would probably be faster by searching for the last "/".
# '\\\\MYHOST-HP\\root\\cimv2:Win32_Process.Handle="0"'  => "root\\cimv2:Win32_Process"
# https://jdd:test@acme.com:5959/cimv2:Win32_SoftwareFeature.Name="Havana",ProductName="Havana",Version="1.0"  => ""
def parse_namespace_type(ns_entity_type):
    # sys.stderr.write("ParseEntityType entity_type=%s\n" % ns_entity_type )
    nsSplit = ns_entity_type.split(":")
    if len(nsSplit) == 1:
        entity_namespace = ""
        entity_type = nsSplit[0]
    else:
        entity_namespace = nsSplit[0]
        entity_type = nsSplit[1]
    return entity_namespace, entity_type

################################################################################


def ScriptizeCimom(path, entity_type, cimom):
    """This is a adhoc solution, for a local use."""
    return uriRoot + path + "?" + EncodeEntityId(cimom + "/" + entity_type,"")


def Scriptize(path, entity_type, entity_id):
    """Properly encodes type and id into a URL."""
    return uriRoot + path + "?" + EncodeEntityId(entity_type, entity_id)

################################################################################


# TODO: Consider base64 encoding of all arguments, with "Xid="
# This would give the same encoding for all parameters whetever their class.
xidCgiDelimiter = "?xid="


def EntityClassUrl(entity_type, entity_namespace="", entity_host="", category=""):
    """This creates the URL of a class, "Survol, "WMI" or "WBEM"."""
    if entity_type is None:
        entity_type = ""

    # WBEM: https://jdd:test@acme.com:5959/cimv2:Win32_SoftwareFeature.Name="Havana",ProductName="Havana",Version="1.0"
    if category == "WBEM":
        moniker_class = entity_host + "/" + entity_namespace + ":" + entity_type + "."
    # WMI : \\MYHOST-HP\root\cimv2:Win32_Process.Handle="0"
    elif category == "WMI":
        moniker_class = "\\\\" + entity_host + "\\" + entity_namespace + ":" + entity_type + "."
    # This is temporary.
    else:
        # We could simplify the format, if no namespace nor hostname.
        moniker_class = ""
        if entity_host:
            moniker_class += entity_host + "@"
        # Should not happen.
        if entity_namespace:
            moniker_class += entity_namespace + "/:"
        moniker_class += entity_type + "."

    # TODO: See also EntityUrlFromMoniker.

    url = uriRoot + "/class_type_all.py" + xidCgiDelimiter + EncodeUri(moniker_class)
    return url


def EntityClassNode(entity_type, entity_namespace="", entity_host="", category=""):
    """This creates the node of a class, "Survol" (Default), "WMI" or "WBEM"."""
    url = EntityClassUrl(entity_type, entity_namespace, entity_host, category)

    # sys.stdout.write("EntityClassUrl url=%s\n" % url)
    return NodeUrl(url)


def KWArgsToEntityId(className, **kwargsOntology):
    """From key-value pairs, this builds an entity_id in the good property order."""
    entity_id = ""
    delim = ""
    keys_onto = OntologyClassKeys(className)

    # The dictionary is not properly ordered because it depends
    # on the Python version, and these data are given by a user application.

    for arg_key in keys_onto:
        try:
            argVal = kwargsOntology[arg_key]
        except KeyError:
            ERROR("KWArgsToEntityId className=%s. No key %s",className, arg_key)
            raise

        # TODO: The values should be encoded !!!
        entity_id += delim + "%s=%s" % (arg_key, argVal)
        delim = ","
    # The values might come from many different origins
    if not is_py3:
        if type(entity_id) == unicode:
            entity_id = entity_id.encode("utf-8")
    return entity_id


def EntityUri(entity_type, *entity_ids):
    return EntityUriDupl(entity_type, *entity_ids)


def EntityUriDupl(entity_type, *entity_ids, **extra_args):
    # sys.stderr.write("EntityUriDupl %s\n" % str(entity_ids))

    keys = OntologyClassKeys(entity_type)

    if len(keys) != len(entity_ids):
        WARNING("EntityUriDupl Different lens:%s and %s", str(keys), str(entity_ids))
    entity_id = ",".join("%s=%s" % pairKW for pairKW in zip(keys, entity_ids))
    
    # Extra arguments, differentiating duplicates.
    entity_id += "".join( ",%s=%s" % (extArg, extra_args[extArg]) for extArg in extra_args)

    url = Scriptize("/entity.py", entity_type, entity_id)
    return NodeUrl(url)

################################################################################


def EntityScriptFromPath(moniker_entity, is_class, is_namespace, is_hostname):
    """Probably not necessary because we apparently always know
    if we need a WMI, WBEM or custom scripts. Not urgent to change this."""
    if moniker_entity[0] == '\\':
        ent_idx = 0
    elif moniker_entity[0:4] == 'http':
        ent_idx = 1
    else:
        ent_idx = 2

    if is_hostname:
        return ('namespaces_wmi.py','namespaces_wbem.py','entity.py')[ent_idx]
    elif is_namespace:
        return ('objtypes_wmi.py','objtypes_wbem.py','objtypes.py')[ent_idx]
    elif is_class:
        return ('class_wmi.py','class_wbem.py','class_type_all.py')[ent_idx]
    else:
        return ('entity_wmi.py','entity_wbem.py','entity.py')[ent_idx]

# WMI, WBEM and Survol have the similar monikers.
# TODO: This should split the arguments and reformat them according to the class.
# TODO: This, because some parameters must be reformatted,
# TODO: for example CIM_ComputerSystem.Name must be in lowercase.
# TODO: The problem can be fixed by converting all hostnames to uppercase,
# TODO: but we must be sure that WBEM and WMI will follow the same standard.
# TODO: Probably same problem with CIM_DataFile on Windows because of backslashes
# TODO: as directory separator.
def EntityUrlFromMoniker(monikerEntity, is_class=False, is_namespace=False, is_hostname=False):
    script_path = EntityScriptFromPath(monikerEntity, is_class, is_namespace, is_hostname)

    # sys.stderr.write("EntityUrlFromMoniker script_path=%s\n"%script_path)
    url = uriRoot + "/" + script_path + xidCgiDelimiter + EncodeUri(monikerEntity)
    return url

################################################################################


def ComposeTypes(*hierarchical_entity_types):
    """Used to define subtypes."""
    # TODO: Find another solution more compatible with WBEM and WMI logic.
    return ".".join(hierarchical_entity_types)

################################################################################


def CopyFile( mime_type, file_name):
    """Read and write by chunks, so that it does not use all memory."""
    sys.stderr.write("CopyFile type globalOutMach=%s\n" % type(globalOutMach))

    fil_des = open(file_name, "rb")

    globalOutMach.HeaderWriter(mime_type)

    outFd = globalOutMach.OutStream()

    # https://stackoverflow.com/questions/2374427/python-2-x-write-binary-output-to-stdout
    if is_apache_server():
        if isPlatformWindows:
            if not is_py3:
                import os, msvcrt
                msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)

    # This is a bit tricky for WSGI if an error occurs:
    # The header must always be sent before the content, and once only.
    # os.environ["SERVER_SOFTWARE"] = "WSGIServer/0.2"
    while True:
        chunk = fil_des.read(1000000)
        if not chunk:
            break
        #  or os.environ["SERVER_SOFTWARE"].startswith("Apache/")
        if is_wsgi_server():
            #outFd.write(chunk.decode('latin1'))
            #outFd.write(chunk.decode('string_escape'))
            #outFd.write(u"chunk.encode()")
            try:
                outFd.write(chunk)
                #outFd.write(chunk.decode())
            except:
                # 'ascii' codec can't decode byte 0xf3 in position 1: ordinal not in range(128).
                outFd.write(u"Cannot display:%s" % file_name)
        else:
            outFd.write(chunk)

    outFd.flush()
    fil_des.close()

################################################################################


# By the way, when calling a RDF source, we should check the type of the
# MIME document and if this is not RDF, the assumes it's an error 
# which must be displayed.
# This is used as a HTML page but also displayed in Javascript in a DIV block.
# TODO: Change this for WSGI.
def InfoMessageHtml(message):
    #gblLogger.warning("InfoMessageHtml:%s",message)
    globalOutMach.HeaderWriter("text/html")

    #gblLogger.debug("InfoMessageHtml:Sending content")
    WrtAsUtf(
        "<html><head><title>Error: Process=%s</title></head>"
        % str(os.getpid()))

    WrtAsUtf("<body>")

    WrtAsUtf("<b>" + message + "</b><br>")

    # On Linux it says: "OSError: [Errno 2] No such file or directory"
    WrtAsUtf('<table>')

    if is_py3:
        WrtAsUtf("<tr><td>Login</td><td>%s</td></tr>"%os.getlogin())

    WrtAsUtf("<tr><td>Cwd</td><td>%s</td></tr>" % os.getcwd())
    WrtAsUtf("<tr><td>OS</td><td>%s</td></tr>"%sys.platform)
    WrtAsUtf("<tr><td>Version</td><td>%s</td></tr>"%sys.version)
    
    WrtAsUtf('</table>')

    # http://desktop-ni99v8e:8000/survol/www/configuration.htm
    configUrl = uriRoot + "/edit_configuration.py"
    WrtAsUtf('<a href="%s">Setup</a>.<br>'%configUrl)
    envsUrl = uriRoot + "/print_environment_variables.py"
    WrtAsUtf('<a href="%s">Environment variables</a>.<br>'%envsUrl)
    homeUrl = TopUrl("", "")
    WrtAsUtf('<a href="%s">Return home</a>.<br>'%homeUrl)

    WrtAsUtf("""
    </body></html>
    """)
    gblLogger.debug("InfoMessageHtml:Leaving")

################################################################################


# Returns the list of available object types: ["process", "file," group", etc...]
def ObjectTypesNoCache():
    directory = gblTopScripts + "/sources_types"
    DEBUG("ObjectTypesNoCache directory=%s", directory)

    ld = len(directory)
    for path, dirs, files in os.walk(directory):
        if len(path) == ld:
            prefix = ""
        else:
            prefix = path[ld +1:].replace("\\","/") + "/"
        for one_dir in dirs:
            if one_dir != "__pycache__":
                yield prefix + one_dir


_gbl_object_types = None


# TODO: Should concatenate this to localOntology. Default value is "Id".
def ObjectTypes():
    global _gbl_object_types

    if _gbl_object_types is None:
        _gbl_object_types = set(ObjectTypesNoCache())
        # sys.stderr.write("ObjectTypes glbObjectTypes="+str(glbObjectTypes)+"\n")

    return _gbl_object_types

################################################################################

# These functions are used in scripts, to tell if it is usable or not.

isPlatformLinux = 'linux' in sys.platform
isPlatformWindows = 'win' in sys.platform


def UsableLinux(entity_type, entity_ids_arr):
    """Linux only"""
    return isPlatformLinux


def UsableWindows(entity_type, entity_ids_arr):
    """Windows only"""
    return isPlatformWindows


# Tells if a file is executable code or library.
# TODO: This function should be moved to CIM_DataFile/__init__.py
def UsableWindowsBinary(entity_type, entity_ids_arr):
    """Windows executable or code file"""
    if not UsableWindows(entity_type, entity_ids_arr):
        return False
    full_file_name = entity_ids_arr[0]
    if os.path.isdir(full_file_name):
        return False
    filename, file_extension = os.path.splitext(full_file_name)
    # TODO: Must add library type for ELF and PE ?
    return file_extension.upper() in [".EXE", ".DLL", ".COM", ".OCX", ".SYS", ".ACM", ".BPL", ".DPL"]


# Applies for nm, dll, elftools.
def UsableLinuxBinary(entity_type, entity_ids_arr):
    """Linux executable or code file"""
    if not UsableLinux(entity_type, entity_ids_arr):
        return False
    full_file_name = entity_ids_arr[0]
    if os.path.isdir(full_file_name):
        return False
    filename, file_extension = os.path.splitext(full_file_name)
    # TODO: Must add library type for ELF and PE ?
    if file_extension in [".so", ".lib"]:
        return True
    # TODO: Finish this. Use "magic" module ??
    return True
    

def is_snapshot_behaviour():
    """Used by scripts named like events_generator_*.py which can write a continuous flow of events.
    They also must be able to run in snapshot mode, by default, and return RDF triples."""
    sys.stderr.write("is_snapshot_behaviour QUERY_STRING=%s\n" % os.environ["QUERY_STRING"])
    return "mode=" + "daemon" not in os.environ["QUERY_STRING"]

################################################################################


# For example gFuncName="Graphic_shape" etc... This seeks for a function in this name.
# This searches in several modules, starting with the module of the entity,
# then the upper module etc...
def HierarchicalFunctionSearchNoCache(typeWithoutNS, gFuncName):

    # for the first loop it takes the entire string.
    last_dot = len(typeWithoutNS)
    while last_dot > 0:

        topModule = typeWithoutNS[:last_dot]
        chopped_entity_type = typeWithoutNS[:last_dot]

        # Load the module of this entity to see if it defines the graphic function.
        entity_module = GetEntityModule(chopped_entity_type)

        if entity_module:
            try:
                gFuncAddr = getattr(entity_module,gFuncName)
                return gFuncAddr
            except AttributeError:
                pass

        # Then try the upper level module.
        last_dot = typeWithoutNS.rfind(".",0,last_dot)

    return None


# This caches the result of HierarchicalFunctionSearchNoCache()
_dict_hierarchical_function_search = {}


# TODO: This is similar to Python inheritance.
# TODO: Reuse the CIM hierarchy of classes.
# TODO: Difficulty is that all scripts must be changed.
# TODO: This is discussed here:
# https://softwareengineering.stackexchange.com/questions/298019/how-to-achieve-inheritance-when-using-just-modules-and-vanilla-functions-in-pyth
def HierarchicalFunctionSearch(type_without_ns, function_name):
    global _dict_hierarchical_function_search
    # Safety check.
    if type_without_ns.find(".") >= 0:
        raise "HierarchicalFunctionSearch Invalid type_without_ns=%s" % type_without_ns

    type_without_ns = type_without_ns.replace("/", ".")

    try:
        return _dict_hierarchical_function_search[function_name][type_without_ns]
    except KeyError:
        funcObj = HierarchicalFunctionSearchNoCache(type_without_ns, function_name)
        try:
            _dict_hierarchical_function_search[function_name][type_without_ns] = funcObj
        except KeyError:
            _dict_hierarchical_function_search[function_name] = {type_without_ns : funcObj}
        return funcObj


################################################################################

# This describes for each entity type, the list of parameters names needed
# to define an object of this class. For example:
# "dbus/connection"     : ( ["Bus","Connect"], ),
# "dbus/interface"      : ( ["Bus","Connect","Obj","Itf"], ),
# "symbol"              : ( ["Name","File"], ), # Must be defined here, not in the module.
_local_ontology = {
}

# The key must match the DMTF standard. It might contain a namespace.
# TODO: Replace this by a single lookup in a single dict
# TODO: ... made of localOntology added to the directory of types.
#
# NOTE: WMI flags the keys attributes, which are part of the path.
# https://docs.microsoft.com/en-us/windows/desktop/WmiSdk/key-qualifier
# The Key qualifier indicates whether the property is part of the namespace handle.
# If more than one property has the Key qualifier,
# then all such properties collectively form the key (a compound key).
# When taken together, the key properties must supply a unique reference for each class instance.
#
# OpenLMI stores the definition of files in XML format:
# /var/lib/Pegasus/repository/root#cimv2/classes/CIM_Process.CIM_EnabledLogicalElement
#
# <PROPERTY NAME="Handle"  CLASSORIGIN="CIM_Process" TYPE="string">
#   <QUALIFIER NAME="Key" TYPE="boolean" OVERRIDABLE="false">
#     <VALUE>TRUE</VALUE>
#   </QUALIFIER>
# </PROPERTY>
#
# OpenPegasus does not do the same. See the file: CIM_Process.mof : All attributes are keys
#      [Key, Description ( "..."),
#       MappingStrings { "MIF.DMTF|Process Information|001.1" }]
#   string Handle;
#
# Conclusions:
# - There is no unique representation of the CIM classes.
# - They do not represent keys the same way.
# => Conclusion: It makes sense that Survol has its own representation.
#


def OntologyClassKeys(entity_type):
    # sys.stderr.write("OntologyClassKeys entity_type=%s Caller=%s\n"%(entity_type, sys._getframe(1).f_code.co_name))

    try:
        # TODO: If cannot find it, load the associated module and retry.
        return _local_ontology[entity_type][0]
    except KeyError:
        pass

    # Maybe the ontology is defined in the related module if it exists.
    entity_module = GetEntityModule(entity_type)
    if     entity_module:
        try:
            entity_ontology_all = entity_module.EntityOntology()
            _local_ontology[entity_type] = entity_ontology_all
            # sys.stderr.write("OntologyClassKeys entity_type=%s loaded entity_ontology_all=%s\n" % (entity_type,str(entity_ontology_all)))
            return entity_ontology_all[0]
        except AttributeError:
            pass

    # It does not have a ontology, so it is a domain.
    _local_ontology[entity_type] = ([],)
    return []


def EntityIdToArray(entity_type, entity_id):
    """Used for calling ArrayInfo. The order of arguments is strictly the ontology's.
    It extracts the values of the ontology parameters and returns them in a list."""
    onto_keys = OntologyClassKeys(entity_type)
    #sys.stderr.write("lib_util.EntityIdToArray entity_type=%s entity_id=%s\n"%(entity_type,entity_id))
    dict_ids = SplitMoniker(entity_id)
    # sys.stderr.write("EntityIdToArray dict_ids=%s\n" % ( str(dict_ids) ) )
    # For the moment, this assumes that all keys are here.
    # Later, drop this constraint and allow WQL queries.
    try:
        def DecodeCgiArg(a_key):
            #sys.stderr.write("DecodeCgiArg aKey=%s type=%s dict_ids=%s\n"%(aKey,type(aKey),str(dict_ids)))
            a_val_raw = dict_ids[a_key]
            try:
                val_decod = a_key.ValueDecode(a_val_raw)
                return val_decod
            except AttributeError:
                return a_val_raw
        return [ DecodeCgiArg( aKey ) for aKey in onto_keys ]
    except KeyError:
        gblLogger.error("EntityIdToArray missing key: type=%s id=%s onto=%s", entity_type , entity_id, str(onto_keys) )
        raise


################################################################################


# Adds a key value pair at the end of the url with the right delimiter.
# TODO: Checks that the argument is not already there.
# TODO: Most of times, it is used for changing the mode.
def _concatenate_cgi_argument(url, keyvalpair):
    if url.rfind( '?' ) == -1:
        return url + "?" + keyvalpair
    else:
        return url + "&" + keyvalpair


# This is very primitive and maybe should be replaced by a standard function,
# but lib_util.EncodeUri() replaces "too much", and SVG urls cannot encode an ampersand...
# The problems comes from "&mode=edit" or "&mode=html" etc...
# TODO: If we can fix this, then "xid" can be replaced by "entity_type/entity_id"
def UrlToSvg(url):
    return url.replace("&", "&amp;amp;")


def UrlNoAmp(url):
    return url.replace("&amp;", "&").replace("&amp;", "&")

################################################################################


# In an URL, this replace the CGI parameter "http://....?mode=XXX" by "mode=YYY".
# If there is no such parameter, then it is removed. If the input parameter is
# an empty string, then it is removed from the URLs.
# Used for example as the root in entity.py, obj_types.py and class_type_all.py.
def request_uri_with_mode(other_mode):
    # When in merge_scripts.py for merging several scripts,
    # the request uri is prefixed by a host:
    # HttpPrefix()=http://myhost-hp:8000
    # RequestUri()=http://myhost-hp:80/Survol/survol/entity.py?xid=CIM_Process.Handle=1900
    str_request_uri = RequestUri()

    # strRequestUri=/survol/print_internal_data_as_json.py
    if str_request_uri.startswith("http"):
        script = str_request_uri
    else:
        script = HttpPrefix() + str_request_uri
    return url_mode_replace(script, other_mode)


def url_mode_replace(script, other_mode):
    """ In an Url, replaces, removes or adds the value of the argument "mode".
    This is a key argument for CGI scripts. """

    mtch_url = re.match(r"(.*)([\?\&])mode=[^\&]*(.*)", script)

    if other_mode:
        if mtch_url:
            updated_url = mtch_url.group(1) + mtch_url.group(2) + "mode=" + other_mode + mtch_url.group(3)
        else:
            updated_url = _concatenate_cgi_argument(script, "mode=" + other_mode)
    else:
        # We want to remove the mode.
        if mtch_url:
            if mtch_url.group(2) == '?':
                # "mode" IS the first argument.
                if mtch_url.group(3):
                    updated_url = mtch_url.group(1) + "?" + mtch_url.group(3)[1:]
                else:
                    updated_url = mtch_url.group(1)
            else:
                # "mode" is NOT the first argument.
                updated_url = mtch_url.group(1) + mtch_url.group(3)
        else:
            # Nothing to do because it has no cgi arguments.
            updated_url = script

    # TODO: PROBLEMS IF THE URL CONTAINS BACKSLASHES SUCH AS HERE:
    # "http://127.0.0.1:8000/survol/sources_types/CIM_DataFile/file_stat.py?xid=CIM_DataFile.Name%3DC%3A\Program%20Files%20%28x86%29\NETGEAR\WNDA3100v3\WNDA3100v3.EXE"
    return updated_url


def RootUri():
    calling_url = request_uri_with_mode("")
    calling_url = calling_url.replace("&", "&amp;")
    DEBUG("RootUri calling_url=%s", calling_url)
    return NodeUrl(calling_url)

################################################################################

# https://developer.mozilla.org/fr/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types
def get_url_mode(url):
    """Extracts the mode from an URL."""

    # Maybe it contains a MIME type: application/java-archive,
    # application/vnd.ms-powerpoint, audio/3gpp2, application/epub+zip

    mtch_url = re.match(r".*[\?\&]mode=([^\&]*).*", url)
    if mtch_url:
        return mtch_url.group(1)
    return ""


# The display mode can come from the previous URL or from a CGI environment.
def GuessDisplayMode():
    arguments = cgi.FieldStorage()
    try:
        try:
            mode = arguments["mode"].value
        except AttributeError:
            # In case there are several mode arguments,
            # hardcode to "info". Consequence of a nasty Javascript bug.
            mode = "info"
        if mode != "":
            return mode
    except KeyError:
        pass

    try:
        # HTTP_REFERER=http://127.0.0.1/PythonStyle/print.py?mode=xyz
        referer = os.environ["HTTP_REFERER"]
        mode_referer = get_url_mode(referer)
        # If we come from the edit form, we should not come back to id.
        # TODO: HOW CAN WE COME BACK TO THE FORMER DISPLAY MODE ??
        if mode_referer != "":
            if mode_referer == "edit":
                # TODO: Should restore the original edit mode.
                # enter_edition_mode
                return ""
            else:
                return mode_referer

    except KeyError:
        pass

    try:
        # When called from another module, cgi.FieldStorage might not work.
        script = os.environ["SCRIPT_NAME"]
        mode = get_url_mode( script )
        if mode != "":
            return mode
    except KeyError:
        pass

    mode = ""
    return mode

################################################################################


def SplitMoniker(xid):
    """The input string is an entity_id: "key1=val1&key2=val2&key3=val3",
    i.e. what comes after "xid=<class>." in an object URL.
    This returns a dictionary of key-values.
    """

    splt_lst = re.findall(r'(?:[^,"]|"(?:\\.|[^"])*")+', xid)

    # sys.stderr.write("SplitMoniker splt_lst=%s\n" % ";".join(splt_lst) )

    resu = dict()
    for splt_wrd in splt_lst:
        mtch_equal_quote = re.match(r'([A-Z0-9a-z_]+)="(.*)"', splt_wrd)
        if mtch_equal_quote:
            # If there are quotes, they are dropped.
            resu[mtch_equal_quote.group(1)] = mtch_equal_quote.group(2)
        else:
            mtch_equal_no_quote = re.match(r'([A-Z0-9a-z_]+)=(.*)', splt_wrd)
            if mtch_equal_no_quote:
                resu[mtch_equal_no_quote.group(1)] = mtch_equal_no_quote.group(2)

    # sys.stderr.write("SplitMoniker resu=%s\n" % str(resu) )

    return resu


def SplitMonikToWQL(moniker_to_split, class_name):
    """Builds a WQL (WMI Query Language) query from a Moniker.
    This allows to search for an object in the CIM repository,
    whatever the attribute values are, or if it is a Survol object."""
    gblLogger.debug("SplitMonikToWQL splitMonik=[%s]", str(moniker_to_split))
    a_qry = 'select * from %s' % class_name
    qry_delim = "where"
    for qry_key in moniker_to_split:
        qry_val = moniker_to_split[qry_key]
        a_qry += ' %s %s="%s"' % (qry_delim, qry_key, qry_val)
        qry_delim = "and"

    DEBUG("Query=%s", a_qry)
    return a_qry


def Base64Encode(input_text):
    if is_py3:
        if isinstance(input_text, bytes):
            txtToB64Encode = input_text
        else:
            txtToB64Encode = input_text.encode('utf-8')
        return base64.urlsafe_b64encode(txtToB64Encode).decode('utf-8')
    else:
        return base64.urlsafe_b64encode(input_text)


def Base64Decode(input_text):
    # The padding might be missing which is not a problem:
    # https://stackoverflow.com/questions/2941995/python-ignore-incorrect-padding-error-when-base64-decoding
    missing_padding = len(input_text) % 4

    try:
        if is_py3:
            if missing_padding != 0:
                input_text += '=' * (4 - missing_padding)
            resu = base64.urlsafe_b64decode(input_text.encode('utf-8')).decode('utf-8')
        else:
            if missing_padding != 0:
                input_text += b'=' * (4 - missing_padding)
            resu = base64.urlsafe_b64decode(str(input_text))
        return resu
    except Exception as exc:
        gblLogger.error("CANNOT DECODE: symbol=(%s):%s", input_text, str(exc))
        return input_text + ":" + str(exc)


def split_url_to_entity(calling_url):
    """This receives an URL and parses it.
    Input examples:
    http://LOCAL_MODE:80/LocalExecution/sources_types/Win32_UserAccount/Win32_NetUserGetGroups.py?xid=Win32_UserAccount.Domain%3Dthe_machine%2CName%3Drchateau"
    "http://the_machine:8000/survol/sources_types/CIM_Directory/doxygen_dir.py?xid=CIM_Directory.Name%3DD%3A"
    """

    parse_url = survol_urlparse(calling_url)
    query = parse_url.query

    params = parse_qs(query)

    xidParam = params['xid'][0]
    (entity_type, entity_id, entity_host) = ParseXid(xidParam)
    entity_id_dict = SplitMoniker(entity_id)

    return parse_url.path, entity_type, entity_id_dict


################################################################################


# Different stream behaviour due to string vs binary.
if is_py3:
    outputHttp = sys.stdout.buffer
else:
    outputHttp = sys.stdout

################################################################################

# This is for WSGI compatibility.
class OutputMachineCgi:
    def __init__(self):
        pass

    def HeaderWriter(self, mime_type, extra_arguments=None):
        gblLogger.debug("OutputMachineCgi.WriteHeadContentType:%s", mime_type)
        HttpHeaderClassic(outputHttp, mime_type, extra_arguments)

    def OutStream(self):
        return outputHttp


# WSGI changes this to another object with same interface.
# Overriden in wsgiserver.py.
globalOutMach = OutputMachineCgi()

################################################################################

# This parameter in the display page of an object (entity.py),
# indicates if all scripts which can operate on this object, must be displayed,
# whether they can work or not.
# By default, it is False.
# For example, a script running on a Windows platform should not be displayed
# when running on Linux. Or if a specific Python module is needed,
# scripts using it should not be displayed. Same if the script has a syntax
# error. By setting this flag, it is easy to understand which scripts
# could be used and why they are not displayed.
paramkeyShowAll = "Show all scripts"

################################################################################


# Default destination for the RDF, HTML or SVG output.
def get_default_output_destination():
    return globalOutMach.OutStream()


def DfltOutMach():
    return globalOutMach


def SetGlobalOutMach(outmach_something):
    global globalOutMach
    globalOutMach = outmach_something


# environ["SERVER_SOFTWARE"] = "WSGIServer/0.2"
# This must be calculated each time because the WSGI server sets this environment
# variable when an URL is loaded, after module init.
is_wsgi_server_data = None
def is_wsgi_server():
    global is_wsgi_server_data
    if not is_wsgi_server_data:
        try:
            is_wsgi_server_data = os.environ["SERVER_SOFTWARE"].startswith("WSGIServer")
        except KeyError:
            # FIXME: This is wrong, because it assumes that default server is WSGI.
            is_wsgi_server_data = "DefaultServerSoftware"
    return is_wsgi_server_data


def is_apache_server():
    return os.environ["SERVER_SOFTWARE"].startswith("Apache/")


# Depending if the stream is a socket, a file or standard output,
# if Python 2 or 3, Windows or Linux, some complicated tests or conversions are needed.
# This writes to:
# - Apache socket.
# - WSGI stream.
# - lib_client stream.
# - CGI output.
# FIXME: Should always send bytes (Py3) or str (Py2)
def WrtAsUtf(input_str):
    my_output_stream = get_default_output_destination()
    try:
        my_output_stream.write(input_str)
    except:
        try:
            my_output_stream.write(input_str.encode('latin1'))
        except Exception as exc:
            sys.stderr.write("WrtAsUtf type=%s my_output_stream=%s caught %s\n"
                             % (type(input_str), type(my_output_stream), exc))


def SetDefaultOutput(wFile):
    """For asynchronous display."""
    # TODO: NEVER TESTED, JUST TEMP SYNTAX FIX.
    sys.stderr.write("SetDefaultOutput NEVER TESTED\n")
    global outputHttp
    outputHttp = wFile


# contentType = "text/rdf", "text/html", "image/svg+xml", "application/json" etc...
def HttpHeaderClassic(out_dest, content_type, extra_args=None):
    # sys.stderr.write("HttpHeader:%s\n"%content_type)
    # TODO: out_dest should always be the default output.

    stri = "Content-Type: " + content_type + "; charset=utf-8\n"
    if extra_args:
        # extra_args in a array of key-value tuples.
        # The order is preserved, and the same property can appear several times.
        for key_value in extra_args:
            stri += "%s: %s\n" % (key_value[0], key_value[1])
    stri += "\n"

    # Python 3.2
    try:
        # WSGI server and Py3, Apache and Py2.
        out_dest.write(stri)
        return
    except TypeError:
        pass
    # This is needed for the CGI server and Python 3.
    out_dest.write(stri.encode())


def WrtHeader(mimeType, extraArgs=None):
    globalOutMach.HeaderWriter(mimeType, extraArgs)

################################################################################


def _get_entity_module_without_cache_no_catch(entity_type):
    # Here, we want: "sources_types/Azure/location/__init__.py"
    # Example: entity_type = "Azure.location"
    # This works.
    # entity_module = importlib.import_module( ".subscription", "sources_types.Azure")

    entity_type_split = entity_type.split("/")
    if len(entity_type_split) > 1:
        entity_package = "sources_types." + ".".join(entity_type_split[:-1])
        entity_name = "." + entity_type_split[-1]
    else:
        entity_package = "sources_types"
        entity_name = "." + entity_type
    # sys.stderr.write("Loading from new hierarchy entity_name=%s entity_package=%s\n:"%(entity_name,entity_package))
    if (sys.platform.startswith("win") and sys.version_info >= (3, 2) and sys.version_info < (3, 3) ) \
    or (sys.platform.startswith("lin") and sys.version_info >= (3, 2) ):
        entity_module = importlib.import_module(entity_package + entity_name)
    else:
        entity_module = importlib.import_module(entity_name, entity_package)
    # sys.stderr.write("Loaded OK from new hierarchy entity_name=%s entity_package=%s\n:"%(entity_name,entity_package))
    return entity_module


def _get_entity_module_without_cache(entity_type):
    # sys.stderr.write("_get_entity_module_without_cache entity_type=%s\n"%entity_type)

    # Temporary hack to avoid an annoying warning message.
    if entity_type == "provider_script":
        return None

    try:
        return _get_entity_module_without_cache_no_catch(entity_type)
    except ImportError as exc:
        gblLogger.error("_get_entity_module_without_cache entity_type=%s Caught:%s", entity_type, exc)

        import traceback
        gblLogger.error("Exception stack trace:%s", traceback.format_exc())
        gblLogger.error("Current   stack trace:%s", "\n".join(traceback.format_stack()))

        return None


# So we try to load only once.
_cache_entity_to_module = {"": None}


# If it throws, the exception is not hidden.
# If it does not throw, then try to load the module.
def GetEntityModuleNoCatch(entity_type):
    # sys.stderr.write("_get_entity_module_without_cache entity_type=%s\n"%entity_type)

    # Do not throw KeyError exception.
    if entity_type in _cache_entity_to_module:
        return _cache_entity_to_module[ entity_type]

    entity_module = _get_entity_module_without_cache_no_catch(entity_type)
    _cache_entity_to_module[ entity_type] = entity_module
    return entity_module


# Maybe we could return an array because of heritage ?
# Or:  GetEntityModuleFunction(entity_type,functionName):
# ... which would explore from bottom to top.
def GetEntityModule(entity_type):
    # sys.stderr.write("PYTHONPATH="+os.environ['PYTHONPATH']+"\n")
    # sys.stderr.write("sys.path="+str(sys.path)+"\n")
    # sys.stderr.write("GetEntityModule entity_type=%s Caller=%s\n"%(entity_type, sys._getframe(1).f_code.co_name))

    try:
        # Might be None if the module does not exist.
        return _cache_entity_to_module[entity_type]
    except KeyError:
        pass
    entity_module = _get_entity_module_without_cache(entity_type)
    _cache_entity_to_module[entity_type] = entity_module
    return entity_module


def GetScriptModule(current_module, fil):
    """This loads a script as a module. Example:
    currentModule="sources_types.win32" fil="enumerate_top_level_windows.py" """
    if not fil.endswith(".py"):
        ERROR("GetScriptModule module=%s fil=%s not a Python script", current_module, fil)
        return None
    file_base_name = fil[:-3] # Without the ".py" extension.
    if is_py3:
        # Example: importlib.import_module("sources_top.Databases.mysql_processlist")
        #DEBUG("currentModule=%s fil=%s subClass=%s",currentModule,fil,subClass)
        if current_module:
            imported_mod = importlib.import_module(current_module + "." + file_base_name)
        else:
            imported_mod = importlib.import_module(file_base_name)
    else:
        if current_module:
            DEBUG("GetScriptModule file_base_name=%s currentModule=%s", file_base_name, current_module)
            imported_mod = importlib.import_module("." + file_base_name, current_module)
        else:
            imported_mod = importlib.import_module(file_base_name)
    return imported_mod

################################################################################


def FromModuleToDoc(imported_module, fil_default_text):
    """
        Returns the doc string of a module as a literal node. Possibly truncated
        so it can be displayed.
    """
    try:
        doc_modu_all = imported_module.__doc__
        if doc_modu_all:
            doc_modu_all = doc_modu_all.strip()
        # Take only the first non-empty line.
        doc_modu_split = doc_modu_all.split("\n")
        doc_modu = None
        for doc_modu in doc_modu_split:
            if doc_modu     :
                # sys.stderr.write("DOC="+docModu)
                max_len = 40
                if len(doc_modu) > max_len:
                    doc_modu = doc_modu[0:max_len] + "..."
                break
    except:
        doc_modu = ""

    if not doc_modu:
        # If no doc available, just transform the file name.
        doc_modu = fil_default_text.replace("_", " ").capitalize()

    node_module = NodeLiteral(doc_modu)

    return node_module


# This creates a non-clickable node. The text is taken from __doc__ if it exists,
# otherwise the file name is beautifuled.
def DirDocNode(arg_dir, the_dir):
    # sys.stderr.write("DirDocNode arg_dir=%s dir=%s\n"%(arg_dir,dir))
    full_module = arg_dir + "." + the_dir

    try:
        imported_mod = importlib.import_module(full_module)
    except ImportError:
        return None

    # Add three characters otherwise it is truncated just like a Python file extension.
    return FromModuleToDoc(imported_mod, the_dir)


def AppendNotNoneHostname(script, hostname):
    str_url = uriRoot + script
    if hostname:
        # The string "portal" is just there to have a nice title.
        str_url += xidCgiDelimiter + hostname + "@portal."
    return str_url


def UrlPortalWbem(hostname=None):
    """Point to the WBEM portal for a given machine."""
    str_url = AppendNotNoneHostname('/portal_wbem.py',hostname)
    gblLogger.debug("UrlPortalWbem str_url=%s", str_url)
    node_portal = NodeUrl(str_url)
    return node_portal


def UrlPortalWmi(hostname=None):
    """Point to the WMI portal for a given machine."""
    str_url = AppendNotNoneHostname('/portal_wmi.py', hostname)
    node_portal = NodeUrl(str_url)
    return node_portal


# This is used to split a string made of several lines separated by a "\n",
# following multi-line DocString convention.
# "Multi-line docstrings consist of a summary line just like a one-line docstring,
# followed by a blank line, followed by a more elaborate description.
# The summary line may be used by automatic indexing tools;
# it is important that it fits on one line and is separated from the rest of the docstring by a blank line.
# The summary line may be on the same line as the opening quotes or on the next line.
# The entire docstring is indented the same as the quotes at its first line (see example below)."
# The only difference is that the blank line is not needed, but can be there.
def SplitTextTitleRest(title):
    title_split = title.strip().split("\n")

    page_title_first = title_split[0].strip()
    page_title_rest = " ".join( title_split[1:] ).strip()

    return (page_title_first,page_title_rest)


def TimeStamp():
    return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())


# TODO: This should fetch more information from WMI or WBEM ?
# TODO: Check for double insertion.
# domainPredicate and rangePredicate are class names.
# TODO: The data type is not correct.
def AppendPropertySurvolOntology(name_predicate, description_predicate, domain_predicate, range_predicate, map_attributes):
    if range_predicate:
        dataType = None
    else:
        dataType = "string"
    if not description_predicate:
        description_predicate = "Predicate %s" % name_predicate
    if name_predicate in map_attributes:
        map_attributes[name_predicate]["predicate_domain"].append(domain_predicate)
    else:
        map_attributes[name_predicate] = {
            "predicate_type": dataType,
            "predicate_description": description_predicate,
            "predicate_domain" : [domain_predicate],
            "predicate_range" : range_predicate }


# TODO: Should we get classes and properties descriptions from WMI and WBEM ?
def AppendClassSurvolOntology(entity_type, map_classes, map_attributes):

    # This receives a class name from Survol and translates it into a CIM class name.
    # If this is a top-level class, then it is the same string.
    # If this is hierarchical, there might be duplicates.
    # To make thing simpler, slashes are translated into a dot.
    # NOTE: A difference between Survol and CIM, is that survols carries
    # the hierarchy of classes in their names, just like files.
    def SurvolClassToCIM(name_survol_class):
        return name_survol_class.replace("/", ".")

    idx = 0
    base_class = ""
    # Iteration on the base classes starting from the top.
    while idx >= 0:
        next_slash = entity_type.find("/", idx + 1)
        if next_slash == -1:
            intermed_type = entity_type
        else:
            intermed_type = entity_type[:next_slash]

        baseClassNameCIM = SurvolClassToCIM(base_class)
        classNameCIM = SurvolClassToCIM(intermed_type)

        try:
            # This reloads all classes without cache because if it does not load
            # we want to see the error message.
            entity_module = GetEntityModuleNoCatch(entity_type)
            ent_doc = entity_module.__doc__
            if ent_doc:
                ent_doc = ent_doc.strip()
            else:
                ent_doc = "No %s module documentation" % entity_type
        except:
            exc = sys.exc_info()[1]
            ent_doc = "Error:"+str(exc)

        if classNameCIM == "":
            raise Exception("Empty class in AppendClassSurvolOntology: entity_type=%s", entity_type)
        map_classes[classNameCIM] = {"base_class": baseClassNameCIM, "class_description": ent_doc}

        onto_list = OntologyClassKeys(entity_type)
        # We do not have any other information about ontology keys.
        for onto_key in onto_list:
            AppendPropertySurvolOntology(onto_key, "Ontology predicate %s"%onto_key, classNameCIM, None, map_attributes)

        idx = next_slash


def ExtractSurvolOntology():
    """This iterates on all the classes defined by Survol files tree,
    and returns two dictionaries which define classes and predicates,
    compatible with insertion into RDF triplestore."""
    map_classes = {}
    map_attributes = {}

    for entity_type in ObjectTypes():
        AppendClassSurvolOntology(entity_type, map_classes, map_attributes)

    return map_classes, map_attributes

##################################################################################


# FIXME: Survol scripts are not able to return objects,
# FIXME: but they natively create triples which can be fed into the graph:
# FIXME: Survol scripts just return the minimum set of data allowing to join with other clauses.
#
# FIXME: On the other hand, WMI returns objects but cannot natively create RDF triples.
# FIXME: Therefore, it makes sense to create triples from the objects.
def PathAndKeyValuePairsToRdf(grph, subject_path, dict_key_values):
    # Maybe this is a test mode.
    if not grph:
        return
    subject_path_node = NodeUrl(subject_path)

    for key, val in dict_key_values.items():
        grph.add((subject_path_node, key, val))


