
""" Generic module for any kind of Python UI, as distinct from the classes these derive from which contains 
stuff also applicable even without this """

import scriptengine, replayer, definitions, encodingutils
import os, sys, logging, subprocess, time, re
from gridformatter import GridFormatter
from itertools import izip
from random import choice

try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

from traceback import format_exception

# We really need our ConfigParser to be ordered, copied the one from 2.6 into the repository
if sys.version_info[:2] >= (2, 6):
    from ConfigParser import ConfigParser, ParsingError #@UnusedImport
else: # pragma: no cover - not currently running older than 2.5 in regular tests
    from ConfigParser26 import ConfigParser, ParsingError #@Reimport
    
class WidgetAdapter:
    adapterClass = None
    secondaryIdentifiers = [ "Context", "Dialog" ]
    @staticmethod
    def setAdapterClass(adapterCls):
        WidgetAdapter.adapterClass = adapterCls
        
    @classmethod
    def adapt(cls, widget):
        return cls.adapterClass(widget)
    
    def __init__(self, widget):
        self.widget = widget

    def __getattr__(self, name):
        return getattr(self.widget, name)

    def __hash__(self):
        return hash(self.widget)

    def __cmp__(self, other):
        return cmp(self.widget, other)

    def getTitle(self):
        try:
            return self.getWidgetTitle()
        except AttributeError:
            return ""

    def getChildren(self):
        return map(self.adapt, self.getChildWidgets())

    def getType(self):
        return self.widget.__class__.__name__

    def isInstanceOf(self, widgetClass):
        return isinstance(self.widget, widgetClass)
    
    def getTooltip(self):
        return ""

    def findPossibleUIMapIdentifiers(self):
        ids = []
        name = self.getName()
        if not self.isAutoGenerated(name):
            ids.append("Name=" + name)
        
        title = self.getTitle()
        if title:
            ids.append("Title=" + title)
       
        label = self.getLabel()
        if label:
            ids.append("Label=" + label)
            
        tooltip = self.getTooltip()
        if tooltip:
            ids.append("Tooltip=" + tooltip)
        ids.append("Type=" + self.getType())
        dialog = self.getDialogTitle()
        if dialog:
            ids.append("Dialog=" + dialog)
        
        context = self.getContextName()
        if context:
            ids.append("Context=" + context)
        return ids
    
    def getContextName(self):
        return ""
    
    def getDialogTitle(self):
        return ""
    
    def getUIMapIdentifier(self):
        return self.findPossibleUIMapIdentifiers()[0]
    
    def isPreferred(self):
        return False

class GuiEvent(definitions.UserEvent):
    def __init__(self, name, widget, *args):
        definitions.UserEvent.__init__(self, name)
        self.widget = widget
        self.programmaticChange = False
        self.changeMethod = self.getRealMethod(self.getChangeMethod())
        if self.changeMethod:
            allChangeMethods = [ self.changeMethod ] + self.getProgrammaticChangeMethods()
            for method in allChangeMethods:
                self.interceptMethod(method, ProgrammaticChangeIntercept)

    def getRealMethod(self, method):
        if isinstance(method, MethodIntercept):
            return method.method
        else:
            return method
        
    def interceptMethod(self, method, interceptClass):
        if isinstance(method, MethodIntercept):
            method.addEvent(self)
        else:
            setattr(self.getSelf(method), method.__name__, interceptClass(method, self))

    def getSelf(self, method):
        # seems to be different for built-in and bound methods
        try:
            return method.im_self
        except AttributeError:
            return method.__self__

    def getChangeMethod(self):
        pass

    def getProgrammaticChangeMethods(self):
        return []

    def shouldRecord(self, *args):
        return not self.programmaticChange

    def setProgrammaticChange(self, val, *args, **kwargs):
        self.programmaticChange = val

    @classmethod
    def getAssociatedSignatures(cls, widget):
        return set([ cls.getAssociatedSignal(widget) ])

    def widgetDisposed(self):
        return False
    
    def widgetVisible(self):
        return True

    def widgetSensitive(self):
        return True
    
    def isPreferred(self):
        return self.widget.isPreferred()

    def allowsIdenticalCopies(self):
        return False

    def checkWidgetStatus(self):
        if self.widgetDisposed():
            raise definitions.UseCaseScriptError, "widget " + self.describeWidget() + \
                  " has already been disposed."

        if not self.widgetVisible():
            raise definitions.UseCaseScriptError, "widget " + self.describeWidget() + \
                  " is not visible at the moment."

        if not self.widgetSensitive():
            raise definitions.UseCaseScriptError, "widget " + self.describeWidget() + \
                  " is not sensitive to input at the moment."

class MethodIntercept:
    def __init__(self, method, event):
        self.method = method
        self.events = [ event ]
    def addEvent(self, event):
        self.events.append(event)

class ProgrammaticChangeIntercept(MethodIntercept):
    def __call__(self, *args, **kwds):
        # Allow for possibly nested programmatic changes, observation can have knock-on effects
        eventsToBlock = filter(lambda event: not event.programmaticChange, self.events)
        for event in eventsToBlock:
            event.setProgrammaticChange(True, *args, **kwds)
        retVal = apply(self.method, args, kwds)
        for event in eventsToBlock:
            event.setProgrammaticChange(False)
        return retVal



class ScriptEngine(scriptengine.ScriptEngine):
    defaultMapFile = os.path.join(scriptengine.ScriptEngine.storytextHome, "ui_map.conf")
    def __init__(self, enableShortcuts=False, uiMapFiles=[ defaultMapFile ],
                 customEventTypes=[], universalLogging=True, binDir="", **kw):
        self.uiMap = self.createUIMap(uiMapFiles)
        self.binDir = binDir
        self.addCustomEventTypes(customEventTypes)
        self.importCustomEventTypes()
        scriptengine.ScriptEngine.__init__(self, enableShortcuts, universalLogging=universalLogging, **kw)

    def createUIMap(self, uiMapFiles):
        return UIMap(self, uiMapFiles)

    def importCustomEventTypes(self):
        try:
            from customwidgetevents import customEventTypes
            self.addCustomEventTypes(customEventTypes)
        except ImportError, e:
            msg = str(e).strip()
            if msg != "No module named customwidgetevents":
                raise

    def addCustomEventTypes(self, customEventTypes):
        for customWidgetClass, customEventClasses in customEventTypes:
            for widgetClass, currEventClasses in self.eventTypes:
                if widgetClass is customWidgetClass:
                    # Insert at the start, to give first try to the custom events
                    currEventClasses[0:0] = customEventClasses
                    break
            self.eventTypes.insert(0, (customWidgetClass, customEventClasses))

    def findEventClassesFor(self, widget):
        eventClasses = []
        currClass = None
        for widgetClass, currEventClasses in self.eventTypes:
            if widget.isInstanceOf(widgetClass):
                if not currClass or issubclass(widgetClass, currClass):
                    eventClasses = currEventClasses
                    currClass = widgetClass
                elif not issubclass(currClass, widgetClass):
                    eventClasses = eventClasses + currEventClasses # make a copy
        return eventClasses

    def monitorSignal(self, eventName, signalName, widget, *args, **kw):
        if self.active():
            return self._monitorSignal([ eventName ], signalName, WidgetAdapter.adapt(widget), *args, **kw)

    def _monitorSignal(self, eventNames, signalName, widget, argumentParseData=None):
        signalEvent = self._createSignalEvent(eventNames[0], signalName, widget, argumentParseData)
        if signalEvent:
            self._addEventToScripts(signalEvent, eventNames)
            return signalEvent

    def _addEventToScripts(self, event, eventNames):
        if event.name and self.replayerActive():
            self.replayer.addEvent(event, eventNames)
        if event.name and self.recorderActive():
            event.connectRecord(self.recorder.writeEvent)

    def _createSignalEvent(self, eventName, eventDescriptor, widget, argumentParseData):
        for eventClass in self.findEventClassesFor(widget):
            if eventDescriptor in eventClass.getAssociatedSignatures(widget):
                return eventClass(eventName, widget, argumentParseData)
            
    def getEditorEnvironment(self):
        new_env = {}
        for var, value in os.environ.items():
            if var == "PATH":
                new_env[var] = value + os.pathsep + self.binDir
            elif not var.startswith("USECASE_RE"): # Don't transfer our record scripts!
                new_env[var] = value
        return new_env

    def getEditorCmdArgs(self, recordScript, interface):
        mapFiles = self.uiMap.getMapFileNames()
        return [ "storytext_editor", "-m", ",".join(mapFiles), "-i", interface, recordScript ]

    def replaceAutoRecordingForUsecase(self, interface, exitHook):
        self.recorder.closeScripts(exitHook)
        recordScript = os.getenv("USECASE_RECORD_SCRIPT")
        if self.uiMap and recordScript and os.path.isfile(recordScript) and self.recorder.hasAutoRecordings:
            sys.stdout.flush()
            cmdArgs = self.getEditorCmdArgs(recordScript, interface)
            env = self.getEditorEnvironment()
            if os.name == "posix":
                os.execvpe(cmdArgs[0], cmdArgs, env) #@UndefinedVariable
            else:
                subprocess.call(cmdArgs, env=env)

    def replaceAutoRecordingForShortcut(self, script):
        if self.uiMap and self.binDir and self.recorder.hasAutoRecordings:
            cmdArgs = self.getEditorCmdArgs(script.scriptName, "gtk")
            subprocess.call(cmdArgs, env=self.getEditorEnvironment())
    
    def getClassName(self, widgetClass, module):
        return module + "." + widgetClass.__name__

    def getFormatted(self, text, html, title):
        if html:
            return '<div class="Text_Header">' + title + "</div>\n" + \
                '<div class="Text_Normal">' + text + "</div>"
        else:
            return text

    def run(self, options, args):
        if options.supported:
            return self.describeSupportedWidgets()
        elif options.supported_html:
            return self.describeSupportedWidgets(html=True)
        elif options.insert_shortcuts:
            return self.rerecord()
        else:
            try:
                return scriptengine.ScriptEngine.run(self, options, args)
            finally:
                if not options.disable_usecase_names:
                    self.replaceAutoRecordingForUsecase(options.interface, exitHook=False)

    def handleAdditionalOptions(self, options):
        if options.screenshot or os.environ.has_key("USECASE_REPLAY_SCREENSHOTS"):
            Describer.writeScreenshots = True
        if options.maxoutputwidth:
            Describer.maxOutputWidth = int(options.maxoutputwidth)
        if options.imagedescription:
            Describer.imageDescriptionType = options.imagedescription
        if options.pathstoimages:
            Describer.imagePaths = options.pathstoimages.split(",")
        if options.exclude_describe:
            for excludeStr in options.exclude_describe.split(","):
                # Jython swallows these characters under Windows unfortunately, see http://bugs.jython.org/issue1599
                # Add an additional syntax there
                sep = "NOT" if "NOT" in excludeStr else "!"
                parts = excludeStr.split(sep)
                Describer.excludeClassNames[parts[0]] = parts[1:]
        if options.min_field_widths:
            for subStr in options.min_field_widths.split(","):
                fieldName, minWidthStr = subStr.split("=")
                Describer.minFieldWidths[fieldName] = int(minWidthStr)

    def run_python_or_java(self, args):
        # Two options here: either a Jython program and hence a .py file, or a Java class
        # If it's a file, assume it's Python
        if os.path.isfile(args[0]):
            self.run_python_file(args)
        else:
            exec "import " + args[0] + " as _className"
            # Java doesn't use the standard convention of having the first item in args be the
            # actual program name
            _className.main(args[1:]) #@UndefinedVariable

    def describeSupportedWidgets(self, html=False):
        toolkit, module, actionWord, linkPrefix = self.getDescriptionInfo()
        intro = """The following lists the %s widget types and the associated %s on them which 
StoryText %s is currently capable of recording and replaying. Any type derived from the listed
types is also supported.
""" % (toolkit, actionWord, definitions.__version__)
        print self.getFormatted(intro, html, toolkit + " Widgets and " + actionWord + " supported for record/replay")
        classes = self.getRecordReplayInfo(module)
        classNames = sorted(classes.keys())
        if html:
            self.writeHtmlTable(classNames, classes, linkPrefix)
        else:
            self.writeAsciiTable(classNames, classes)

        logIntro = """
The following lists the %s widget types whose status and changes StoryText %s is 
currently capable of monitoring and logging. Any type derived from the listed types 
is also supported but will only have features of the listed type described.
""" % (toolkit, definitions.__version__)
        print self.getFormatted(logIntro, html, toolkit + " Widgets supported for automatic logging")
        classNames = [ self.getClassName(w, module) for w in self.getSupportedLogWidgets() ]
        classNames.sort()
        if html:
            self.writeHtmlList(classNames, module, linkPrefix)
        else:
            for className in classNames:
                print className
        return True
    
    def rerecord(self):
        commands = self.replayer.getCommands()
        while len(commands) > 0:
            for command in commands:
                self.recorder.record(command)
            commands = self.replayer.getCommands()
        return True
    
    def getRecordReplayInfo(self, module):
        classes = {}
        for widgetClass, currEventClasses in self.eventTypes:
            if len(currEventClasses):
                self.addSignals(classes, widgetClass, currEventClasses, module)
        return classes

    def addSignals(self, classes, widgetClass, currEventClasses, module):
        signalNames = set()
        for eventClass in currEventClasses:
            signatures = eventClass.getAssociatedSignatures(None)
            signalNames.update(signatures)
        className = self.getClassName(widgetClass, module)
        classes[className] = sorted(signalNames)

    def writeAsciiTable(self, classNames, classes):
        for className in classNames:
            print className.ljust(self.getClassNameColumnSize()) + ":", " , ".join(classes[className])

    def getClassNameColumnSize(self):
        return 25 # seems to work, mostly

    def writeHtmlTable(self, classNames, classes, linkPrefix):
        print '<div class="Text_Normal"><table border=1 cellpadding=1 cellspacing=1>'
        for className in classNames:
            print '<tr><td>' + self.getLink(className, linkPrefix) + '</td><td><div class="Table_Text_Normal">' + \
                " , ".join(classes[className]) + "</div></td></tr>"
        print "</table></div>"

    def getLink(self, className, linkPrefix):
        docName = self.getDocName(className)
        return '<a class="Text_Link" href=' + linkPrefix + \
            docName + '.html>' + className + '</a>'

    def getDocName(self, className):
        return className.split(".")[-1].lower()

    def writeHtmlList(self, classNames, module, linkPrefix):
        print '<div class="Text_Normal">'
        for className in classNames:
            print '<li>' + self.getLink(className, linkPrefix)
        print '</div><div class="Text_Normal"><i>(Note that a textual version of this page can be auto-generated by running "storytext -s -i ' + module.lower() + '")</i></div>'

    @classmethod
    def getDisplayName(cls, signalName):
        return cls.signalDescs.get(signalName)

    @classmethod
    def getColumnDisplayName(cls, signalName):
        return cls.columnSignalDescs.get(signalName, signalName)


class WriteParserHandler:
    def __init__(self, fileName, parser):
        self.fileName = fileName
        self.parser = parser
        self.changed = False

    def write(self):
        if self.changed:
            dirName = os.path.dirname(self.fileName)
            if dirName and not os.path.isdir(dirName):
                os.makedirs(dirName)
            self.parser.write(encodingutils.openEncoded(self.fileName, "w"))
            self.changed = False

    def add_section(self, *args):
        self.changed = True
        self.parser.add_section(*args)

    def set(self, *args):
        self.changed = True
        self.parser.set(*args)

    def __getattr__(self, name):
        return getattr(self.parser, name)
    
    
class ParserSectionDict(OrderedDict):
    def __init__(self, fileName, *args, **kw):
        OrderedDict.__init__(self, *args, **kw)
        self.readingFiles = fileName
        
    def __getitem__(self, key):
        if self.readingFiles:
            msg = "UI map file(s) at " + self.readingFiles + " has duplicated sections for widgets identified by '" + key + "', the earlier ones will be ignored"
            sys.stderr.write("WARNING: " + msg + ".\n")
        return OrderedDict.__getitem__(self, key)

    def values(self):
        # Fix for python 2.7... which calls __getitem__ internally
        origFile = self.readingFiles
        self.readingFiles = None
        ret = OrderedDict.values(self)
        self.readingFiles = origFile
        return ret
    
class UIMapFileParser(ConfigParser):
    def __init__(self, filenames, **kw):
        ConfigParser.__init__(self, **kw)
        # There isn't a nice way to change the behaviour on getting a duplicate section
        # so we use a nasty way :)
        self._sections = ParserSectionDict(",".join(filenames))
        
    def optionxform(self, optionstr):
        return optionstr # don't lowercase
    
    def read(self, filenames):
        for filename in filenames:
            try:
                fp = encodingutils.openEncoded(filename)
            except IOError:
                continue
            self._read(fp, filename)
            fp.close()
        self._sections.readingFiles = None
                
        
class UIMapFileHandler:
    quoteChars = [ ("'", "APOSTROPHE") ]
    bracketChars = [ ("[", "OPENBRACKET"), ("]", "CLOSEBRACKET")]
    regexChars = re.compile("[\^\$\[\]\{\}\\\*\?\|\+]")
    def __init__(self, uiMapFiles): 
        self.readFiles(uiMapFiles)
        self.regexSections = []
        for section in self.readParser.sections():
            if self.regexChars.search(section) != None:
                try:
                    self.regexSections.append(re.compile(section))
                except re.error:
                    pass
                
    def readFiles(self, uiMapFiles):
        # See top of file: uses the version from 2.6
        self.writeParsers = [ WriteParserHandler(f, self.makeParser([ f ])) for f in uiMapFiles ]
        if len(self.writeParsers) == 1:
            self.readParser = self.writeParsers[0]
        else:
            self.readParser = self.makeParser(uiMapFiles)
            
    def makeParser(self, filenames):
        parser = UIMapFileParser(filenames, dict_type=OrderedDict)
        try:
            parser.read(filenames)
            return parser
        except ParsingError:
            raise definitions.UseCaseScriptError, "ERROR: could not parse UI map file(s) at " + ",".join(filenames)

    def storeInfo(self, sectionName, signature, eventName):
        sectionName = self._escape(sectionName, self.bracketChars)
        if not self.readParser.has_section(sectionName):
            self.writeParsers[-1].add_section(sectionName)
           
        signature = signature.replace("::", "-") # Can't store :: in ConfigParser unfortunately
        if not self.readParser.has_option(sectionName, signature):
            for writeParser in self.writeParsers:
                if writeParser.has_section(sectionName):
                    writeParser.set(sectionName, signature, eventName)
            
    def findWriteParser(self, section):
        for parser in self.writeParsers:
            if parser.has_section(section):
                return parser

    def updateSectionAndOptionNames(self, section, newSectionName, option, newOptionName):
        section = self._escape(section, self.bracketChars)
        newSectionName = self._escape(newSectionName, self.bracketChars)
        writeParser = self.findWriteParser(section)
        removeSection = False
        if not writeParser.has_section(newSectionName):
            writeParser.add_section(newSectionName)
            removeSection = True
        for name, value in self.readParser.items(section):
            optName = newOptionName if name == option else name
            if name == option:
                writeParser.remove_option(newSectionName, option)
            writeParser.set(newSectionName, optName, value)

        if removeSection:
            writeParser.remove_section(section)
        writeParser.write()
        return newSectionName
    
    def write(self, *args):
        for parserHandler in self.writeParsers:
            parserHandler.write()

    def __getattr__(self, name):
        return getattr(self.readParser, name)

    def findSectionsAndOptions(self, valueString):
        details = []
        for section in self.readParser.sections():
            for optionName, value in self.readParser.items(section):
                if value and valueString.startswith(value):
                    details.append((self._unescape(section, self.bracketChars), optionName))
        return details

    def splitOptionValue(self, valueString):
        for section in self.readParser.sections():
            for optionName, value in self.readParser.items(section):
                if value and valueString.startswith(value):
                    return value, valueString.replace(value, "").strip()
        return None, None
    
    def updateOptionValue(self, section, option, newValue):
        section = self._escape(section, self.bracketChars)
        writeParser = self.findWriteParser(section)
        writeParser.set(section, option, newValue)
        writeParser.write()

    def hasInfo(self):
        return len(self.readParser.sections()) > 0
    
    def getSection(self, section):
        rawSectionName = self._escape(section, self.bracketChars)
        if self.readParser.has_section(rawSectionName):
            return section
        
        for regex in self.regexSections:
            if regex.match(rawSectionName):
                return regex.pattern

    def items(self, section):
        return self.readParser.items(self._escape(section, self.bracketChars))
    
    def escape(self, text):
        return self._escape(text, self.quoteChars + self.bracketChars)

    def unescape(self, text):
        return self._unescape(text, self.quoteChars + self.bracketChars)
    
    def _escape(self, text, chars):
        for char, name in chars:
            text = text.replace(char, "<" + name + ">")
        return text
    
    def _unescape(self, text, chars):
        for char, name in chars:
            text = text.replace("<" + name + ">", char)
        return text

class UIMap:
    ignoreWidgetTypes = []
    def __init__(self, scriptEngine, uiMapFiles):
        self.fileHandler = UIMapFileHandler(uiMapFiles)
        self.scriptEngine = scriptEngine
        self.windows = []
        self.logger = logging.getLogger("gui map")
        self.logger.debug("Reading ui map files at " + repr(uiMapFiles))

    def readFiles(self, uiMapFiles):
        self.fileHandler.readFiles(uiMapFiles)

    def findWidgetDetails(self, scriptCommand):
        return self.fileHandler.findSectionsAndOptions(scriptCommand)

    def getMapFileNames(self):
        return [ parser.fileName for parser in self.fileHandler.writeParsers ]

    def monitorAndStoreWindow(self, window):
        if window not in self.windows:
            self.windows.append(window)
            self.monitorWindow(WidgetAdapter.adapt(window))

    def monitorWindow(self, window):
        self.logger.debug("Monitoring new window with title " + repr(window.getTitle()))
        self.monitor(window)

    def monitor(self, widget, excludeWidgets=[]):
        if widget.widget not in excludeWidgets:
            self.monitorWidget(widget)
            self.monitorChildren(widget, excludeWidgets)
        
    def monitorChildren(self, widget, *args, **kw):
        for child in widget.getChildren():
            self.monitor(child, *args, **kw)

    def monitorWidget(self, widget):
        signaturesInstrumented, autoInstrumented = self.instrumentFromMapFile(widget)
        if self.scriptEngine.recorderActive() or not self.fileHandler.hasInfo():
            widgetType = widget.getType()
            for signature in self.findAutoInstrumentSignatures(widget, signaturesInstrumented):
                identifier = self.getAutoInstrumentIdentifier(widget)
                autoEventName = "Auto." + widgetType + "." + signature + ".'" + identifier + "'"
                signalName, argumentParseData = self.parseSignature(signature)
                self.autoInstrument([ autoEventName ], signalName, widget, argumentParseData, widgetType)
        return autoInstrumented
    
    def getFullWidgetDescriptor(self, widget):
        basicId = ", ".join(widget.findPossibleUIMapIdentifiers())
        if basicId.startswith("Name="):
            basicId = basicId.split(", ")[0]
        return basicId

    def getAutoInstrumentIdentifier(self, widget):
        basicId = self.getFullWidgetDescriptor(widget)
        return self.fileHandler.escape(basicId)

    def instrumentFromMapFile(self, widget):
        widgetType = widget.getType()
        if widgetType in self.ignoreWidgetTypes:
            return set(), False
        signaturesInstrumented = set()
        autoInstrumented = False
        for signature, eventNames in self.findAllSignatureInfo(widget):
            if self.tryAutoInstrument(eventNames, signature, signaturesInstrumented, widget, widgetType):
                autoInstrumented = True
        return signaturesInstrumented, autoInstrumented

    def tryAutoInstrument(self, eventNames, signature, signaturesInstrumented, widget, widgetType):
        try:
            signalName, argumentParseData = self.parseSignature(signature)
            if self.autoInstrument(eventNames, signalName, widget, argumentParseData, widgetType):
                signaturesInstrumented.add(signature)
                # sometimes extra data is optional configuration, need to note that we don't need 
                # several variants in the recorder
                signaturesInstrumented.add(signalName) 
                return True
        except definitions.UseCaseScriptError, e:
            sys.stderr.write(encodingutils.encodeToLocale("ERROR in UI map file: " + unicode(e) + "\n"))
        return False

    def findAutoInstrumentSignatures(self, widget, preInstrumented):
        signatures = []
        for eventClass in self.scriptEngine.findEventClassesFor(widget):
            for signature in eventClass.getAssociatedSignatures(widget):
                if signature not in signatures and signature not in preInstrumented:
                    signatures.append(signature)
        return signatures
    
    def findAllSignatureInfo(self, widget):
        info = OrderedDict()
        for section in self.findSections(widget):
            self.logger.debug("Reading map file section " + repr(section) + " for widget of type " + widget.getType())
            for signature, eventName in self.fileHandler.items(section):
                eventNames = info.setdefault(signature, [])
                if eventName not in eventNames:
                    eventNames.append(eventName)
        return info.items()
    
    @staticmethod
    def combinations(iterable, r):
        # From Python 2.6 this is included, but must support 2.5. Copy it in...
        # combinations('ABCD', 2) --> AB AC AD BC BD CD
        # combinations(range(4), 3) --> 012 013 023 123
        pool = tuple(iterable)
        n = len(pool)
        if r > n:
            return
        indices = range(r)
        yield tuple(pool[i] for i in indices)
        while True:
            for i in reversed(range(r)):
                if indices[i] != i + n - r:
                    break
            else:
                return
            indices[i] += 1
            for j in range(i+1, r):
                indices[j] = indices[j-1] + 1
            yield tuple(pool[i] for i in indices)
    
    def allUIMapIdCombinations(self, widget):
        ids = widget.findPossibleUIMapIdentifiers()
        for i in range(len(ids), 0, -1):
            for sectionNameParts in self.combinations(ids, i):
                sectionName = ", ".join(sectionNameParts)
                if self.isSensibleSectionName(sectionName, i):
                    yield sectionName
    
    def isSensibleSectionName(self, sectionName, partCount):
        if partCount == 1:
            return all((not sectionName.startswith(idType + "=") for idType in WidgetAdapter.secondaryIdentifiers))
        else:
            return not sectionName.startswith("Name=")
    
    def findSections(self, widget):
        sections = []
        for sectionName in self.allUIMapIdCombinations(widget):
            self.logger.debug("Looking up section name " + repr(sectionName))
            actualSection = self.fileHandler.getSection(sectionName)
            if actualSection:
                sections.append(actualSection)
        return sections

    def parseSignature(self, signature):
        parts = signature.split(".", 1)
        signalName = parts[0]
        if len(parts) > 1:
            return signalName, parts[1]
        else:
            return signalName, None

    def autoInstrument(self, eventNames, signalName, widget, argumentParseData, *args):
        self.logger.debug("Monitor " + ",".join(eventNames) + ", " + signalName + ", " + widget.getType() + ", " + str(argumentParseData))
        self.scriptEngine._monitorSignal(eventNames, signalName, widget, argumentParseData)
        return True
        
# Base class for all replayers using a GUI
class UseCaseReplayer(replayer.UseCaseReplayer):
    def __init__(self, uiMap, universalLogging, recorder, **kw):
        replayer.UseCaseReplayer.__init__(self, **kw)
        self.readingEnabled = False
        self.uiMap = uiMap
        self.loggerActive = universalLogging
        self.recorder = recorder
        self.delay = float(os.getenv("USECASE_REPLAY_DELAY", 0.0))
        
    def handleComment(self, comment):
        self.recorder.storeComment(comment)
        
    def enableReading(self):
        self.readingEnabled = True
    
    def getParseError(self, scriptCommand):
        widgetDetails = self.uiMap.findWidgetDetails(scriptCommand)
        if widgetDetails:
            errs = []
            for widgetDescriptor, actionName in widgetDetails:
                errs.append("no widget found with descriptor '" + widgetDescriptor + "' to perform action '" + actionName + "' on.")
            return "\n".join(errs)
        else:
            return "could not find matching entry in UI map file."


# Use the idle handlers instead of a separate thread for replay execution
# Used for GTK, tkinter, wxPython
class IdleHandlerUseCaseReplayer(UseCaseReplayer):
    def __init__(self, *args, **kw):
        UseCaseReplayer.__init__(self, *args, **kw)
        self.idleHandler = None
        self.tryAddDescribeHandler()

    def isMonitoring(self):
        return self.loggerActive or (self.recorder.isActive() and self.uiMap)
        
    def tryAddDescribeHandler(self):
        if self.idleHandler is None and self.isMonitoring():
            self.idleHandler = self.makeDescribeHandler(self.handleNewWindows)
        else:
            self.idleHandler = None

    def enableReading(self):
        self.readingEnabled = True
        self._disableIdleHandlers()
        self.enableReplayHandler()

    def makeDescribeHandler(self, method):
        return self.makeIdleHandler(method)

    def makeIdleReplayHandler(self, method):
        return self.makeIdleHandler(method)

    def callReplayHandlerAgain(self, *args):
        self.enableReplayHandler()

    def _disableIdleHandlers(self):
        if self.idleHandler is not None:
            self.removeHandler(self.idleHandler)
            self.idleHandler = None
    
    def enableReplayHandler(self):
        self.idleHandler = self.makeReplayHandler(self.describeAndRun)

    def handleNewWindows(self):
        for window in self.findWindowsForMonitoring():
            if self.uiMap and (self.isActive() or self.recorder.isActive()):
                self.uiMap.monitorAndStoreWindow(window)
            if self.loggerActive:
                self.describeNewWindow(window)
        return True

    def makeReplayHandler(self, method):
        if self.delay:
            milliseconds = int(self.delay * 1000)
            return self.makeTimeoutReplayHandler(method, milliseconds)
        else:
            return self.makeIdleReplayHandler(method)

    def describeAndRun(self, *args):
        self.handleNewWindows()
        if self.readingEnabled:
            self.readingEnabled = self.runNextCommand()[0]
            if not self.readingEnabled:
                self.idleHandler = None
                self.tryAddDescribeHandler()
                if not self.idleHandler and self.uiMap: # pragma: no cover - cannot test with replayer disabled
                    # End of shortcut: reset for next time
                    self.logger.debug("Shortcut terminated: Resetting UI map ready for next shortcut")
                    self.uiMap.windows = [] 
                    self.events = {}
        if self.readingEnabled:
            return self.callReplayHandlerAgain(*args)
        else:
            return False
        
# Base class for Java replayers, both of which run in a separate thread
class ThreadedUseCaseReplayer(UseCaseReplayer):
    def waitForReenable(self):
        self.logger.debug("Waiting for replaying to be re-enabled...")
        while not self.readingEnabled:
            time.sleep(0.1) # don't use the whole CPU while waiting

    def describeAndRun(self, describeMethod, replayFailureMethod=None):
        if not self.readingEnabled:
            self.waitForReenable()
        while True:
            if self.delay:
                self.logger.debug("Sleeping for " + str(self.delay) + " seconds...")
                time.sleep(self.delay)
            proceed, wait = self.runNextCommand(describeMethod=describeMethod, replayFailureMethod=replayFailureMethod)
            if not proceed:
                self.readingEnabled = self.waitingCompleted()
                if wait:
                    self.waitForReenable()
                else:
                    self.logger.debug("No command to run, no waiting to do: exiting replayer")
                    break

    def tryParseRepeatedly(self, commandWithArg, replayFailureMethod):
        attemptCount = 50
        command = None
        for attempt in range(attemptCount):
            try:
                command, argumentString = self.parseCommand(commandWithArg)
                event, parsedArguments = self.checkWidgetStatus(command, argumentString)
                return command, argumentString, event, parsedArguments
            except definitions.UseCaseScriptError:
                # We don't terminate scripts if they contain errors
                if attempt == attemptCount - 1:
                    raise
                else:
                    type, value, _ = sys.exc_info()
                    self.logger.debug("Error, final event failed, waiting and retrying: " + str(value))
                    if replayFailureMethod:
                        replayFailureMethod(str(value), self.events.get(command, []))
                    time.sleep(0.1)
        
    def checkAndParse(self, event, compositeEventProxy):
        event.checkWidgetStatus()
        parsedArguments = event.parseArguments(compositeEventProxy.unparsedArgs)
        if isinstance(parsedArguments, definitions.CompositeEventProxy):
            compositeEventProxy.updateFromProxy(parsedArguments)
            return None, None
        else:
            if compositeEventProxy.hasEvents():
                compositeEventProxy.addEvent(event, parsedArguments)
                return compositeEventProxy, ""
            else:
                return event, parsedArguments
    
    def getPossibleEvents(self, events):
        # We may have several plausible events with this name,
        # but some of them won't work because widgets are disabled, invisible etc
        # Go backwards to preserve back-compatibility, previously only the last one was considered.
        # The more recently it was added, the more likely it is to work also
        checkedEvents = []
        badEvents = []
        for event in events:
            try:
                event.checkWidgetStatus()
                checkedEvents.insert(0, event)
            except definitions.UseCaseScriptError:
                badEvents.insert(0, event)
        return checkedEvents if checkedEvents else badEvents[:1]

    def checkForAmbiguityError(self, allInterpretations):
        sampleEvent = allInterpretations[0][0]
        uiMapSections = self.uiMap.findWidgetDetails(sampleEvent.name)
        # Only warn if they're in the same section, otherwise assume ambiguity is deliberate
        if len(set(uiMapSections)) == 1:
            fullIdentifiers = [self.uiMap.getFullWidgetDescriptor(e.widget) for e, _ in allInterpretations]
            if len(set(fullIdentifiers)) == 1:
                if not sampleEvent.allowsIdenticalCopies():
                    raise definitions.UseCaseScriptError, self.getIdenticalWidgetError(uiMapSections[0][0], fullIdentifiers[0])
            else:
                raise definitions.UseCaseScriptError, self.getIdenticalSectionError(uiMapSections[0][0], fullIdentifiers)

    def getIdenticalWidgetError(self, uiMapSection, fullIdentifier):
        err = "Instruction could be interpreted more than one way!\n"
        err += "There is more than one widget matching [" + uiMapSection + "] and no other attributes which distinguish them.\n"
        if "Tooltip=" not in fullIdentifier:
            err += "You can fix this by adding distinguishing tooltips to the different widgets.\n"
            err += "You can also fix this by adding widget names/internal IDs if this is not desirable.\n"
        else:
            err += "You can fix this by adding widget names/internal IDs to the different widgets.\n"
        err += "(see the StoryText documentation for how to do this for your chosen toolkit)."
        return err
    
    def getIdenticalSectionError(self, uiMapSection, fullIdentifiers):
        err = "Instruction could be interpreted more than one way!\n"
        err += "There is more than one widget matching [" + uiMapSection + "], but there are other attributes that may be used to distinguish them.\n"
        err += "The widgets would be identified fully by sections as follows:\n"
        for identifier in fullIdentifiers:
            err += "[" + identifier + "]\n"
        err += "It is suggested to replace the section appropriately in your UI map file and add distinguishing names for these widgets."
        return err

    def checkWidgetStatus(self, commandName, argumentString):
        if commandName == replayer.signalCommandName:
            return
        
        allInterpretations = []
        possibleEvents = self.getPossibleEvents(self.events[commandName])
        compositeEventProxy = definitions.CompositeEventProxy(argumentString)
        for i, event in enumerate(possibleEvents):
            try:
                self.logger.debug("Check widget status for " + repr(commandName) + ", event of type " + event.__class__.__name__) 
                parsedEvent, parsedArguments = self.checkAndParse(event, compositeEventProxy)
                if parsedEvent:
                    if parsedEvent.isPreferred():
                        return parsedEvent, parsedArguments
                    else:
                        allInterpretations.append((parsedEvent, parsedArguments))
            except definitions.UseCaseScriptError:
                if i == len(possibleEvents) - 1 and len(allInterpretations) == 0:
                    raise
                else:
                    type, value, _ = sys.exc_info()
                    self.logger.debug("Error, trying another: " + str(value))
        
        if len(allInterpretations) == 0:
            return compositeEventProxy, ""
        
        if len(allInterpretations) > 1:
            self.checkForAmbiguityError(allInterpretations)
            self.logger.debug("Ambiguity: picking control at random!")
            return choice(allInterpretations)
        return allInterpretations[0]
    
                                
    def writeWarnings(self, event):
        warn = event.getWarning()
        if warn:
            self.write("Warning: not replaying full command, " + warn)

    def _parseAndProcess(self, command, describeMethod, replayFailureMethod):
        try:
            commandName, argumentString, event, parsedArguments = self.tryParseRepeatedly(command, replayFailureMethod)
            describeMethod()
            self.describeAppEventsHappened()
        except:
            describeMethod()
            self.describeAppEventsHappened()
            self.write("")
            raise
        self.logger.debug("About to perform " + repr(commandName) + " with arguments " + repr(argumentString))
        if event:
            self.describeEvent(commandName, argumentString)
            self.writeWarnings(event)
            event.generate(parsedArguments)
        else:
            self.processSignalCommand(argumentString)
                


class WidgetCounter:
    def __init__(self, equalityMethod=None):
        self.widgetNumbers = []
        self.nextWidgetNumber = 1
        self.describedNumber = 0
        self.customEqualityMethod = equalityMethod

    def widgetsEqual(self, widget1, widget2):
        if self.customEqualityMethod:
            return widget1 is widget2 or self.customEqualityMethod(widget1, widget2)
        else:
            return widget1 is widget2

    def getWidgetNumber(self, widget):
        for currWidget, number in self.widgetNumbers:
            if (not hasattr(currWidget, "isDisposed") or not currWidget.isDisposed()) and self.widgetsEqual(widget, currWidget):
                return number
        return 0

    def getId(self, widget):
        number = self.getWidgetNumber(widget)
        if not number:
            number = self.nextWidgetNumber
            self.widgetNumbers.append((widget, self.nextWidgetNumber))
            self.nextWidgetNumber += 1
        return str(number)

    def getWidgetsForDescribe(self):
        widgets = self.widgetNumbers[self.describedNumber:]
        self.describedNumber = len(self.widgetNumbers)
        return widgets



# Base class for everything except GTK's describer, which works a bit differently
class Describer(object):
    maxOutputWidth = 130
    minFieldWidths = {}
    writeScreenshots = False
    imagePaths = []
    imageDescriptionType = None
    excludeClassNames = {}
    def __init__(self):
        self.logger = encodingutils.getEncodedLogger("gui log")
        self.windows = set()
        self.widgetsWithState = OrderedDict()
        self.imageCounter = WidgetCounter(self.imagesEqual)
        self.structureLog = logging.getLogger("widget structure")

    def imagesEqual(self, image1, image2):
        return image1 == image2

    def describe(self, window):
        if window in self.windows or not self.checkWindow(window):
            return
        
        if self.structureLog.isEnabledFor(logging.DEBUG):
            self.structureLog.info("New window:")
            self.describeStructure(window)

        self.windows.add(window)
        title = self.getSpecificState(window)
        message = "-" * 10 + " " + self.getWindowString() + " '" + title + "' " + "-" * 10
        self.widgetsWithState[window] = title
        self.logger.info("\n" + message)
        self.logger.info(self.getWindowContentDescription(window))
        footerLength = min(len(message), 100) # Don't let footers become too huge, they become ugly...
        self.logger.info("-" * footerLength)

    def checkWindow(self, window):
        return True

    def getWindowContentDescription(self, window):
        return self.getChildrenDescription(window)

    def getWindowString(self):
        return "Window"

    def findStateChanges(self, *args):
        defunctWidgets = []
        stateChanges = []
        for widget, oldState in self.widgetsWithState.items():
            if not self.shouldCheckForUpdates(widget, *args):
                continue
            
            try:
                state = self.getState(widget)
            except:
                # If the frame where it existed has been removed, for example...
                message = "Warning: The following exception has been thrown:\n"
                self.logger.debug(message + getExceptionString())
                if self.logger.isEnabledFor(logging.DEBUG):
                    exc = sys.exc_info()[1]
                    if hasattr(exc, "printStackTrace"):
                        exc.printStackTrace()
                defunctWidgets.append(widget)
                continue

            if state != oldState:
                stateChanges.append((widget, oldState, state))
                self.widgetsWithState[widget] = state
            
        for widget in defunctWidgets:
            del self.widgetsWithState[widget]
        return stateChanges

    def shouldCheckForUpdates(self, *args):
        return True

    def describeStateChanges(self, stateChanges, describedForAppearance=[]):
        for widget, oldState, state in stateChanges:
            if not describedForAppearance or not self.hasMarkedAncestor(widget, describedForAppearance):
                changeDesc = self.getStateChangeDescription(widget, oldState, state).rstrip()
                if changeDesc:
                    self.logger.info(changeDesc)

    def describeUpdates(self):
        stateChanges = self.findStateChanges()
        self.describeStateChanges(stateChanges)

    def getStateChangeDescription(self, widget, oldState, state):
        if isinstance(widget, self.getWindowClasses()):
            return "Changed title of " + self.getWindowString().lower() + " to '" + state + "'"
        else:
            return self.getUpdatePrefix(widget, oldState, state) + self.getDescription(widget)

    def getUpdatePrefix(self, widget, oldState, state):
        if isinstance(widget, self.getTextEntryClass()):
            return "Updated "
        else:
            return "\n"

    def addToDescription(self, desc, newText, allowEmpty=False):
        if newText or allowEmpty:
            if desc:
                desc += "\n"
            desc += newText.rstrip() + "\n"
        return desc

    def getDescription(self, *args, **kw):
        return self.convertToString(self._getDescription(*args, **kw))
        
    def getChildrenDescription(self, *args, **kw):
        return self.convertToString(self._getChildrenDescription(*args, **kw))
        
    def convertToString(self, obj):
        # Bit of a pain, unicode doesn't inherit from string for some reason
        return unicode(obj) if isinstance(obj, GridFormatter) else obj

    def _getDescription(self, widget):
        desc = ""
        widgetDesc = self.getWidgetDescription(widget)
        if isinstance(widgetDesc, GridFormatter):
            return widgetDesc
        desc = self.addToDescription(desc, widgetDesc)
        childDesc = self._getChildrenDescription(widget)
        if desc or not isinstance(childDesc, GridFormatter):
            desc = self.addToDescription(desc, self.convertToString(childDesc))
            return desc.rstrip()
        else:
            return childDesc
            
    @classmethod
    def describeClass(cls, className):
        return cls.excludeClassNames.get(className) != []
    
    def getWidgetDescription(self, widget):
        for widgetClass in self.stateWidgets + self.statelessWidgets:
            if isinstance(widget, widgetClass):
                describeClassName = widgetClass.__name__
                actualClassName = widget.__class__.__name__
                if self.describeClass(describeClassName) and self.describeClass(actualClassName):
                    methodName = "get" + describeClassName.replace("$", "") + "Description"
                    return getattr(self, methodName)(widget)
                else:
                    return ""

        for widgetClass in self.ignoreWidgets:
            if isinstance(widget, widgetClass):
                return ""
        
        return self.widgetTypeDescription(widget.__class__.__name__) # pragma: no cover - should be unreachable

    def widgetTypeDescription(self, typeName): # pragma: no cover - should be unreachable
        return "A widget of type '" + typeName + "'" 

    def getState(self, widget):
        state = self.getSpecificState(widget)
        return state.strip()

    def getSpecificState(self, widget):
        for widgetClass in self.stateWidgets:
            if isinstance(widget, widgetClass):
                methodName = "get" + widgetClass.__name__ + "State"
                return getattr(self, methodName)(widget)
        return ""

    def combineElements(self, elements):
        elements = filter(len, elements)
        if len(elements) <= 1:
            return "".join(elements)
        else:
            rows = elements[0].split("\n")
            rows[0] += " ("
            basicLengths = map(len, rows)
            for elIx, el in enumerate(elements[1:]):
                elRows = el.split("\n")
                if len(elRows) > 1:
                    self.equaliseRows(rows)

                for i, elRow in enumerate(elRows):
                    while len(rows) <= i:
                        rows.append(" " * len(rows[-1]))
                    rows[i] += elRow
                if len(elRows) > 1:
                    self.equaliseRows(rows)
                if elIx != len(elements) - 2:
                    rows[0] += ", "
            strippedRows = [ r.rstrip() for r in rows ]
            strippedRows[self.getLastDetailRow(strippedRows, basicLengths)] += ")"
            return "\n".join(strippedRows)

    def getLastDetailRow(self, rows, basicLengths):
        for ix in range(-1, -1 - len(rows), -1):
            if len(rows[ix]) > basicLengths[ix]:
                return ix
        
    def equaliseRows(self, rows):
        if len(rows) > 1:
            maxLen = max((len(r) for r in rows))
            for i, r in enumerate(rows):
                rows[i] = r.ljust(maxLen)

    ##Debug code
    def getRawData(self, widget, useModule=False,
                   visibleMethodNameOverride=None, layoutMethodNameOverride=None):
        basic = ""
        if useModule:
            basic = widget.__class__.__module__ + "."
        basic += widget.__class__.__name__ + " " + str(id(widget))
        if hasattr(widget, "isDisposed") and widget.isDisposed():
            return basic
        layout = None
        layoutMethodName = layoutMethodNameOverride or "getLayout"
        if hasattr(widget, layoutMethodName):
            layout = getattr(widget, layoutMethodName)()

        if layout is not None:
            layoutText = layout.__class__.__name__
            if useModule:
                layoutText = layout.__class__.__module__ + "." + layoutText
            elements = [ layoutText ] + self.getRawDataLayoutDetails(layout, widget)
            basic += " (" + ", ".join(elements) + ")"
        visibleMethodName = visibleMethodNameOverride or self.visibleMethodName
        if hasattr(widget, visibleMethodName) and not self.isVisible(widget, visibleMethodName):
            basic += " (invisible)"
        return basic

    def addHeaderAndFooter(self, widget, text):
        header = "=" * 10 + " " + widget.__class__.__name__ + " " + "=" * 10
        return header + "\n" + self.fixLineEndings(text.rstrip()) + "\n" + "=" * len(header)

    def getRawDataLayoutDetails(self, *args):
        return []

    def getWidgetChildren(self, widget):
        return getattr(widget, self.childrenMethodName)() if hasattr(widget, self.childrenMethodName) else []
        
    def describeStructure(self, widget, indent=0, **kw):
        rawData = self.getRawData(widget, useModule=True, **kw)
        self.structureLog.info("-" * 2 * indent + rawData)
        for child in self.getWidgetChildren(widget):
            self.describeStructure(child, indent+1, **kw)
                
    def getAndStoreState(self, widget):
        state = self.getState(widget)
        self.widgetsWithState[widget] = state
        return state

    def getItemDescription(self, item, prefix, *args):
        elements = []
        if hasattr(item, "getText") and item.getText():
            elements.append(item.getText())
        elements += self.getPropertyElements(item, *args)
        desc = self.combineElements(elements)
        if desc:
            return prefix + desc

    def getItemBarDescription(self, *args, **kw):
        return "\n".join(self.getAllItemDescriptions(*args, **kw))

    def formatTable(self, headerRow, rows, columnCount):
        if columnCount == 0:
            return ""

        if headerRow:
            colWidths = GridFormatter([ headerRow ] + rows, columnCount, allowOverlap=False).findColumnWidths()
            self.adjustForMinFieldWidths(colWidths, headerRow)
            header = GridFormatter([ headerRow ], columnCount).formatCellsInGrid(colWidths)
            body = GridFormatter(rows, columnCount).formatCellsInGrid(colWidths)
            line = "_" * sum(colWidths) + "\n"
            return self.formatWithSeparators(header, body, line)
        else:
            formatter = GridFormatter(rows, columnCount, allowOverlap=False)
            colWidths = formatter.findColumnWidths()
            body = formatter.formatCellsInGrid(colWidths)
            line = "_" * sum(colWidths) + "\n"
            return line + body + "\n" + line
        
    def adjustForMinFieldWidths(self, colWidths, headerRow):
        for i, columnName in enumerate(headerRow):
            minWidth = self.getMinFieldWidth(columnName)
            if minWidth is not None and minWidth > colWidths[i]:
                colWidths[i] = minWidth
                
    def getMinFieldWidth(self, columnName):
        if columnName in self.minFieldWidths:
            return self.minFieldWidths[columnName]
        elif "(" in columnName:
            return self.minFieldWidths.get(columnName.split("(")[0])
        
    def formatWithSeparators(self, header, body, line):
        return line + header + "\n" + line + body + "\n" + line

    def shouldDescribeChildren(self, widget):
        return hasattr(widget, self.childrenMethodName) and not isinstance(widget, self.ignoreChildren) and self.describeClass(widget.__class__.__name__)

    def _getChildrenDescription(self, widget):
        return self.formatChildrenDescription(widget) if self.shouldDescribeChildren(widget) else ""

    def shouldFormatAsGrid(self, columns):
        return columns > 1

    def layoutSortsChildren(self, widget):
        return True

    def isVisible(self, widget, visibleMethodNameOverride=None):
        visibleMethodName = visibleMethodNameOverride or self.visibleMethodName
        return getattr(widget, visibleMethodName)()

    def getHorizontalSpan(self, *args):
        return 1

    def formatChildrenDescription(self, widget):
        children = self.getWidgetChildren(widget)
        visibleChildren = filter(self.isVisible, children)
        sortedChildren = self.sortChildren(widget, visibleChildren)
        childDescriptions = map(self._getDescription, sortedChildren)
        if not self.usesGrid(widget):
            self.removeEmptyDescriptions(sortedChildren, childDescriptions)
        if len(childDescriptions) > 1:
            grid, columns = self.tryMakeGrid(widget, sortedChildren, childDescriptions)
            if self.shouldFormatAsGrid(columns):
                maxWidth = self.getMaxDescriptionWidth(widget)
                formatter = GridFormatter(grid, columns, maxWidth)
                return self.handleGridFormatter(formatter)
            elif grid:
                childDescriptions = [ row[0] for row in grid ]
        
        return self.formatInColumn(childDescriptions)

    def handleGridFormatter(self, formatter):
        # Try to combine horizontal rows into one, so we can take one decision about if they're too wide
        return formatter if formatter.isHorizontalRow() else unicode(formatter)

    def tryMakeGrid(self, widget, sortedChildren, childDescriptions):
        columns = self.getLayoutColumns(widget, len(childDescriptions), sortedChildren)
        if columns > 1:
            horizontalSpans = self.getHorizontalSpans(sortedChildren, columns)
            return self.makeGrid(childDescriptions, horizontalSpans, columns)
        return None, 0

    def getHorizontalSpans(self, sortedChildren, columns):
        return [ self.getHorizontalSpan(c, columns) for c in sortedChildren ]

    def usesGrid(self, widget):
        return False

    def correctSpan(self, index, span, numColumns):
        # Seems to be possible to get spans that would overlap the end of the line
        # Correct these values here, as Eclipse seems to cope OK
        if span == 1:
            return span
             
        currColumn = index % numColumns
        spanRemaining = numColumns - currColumn
        return min(spanRemaining, span)
            
    def makeGrid(self, cellObjects, spans, numColumns):
        grid = []
        index = 0
        horizontalRow = len(cellObjects) == numColumns
        newColumns = numColumns
        for cellObject, span in izip(cellObjects, spans):
            if index % numColumns == 0:
                grid.append([])
            if horizontalRow and isinstance(cellObject, GridFormatter):
                grid[-1] += cellObject.grid[-1]
            else:
                grid[-1].append(self.convertToString(cellObject))
            span = self.correctSpan(index, span, numColumns)
            index += span
            if index % numColumns != 0:
                # If we aren't at line end, introduce extra cells for padding
                for _ in range(span - 1):
                    grid[-1].append("")
        if horizontalRow:
            newColumns = len(grid[-1])
        return grid, newColumns

    def removeEmptyDescriptions(self, sortedChildren, childDescriptions):
        for child, desc in zip(sortedChildren, childDescriptions):
            if not desc:
                sortedChildren.remove(child)
                childDescriptions.remove(desc)

    def getMaxDescriptionWidth(self, widget):
        return self.maxOutputWidth # About a screen or so...

    def formatInColumn(self, childDescriptions):
        if len(childDescriptions) == 1:
            return childDescriptions[0]
        desc = ""
        for childDesc in childDescriptions:
            desc = self.addToDescription(desc, self.convertToString(childDesc), allowEmpty=True)
        
        return desc.rstrip()

    def sortChildren(self, widget, visibleChildren):
        if len(visibleChildren) <= 1 or self.layoutSortsChildren(widget):
            # Trust in the layout, if there is one
            return visibleChildren
        
        xDivides = self.getVerticalDividePositions(visibleChildren)
        # Children don't always come in order, sort them...
        def getChildPosition(child):
            loc = child.getLocation()
            # With a divider, want to make sure everything ends up on the correct side of it
            return self.getDividerIndex(loc.x, xDivides), loc.y, loc.x
            
        visibleChildren.sort(key=getChildPosition)
        return visibleChildren

    def getDividerIndex(self, pos, dividers):
        for i, dividePos in enumerate(dividers):
            if pos < dividePos:
                return i
        return len(dividers)
    
    def fixLineEndings(self, text):
        # Methods return text 'raw' with Windows line endings
        if os.linesep != "\n":
            return text.replace(os.linesep, "\n")
        else:
            return text

    def describeAppearedWidgets(self, stateChangeWidgets, *args):
        newWindows, commonParents = self.categoriseAppearedWidgets(stateChangeWidgets, *args)
        for window in newWindows:
            self.describe(window)
            
        descriptions = map(self.getDescriptionForVisibilityChange, commonParents)
        if self.structureLog.isEnabledFor(logging.DEBUG):
            for parent in commonParents:
                self.structureLog.info("Newly appeared widget:")
                self.describeStructure(parent)
            
        for desc in sorted(descriptions):
            if desc:
                self.logger.info("\nNew widgets have appeared: describing common parent :\n")
                self.logger.info(desc)
        return commonParents
    
    def getMarkedAncestor(self, widget, markedWidgets):
        if widget in markedWidgets:
            return widget
        elif widget.getParent():
            return self.getMarkedAncestor(widget.getParent(), markedWidgets)

    def categoriseAppearedWidgets(self, stateChangeWidgets, *args):
        newWindows, commonParents = [], []
        # Windows only get title changes described
        stateChangesFullDescribe = filter(lambda w: not isinstance(w, self.getWindowClasses()), stateChangeWidgets)
        markedWidgets = self.widgetsAppeared + stateChangesFullDescribe
        for widget in self.widgetsAppeared:
            if not self.widgetShowing(widget, *args):
                self.logger.debug("Widget not showing, ignoring: " + self.getRawData(widget))
                continue

            if isinstance(widget, self.getWindowClasses()): 
                newWindows.append(widget)
            else:
                appearedWidget = self.getWidgetToDescribeForAppearance(widget, markedWidgets)
                if appearedWidget:
                    markedWidgets.append(appearedWidget)
                    commonParents.append(appearedWidget)
                    
        commonParents = [ w for w in commonParents if not self.hasMarkedAncestor(w, markedWidgets) ]
        return newWindows, commonParents

    def hasMarkedAncestor(self, widget, markedWidgets):
        return widget.getParent() is not None and self.getMarkedAncestor(widget.getParent(), markedWidgets) is not None

    def getWidgetToDescribeForAppearance(self, widget, markedWidgets):
        parent = widget.getParent()
        if parent is not None:
            markedAncestor = self.getMarkedAncestor(parent, markedWidgets)
            if markedAncestor:
                if self.logger.isEnabledFor(logging.DEBUG):
                    self.logger.debug("Not describing " + self.getRawData(widget) + " - marked " +
                                          self.getRawData(markedAncestor))
            else:
                return parent

    def getDescriptionForVisibilityChange(self, widget):
        if self.shouldDescribeChildren(widget):
            return self.getChildrenDescription(widget)
        else:
            return self.getDescription(widget)

    def getDiffedDescription(self, widget, oldRows, newRows):
        desc = [self.formatDiffs(old, new) for old,new in zip(oldRows,newRows) if old != new]
        updatePrefix =self.getUpdatePrefix(widget, None, None)
        if updatePrefix == "\nUpdated ":
            updatePrefix += newRows[0] + "\n"
        return self.convertToString(updatePrefix  + "\n".join(desc))

    def formatDiffs(self, oldRow, newRow):
        return "'" + oldRow.strip() + "'" +  " changed to " + "'" + newRow.strip() + "'"

class Indexer:
    allIndexers = {}
    def __init__(self, widget):
        self.widget = widget
        self.logger = logging.getLogger("Indexer")
        
    @classmethod
    def getIndexer(cls, widget):
        # Don't just do setdefault, shouldn't create the Indexer if it already exists
        if widget in cls.allIndexers:
            return cls.allIndexers.get(widget)
        else:
            return cls.allIndexers.setdefault(widget, cls(widget))

    
class TableIndexer(Indexer):
    def __init__(self, widget):
        Indexer.__init__(self, widget)
        self.primaryKeyColumn, self.rowNames = self.findRowNames()
        self.logger.debug("Creating " + self.__class__.__name__ + " with rows " + repr(self.rowNames))
    
    def updateTableInfo(self):
        if self.primaryKeyColumn is None:
            self.primaryKeyColumn, self.rowNames = self.findRowNames()
            self.logger.debug("Rebuilding indexer, primary key " + str(self.primaryKeyColumn) +
                              ", row names now " + repr(self.rowNames))
        else:
            currRowNames = self.getColumn(self.primaryKeyColumn)
            if set(currRowNames) != set([ "<unnamed>" ]):
                self.rowNames = currRowNames
                self.logger.debug("Model changed, row names now " + repr(self.rowNames))

    def getColumn(self, col):
        return [ self.getCellValueToUse(row, col) for row in range(self.getRowCount()) ]
    
    def getColumnTextToUse(self, *args):
        return self.getColumnText(*args) or "<untitled>"
    
    def getCellValueToUse(self, *args):
        return self.getCellValue(*args) or "<unnamed>"
    
    def findRowNames(self):
        firstColumnWithData = None
        if self.getRowCount() > 1:
            for colIndex in range(self.widget.getColumnCount()):
                column = self.getColumn(colIndex)
                uniqueEntries = len(set(column))
                if len(column) > 1 and uniqueEntries == len(column):
                    return colIndex, column
                else:
                    self.logger.debug("Rejecting column " + str(colIndex) + " as primary key : names were " + repr(column))
                    if firstColumnWithData is None and uniqueEntries > 1:
                        firstColumnWithData = colIndex
        # No unique columns to use as row names. Use the first column and add numbers
        return None, self.addIndexes(self.getColumn(firstColumnWithData or 0))
        
    def getIndexedValue(self, index, value, mapping):
        indices = mapping.get(value)
        if len(indices) == 1:
            return value.strip() if self.isBlank(value) else value
        else:
            return value + " (" + str(indices.index(index) + 1) + ")"

    def isBlank(self, text):
        return len(text) > 0 and len(text.strip()) == 0

    def addIndexes(self, values):
        mapping = {}
        for i, value in enumerate(values):
            mapping.setdefault(value, []).append(i)

        return [ self.getIndexedValue(i, v, mapping) for i, v in enumerate(values) ]

    def findColumnIndex(self, columnName):
        for col in range(self.widget.getColumnCount()):
            if self.getColumnTextToUse(col) == columnName:
                return col

    def parseDescription(self, description):
        if " for " in description:
            columnName, rowName = description.split(" for ")
            colIndex = self.findColumnIndex(columnName)
            if colIndex is None:
                raise definitions.UseCaseScriptError, "Could not find column labelled '" + columnName + "' in table."
            return rowName, colIndex
        elif description.endswith(" for"):
            return "", 0
        else:
            return description, 0
    
    def getViewCellIndices(self, description):
        rowName, columnIndex = self.parseDescription(description)
        try:
            rowIndex = self.rowNames.index(rowName)
            return rowIndex, columnIndex
        except ValueError:
            raise definitions.UseCaseScriptError, "Could not find row identified by '" + rowName + "' in table.\nRow names are " + repr(self.rowNames)
                    
    def useColumnTextInDescription(self, **kw):
        return self.widget.getColumnCount() > 1

    def getCellDescription(self, row, col, **kw):
        rowName = self.rowNames[row]
        if self.useColumnTextInDescription(**kw):
            return self.getColumnTextToUse(col) + " for " + rowName
        else:
            return rowName

class TreeIndexer(Indexer):
    def __init__(self, widget):
        Indexer.__init__(self, widget)
        self.allItems = {}
        self.allDescriptions = {}
        self.populate()
        self.logger.debug("Creating " + self.__class__.__name__ + " with descriptions " + repr(self.allDescriptions.values()))
    
    def populate(self):
        self.allItems = {}
        self.allDescriptions = {}
        for item in self.getItems():
            self.storeItem(item)

    def storeItem(self, item):
        desc = self.getDescriptionToStore(item)
        while desc and desc in self.allItems:
            desc = self.addSuffix(desc)
        self.allDescriptions[item] = desc
        self.allItems[desc] = item

    def addSuffix(self, desc):
        if desc.endswith(")"):
            startPos = desc.rfind("(") + 1
            intVal = desc[startPos:-1]
            if intVal.isdigit():
                val = int(intVal)
                return desc[:startPos] + str(val + 1) + ")"
        return desc + " (2)"
    
    def getItem(self, desc):
        return self.allItems.get(desc)
    
    def getItemDescription(self, item):
        return self.allDescriptions.get(item)
    

def getExceptionString():
    return "".join(format_exception(*sys.exc_info()))

class TextLabelFinder:
    def __init__(self, widget, ignoreLabels=[]):
        self.widget = widget
        self.ignoreLabels = ignoreLabels

    def findPrecedingLabel(self, children, *args):
        textPos = children.index(self.widget)
        while textPos > self.getEarliestRelevantIndex(textPos, children, *args):
            prevWidget = children[textPos -1]
            if isinstance(prevWidget, self.getLabelClass()):
                text = self.getLabelText(prevWidget)
                if text and text not in self.ignoreLabels:
                    return text
                else:
                    textPos -= 1
            else:
                return self.findLastLabel(prevWidget)
        return ""
    
    def findLastLabel(self, widget):
        children = self.getChildren(widget)
        if len(children) > 0:
            lastChild = children[-1]
            if isinstance(lastChild, self.getLabelClass()):
                return self.getLabelText(lastChild)
            else:
                return self.findLastLabel(lastChild)
        return ""
        
    def getEarliestRelevantIndex(self, *args):
        return 0
    
    def getLabelText(self, label):
        return label.getText()
    
    def getLabelClass(self):
        return ()
    
    def getContextParentClasses(self):
        return ()
    
    def find(self, useContext=False):
        """ Text widgets often are preceeded by a label, use this as their text, if it exists """
        parent = self.widget.getParent()
        # Tables etc, should not look for labels outside them
        if parent:
            if isinstance(parent, self.getContextParentClasses()):
                return parent.__class__.__name__ + " Cell Editor" if useContext else ""
            children = self.getChildren(parent)
            if self.widget in children: # can't assume this, for example window-type objects often have a parent, but are not one of its children
                if children.index(self.widget) == 0 and self.numRows(children, parent) <= 1: # If we're the first child of our parent, look for the parent in its context
                    return self.__class__(parent, self.ignoreLabels).find()
                else:
                    return self.findPrecedingLabel(children, parent)
        
        return ""
    
    def numRows(self, *args):
        return 0

def removeMarkup(text):
    removed = re.sub("<[^>]*>", "", text)
    return text if removed == text else removed.strip()

# Jython has problems with exceptions thrown from Java callbacks
# Print them out and continue, don't just lose them...
def catchAll(method, *args, **kw):
    try:
        method(*args, **kw)
    except:
        print "Caught exception, trying to write it!"
        sys.stdout.flush()
        sys.stderr.write(getExceptionString() + "\n")

