import PIL,math,os,sys,time,platform
import numpy as np
import pandas as pd
from datetime import datetime
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
from matplotlib.patches import Rectangle
from PIL import Image, ImageDraw, ImageFont
import scipy
from scipy import stats, optimize, ndimage, signal
import scipy.optimize as op
import itertools
import sobol
import json

def is_number(s):
    try:
        int(s)
        return True
    except ValueError:
        return False

def readInstructions(fullpath,fname='Colonyzer.txt',searchUpStream=False):
    '''Read instruction file output by ColonyzerParametryzer.'''
    # Try to read in the Colonyzer input file
    if searchUpStream:
        while not os.path.isfile(os.path.join(fullpath,fname)):
            fullpathnew=os.path.abspath(os.path.join(fullpath,".."))
            if fullpathnew!=fullpath:
                fullpath=fullpathnew
            else:
                raise(ValueError("Searched up to "+fullpath+" but "+fname+" not found in directory structure."))
    fpath=os.path.join(fullpath,fname)
    InsData={}
    if os.path.isfile(fpath):
        print("Reading instruction file output from "+fpath)
        Instructions=open(fpath,'r')
        InsTemp=Instructions.readlines()
        defaultArr=[]
        for x in range(0,len(InsTemp)):
            if InsTemp[x][0] not in ["#","\n","\r"]:
                tlist=InsTemp[x].split(',')
                # default with no date specified
                if len(tlist)==6 and tlist[0]=='default':
                    defaultArr.append([tlist[1],int(tlist[2]),int(tlist[3]),int(tlist[4]),int(tlist[5]),"0000-01-01"])
                # Capturing array of default calibrations together with dates of change (corresponding to dates cameras moved)
                elif len(tlist)==7 and tlist[0]=='default':
                    defaultArr.append([tlist[1],int(tlist[2]),int(tlist[3]),int(tlist[4]),int(tlist[5]),tlist[6].rstrip()])
                else:
                    InsData[tlist[0]]=[tlist[1]]+[int(t.rstrip()) for t in tlist[2:]]
        InsData['default']=defaultArr
    else:
        raise ValueError(fname+" not found in directory")
    return(InsData)

def parsePlateFormat(fmt):
    '''Interpret what type of rectangular plate format intended by the user'''
    if is_number(fmt):
        NoSpots=int(fmt)
    else:
        NoSpots=fmt
    if NoSpots==384:
        nocols,norows = 24,16
    elif NoSpots==1536:
        nocols,norows = 48,32
    elif NoSpots==768:
        nocols,norows = 48,32
    elif NoSpots==117: # Special request by Marcin Plech
        nocols,norows = 13,9
    elif NoSpots==96:
        nocols,norows = 12,8
    elif NoSpots==48:
        nocols,norows = 6,8
    elif not is_number(fmt) and 'x' in NoSpots and len(NoSpots.split("x"))==2:
        tmp=NoSpots.split("x")
        norows,nocols = int(tmp[0]),int(tmp[1])
    else:
        nocols,norows=0,0
        print ("WARNING: Incorrect rectangular grid format specified!")
        
    return((norows,nocols))

def SetUp(instructarr,imageDate=""):
    '''Parse array read in from text file generated by ColonyzerParametryzer.  Returns coordinates of centre of colonies.'''
    # If we have a list of lists ("default")
    if any(isinstance(el, list) for el in instructarr) and len(instructarr)>1:
        if imageDate != "":
            # Run through list and find all dates
            dates=sorted([ins[5] for ins in instructarr])
            dates.reverse()
            # Find calibration date immediately preceding image
            if imageDate>=min(dates):
                lastDate=dates[[i<=imageDate for i in dates].index(True)]
            else:
                print ("Error, image date not in range of calibration dates")
                return
            for j in range(0,len(instructarr)):
                if instructarr[j][5]==lastDate:
                    tmparr=instructarr[j]
            # Analyse based on relevant calibration
            instructarr=tmparr
        else:
            print ("Please specify an image date to use calibration file with multiple defaults.")
            return
    if any(isinstance(el, list) for el in instructarr) and len(instructarr)==1:
        instructarr=instructarr[0]
    norows,nocols=parsePlateFormat(instructarr[0])
    # Otherwise, just use instrucarr directly (i.e. for image specific calibrations
    TLX,TLY,BRX,BRY=instructarr[1],instructarr[2],instructarr[3],instructarr[4]

    tlx,tly=TLX,TLY
    brx,bry=BRX,BRY

    # Best estimates for tile dimensions
    xdimf=float(abs(brx-tlx))/float(nocols-1)
    ydimf=float(abs(bry-tly))/float(norows-1)
    xdim=int(round(xdimf))
    ydim=int(round(ydimf))

    # Best estimates for the starting coordinates
    xstart=max(0,int(round(float(tlx)-0.5*xdimf)))
    ystart=max(0,int(round(float(tly)-0.5*ydimf)))

    # Parameter for specifying the search area (area is (NSearch*2+1)^2)
    NSearch = int(round(3.0*float(min(xdim,ydim))/8.0))
    candx,candy=[],[]
    for ROW in range(norows):
        for COL in range(nocols):
            candy.append(int(round(ystart+ydimf/2.0+float(ROW)*ydimf)))
            candx.append(int(round(xstart+xdimf/2.0+float(COL)*xdimf)))
    return((candx,candy,xdim,ydim))

def contiguous_regions(condition):
    '''Finds contiguous True regions of the boolean array "condition". Returns
    a 2D array where the first column is the start index of the region and the
    second column is the end index.
    http://stackoverflow.com/questions/4494404/find-large-number-of-consecutive-values-fulfilling-condition-in-a-np-array'''
    # Find the indicies of changes in "condition"
    d = np.diff(condition)
    idx, = d.nonzero() 
    # We need to start things after the change in "condition". Therefore, 
    # we'll shift the index by 1 to the right.
    idx += 1
    if condition[0]:
        # If the start of condition is True prepend a 0
        idx = np.r_[0, idx]
    if condition[-1]:
        # If the end of condition is True, append the length of the array
        idx = np.r_[idx, condition.size] # Edit
    # Reshape the result into two columns
    idx.shape = (-1,2)
    return idx

def getMaxima(intensity):
    '''Numerical method to find local maxima in a 1D list with plateaus'''
    npoints=len(intensity)
    diffs=np.diff(intensity)
    zeroregions=contiguous_regions(diffs==0)
    maxima=[]
    for z in zeroregions:
        if z[0]>0 and z[1]<npoints-2 and diffs[z[0]-1]>0 and diffs[z[1]]<0:
            maxima.append(np.mean(z)+1)
    return(maxima)

def optimiseSpot(arr,x,y,rad,RAD,mkPlots=False):
    '''Search from x-RAD to x+RAD for pixel range dx-rad to dx+rad with the greatest mean intensity (coordinates are top-left corners of cultures)'''
    xmin,xmax=max(0,x-RAD),min(arr.shape[1],x+RAD)
    ymin,ymax=max(0,y-RAD),min(arr.shape[0],y+RAD)
    # Generate windowed mean intensities, scanning along x and y axes
    sumx=np.array([np.mean(arr[ymin:ymax,np.max([0,dx-rad]):np.min([arr.shape[1],dx+rad])]) for dx in range(xmin,xmax)],dtype=np.float)
    sumy=np.array([np.mean(arr[np.max([0,dy-rad]):np.min([arr.shape[0],dy+rad]),xmin:xmax]) for dy in range(ymin,ymax)],dtype=np.float)
    # Find all maxima
    maxx=1+np.where(np.diff(np.sign(np.diff(sumx)))==-2)[0]
    maxy=1+np.where(np.diff(np.sign(np.diff(sumy)))==-2)[0]
    # Get maxima with highest peak
    if len(maxx)>0:
        bestx=maxx[0]
        for dx in maxx:
            if sumx[dx]>sumx[bestx]:
                best=dx
        bestx=xmin+bestx
    else:
        bestx=x
    if len(maxy)>0:
        besty=maxy[0]
        for dy in maxy:
            if sumy[dy]>sumy[besty]:
                best=dy
        besty=ymin+besty
    else:
        besty=y
    if mkPlots:
        fig,ax=plt.subplots(2,2)
        ax[0,0].plot(sumx)
        ax[0,0].axvline(x=bestx-xmin,linestyle='--',linewidth=0.5,color="black")
        ax[0,0].set_xlabel("x")
        ax[0,0].set_ylabel("mean intensity")
        ax[0,1].plot(sumy)
        ax[0,1].axvline(x=besty-ymin,linestyle='--',linewidth=0.5,color="black")
        ax[0,1].set_xlabel("y")
        ax[0,1].set_ylabel("mean intensity")
        ax[1,0].imshow(arr[(y-rad):(y+3*rad),(x-rad):(x+3*rad)]).set_clim(0.0,255.0)
        ax[1,0].add_patch(Rectangle((rad,rad),2*rad,2*rad,alpha=1,facecolor="none"))
        ax[1,0].set_xlabel("x")
        ax[1,0].set_ylabel("y")
        ax[1,1].imshow(arr[(besty-rad):(besty+3*rad),(bestx-rad):(bestx+3*rad)]).set_clim(0.0,255.0)
        ax[1,1].add_patch(Rectangle((rad,rad),2*rad,2*rad,alpha=1,facecolor="none"))
        ax[1,1].set_xlabel("x")
        ax[1,1].set_ylabel("y")
        plt.show()
    return(bestx,besty)

def optimiseSpotCANDIDATE(arr,x,y,rad,RAD,mkPlots=False):
        '''Search from x-RAD to x+RAD for pixel range dx-rad to dx+rad with the greatest mean intensity (coordinates are top-left corners of cultures)'''
        xmin,xmax=max(0,x-RAD),min(arr.shape[1],x+RAD)
        ymin,ymax=max(0,y-RAD),min(arr.shape[0],y+RAD)
        # Generate windowed mean intensities, scanning along x and y axes
        #sumx=np.array([np.mean(arr[ymin:ymax,np.max([0,dx-rad]):np.min([arr.shape[1],dx+rad])]) for dx in range(xmin,xmax)],dtype=np.float)
        #sumy=np.array([np.mean(arr[np.max([0,dy-rad]):np.min([arr.shape[0],dy+rad]),xmin:xmax]) for dy in range(ymin,ymax)],dtype=np.float)

        sumx=np.array([np.mean(arr[y:max(arr.shape[0],y+2*rad),xtarg:max(arr.shape[1],xtarg+2*rad)]) for xtarg in range(xmin,xmax)],dtype=np.float)
        sumy=np.array([np.mean(arr[ytarg:max(arr.shape[0],ytarg+2*rad),x:max(arr.shape[1],x+2*rad)]) for ytarg in range(ymin,ymax)],dtype=np.float)

        bestx=xmin+np.argmax(sumx)
        besty=ymin+np.argmax(sumy)

        if mkPlots:
            fig,ax=plt.subplots(2,2)
            ax[0,0].plot(sumx)
            ax[0,0].axvline(x=bestx-xmin,linestyle='--',linewidth=0.5,color="black")
            ax[0,0].set_xlabel("x")
            ax[0,0].set_ylabel("mean intensity")
            ax[0,1].plot(sumy)
            ax[0,1].axvline(x=besty-ymin,linestyle='--',linewidth=0.5,color="black")
            ax[0,1].set_xlabel("y")
            ax[0,1].set_ylabel("mean intensity")
            ax[1,0].imshow(arr[(y-rad):(y+3*rad),(x-rad):(x+3*rad)]).set_clim(0.0,255.0)
            ax[1,0].add_patch(Rectangle((rad,rad),2*rad,2*rad,alpha=1,facecolor="none"))
            ax[1,0].set_xlabel("x")
            ax[1,0].set_ylabel("y")
            ax[1,1].imshow(arr[(besty-rad):(besty+3*rad),(bestx-rad):(bestx+3*rad)]).set_clim(0.0,255.0)
            ax[1,1].add_patch(Rectangle((rad,rad),2*rad,2*rad,alpha=1,facecolor="none"))
            ax[1,1].set_xlabel("x")
            ax[1,1].set_ylabel("y")
            plt.show()
        return(bestx,besty)

def autocor(x):
    '''R-like autocorrelation function'''
    s = np.fft.fft(x)
    res=np.real(np.fft.ifft(s*np.conjugate(s)))/np.var(x)
    res=res[0:len(res)/2]
    return(res)

def showIm(arr,returnIm=False):
    '''Quick 8-bit preview images from float arrays, useful for debugging'''
    imarr=np.array(arr,dtype=np.uint8)
    if(arr.dtype=="bool"):
        imarr=imarr*255
    imnew=Image.fromarray(imarr,"L")
    if returnIm:
        return(imnew)
    else:
        imnew.show()

def rotateGrid(cor,pos,theta):
    '''Rotate points in pos about centre of rotation cor by angle theta (degrees)'''
    rads=2*math.pi*theta/360.0
    s=math.sin(-rads)
    c=math.cos(-rads)
    post=[(p[0]-cor[0],p[1]-cor[1]) for p in pos]
    posr=[(p[1]*s+p[0]*c,p[1]*c-p[0]*s) for p in post]
    posf=[(int(round(p[0]+cor[0])),int(round(p[1]+cor[1]))) for p in posr]
    return(posf)

def convertij(pos, NCol,oneIndexGrid=False,oneIndexVector=False):
    '''Converts row i, col j into row-major vector index'''
    ## Think about this instead: [i for i,g in enumerate(gpos) if g==(14,2)]
    i = pos[0]-oneIndexGrid
    j = pos[1]-oneIndexGrid
    if(j>=NCol):
        raise ValueError("Column index greater than number of columns")
    return(i*NCol+j+oneIndexVector)

def convertind(ind, NCol,oneIndexGrid=False,oneIndexVector=False):
    '''Converts row-major vector index into row i and col j'''
    row = (ind-oneIndexVector)//NCol
    col = (ind-oneIndexVector)-row*NCol
    return((row+oneIndexGrid, col+oneIndexGrid))

def makeGrid(pos0,ny,nx,dy,dx,theta=0,makeGaps=False,gapsOutside=True):
    '''Generate grid coordinates from top left position, grid dimension gap sizes and angle (rotation about top left).'''
    y0,x0=pos0
    rads=2*math.pi*theta/360.0
    s=math.sin(-rads)
    c=math.cos(-rads)
    vpos=[range(ny),range(nx)]
    gpos=list(itertools.product(*vpos))
    pos=[(y0+gp[0]*dy,x0+gp[1]*dx) for gp in gpos]
    pos=rotateGrid(pos0,pos,theta)
    if makeGaps:
        if gapsOutside: # gaps include outside of grid
            gd=1
        else: # gaps only inside grid
            gd=-1
        vgap=[range(ny+gd),range(nx+gd)]
        ggap=list(itertools.product(*vgap))
        gap=[(int(round(y0-gd*dy/2.0+gp[0]*dy)),int(round(x0-gd*dx/2.0+gp[1]*dx))) for gp in gpos]
        gap=rotateGrid(pos0,gap,theta)
    else:
        gap=()
    return((pos,gap))

def checkPoints(arr,ny,nx,pos0,dy,dx,theta=0,gapsOutside=True):
    '''Return pixel intensities at grid points in (typically smoothed) 2D array'''
    pos,gap=makeGrid(pos0,ny,nx,dy,dx,theta,True,gapsOutside)
    pos=list(zip(*pos))
    gap=list(zip(*gap))
    # Check all points on grid lie within image
    #if any(p>=arr.shape[0] or p<0 for p in pos[0]) or any(p>=arr.shape[1] or p>0 for p in pos[1]):
    pos[0]=[max(0,min(arr.shape[0]-1,p)) for p in pos[0]]
    pos[1]=[max(0,min(arr.shape[1]-1,p)) for p in pos[1]]
    gap[0]=[max(0,min(arr.shape[0]-1,p)) for p in gap[0]]
    gap[1]=[max(0,min(arr.shape[1]-1,p)) for p in gap[1]]
    posvals=arr[pos[0],pos[1]]
    gapvals=arr[gap[0],gap[1]]
    res=np.mean(posvals)-np.mean(gapvals)
    return(res)

def fitProjection(proj,delt,n,sp=0.0,st=0.0,gapsOutside=True):
    '''Find grid position that best fits a 1D projection of intensity by brute force'''
    checkinds=range(int(round(delt/2.0)),int(round(len(proj)-delt*n)))
    if gapsOutside:
        gd=1
    else:
        gd=-1
    def getObj(i,proj,sp,st):
        peaks=proj[i:int(round((i+delt*n))):int(round(delt))]
        troughs=proj[int(round((i-gd*delt/2.0))):int(round((i+delt*(n+gd*0.5)))):int(round(delt))]
        diffs=[p-t for p,t in zip(peaks,troughs)+zip(reversed(peaks),reversed(troughs))]
        return(np.mean(diffs)-sp*np.std(peaks)-st*np.std(troughs))
        #return(np.median(peaks)-np.median(troughs)-sp*np.std(peaks)-st*np.std(troughs))
    grds=[getObj(i,proj,sp,st) for i in checkinds]
    maxind=np.argmax(grds)
    return((checkinds[maxind],grds[maxind]))

def estimateLocations(arr,nx,ny,windowFrac=0.25,smoothWindow=0.13,showPlt=False,pdf=None,acmedian=True,rattol=0.1,glob=False,verbose=False,nsol=36):
    '''Automatically search for best estimate for location of culture array (based on culture centres, not top-left corner).'''
    ### 1: Estimate height and width of spots by examining inter-peak distances in autocorrelation function

    # Generate windowed mean intensities, scanning along x and y axes
    # Estimate spot diameter, assuming grid takes up most of the plate
    diam=min(float(arr.shape[0])/ny,float(arr.shape[1])/nx)
    window=int(round(diam*windowFrac))
    sumx=np.array([np.mean(arr[0:arr.shape[0],np.max([0,dx-window]):np.min([arr.shape[1],dx+window])]) for dx in range(0,arr.shape[1])],dtype=np.float)
    sumy=np.array([np.mean(arr[np.max([0,dy-window]):np.min([arr.shape[0],dy+window]),0:arr.shape[1]]) for dy in range(0,arr.shape[0])],dtype=np.float)
    # Smooth intensities to help eliminate small local maxima
    #sumx=ndimage.gaussian_filter1d(sumx,2.5)
    #sumy=ndimage.gaussian_filter1d(sumy,2.5)

    # Look at autocorrelation for first estimate of distance between spots
    maximay=np.where(np.diff(np.sign(np.diff(autocor(sumy))))==-2)[0]
    maximax=np.where(np.diff(np.sign(np.diff(autocor(sumx))))==-2)[0]
    if acmedian:
        # Note that median inter-peak distance is more robust here
        # Mean is thrown by outliers: gives poor initial guess for optimisation routine
        dy=int(round(np.median(np.diff(maximay))))
        dx=int(round(np.median(np.diff(maximax))))
    else:
        dy=int(round(maximay[0]))
        dx=int(round(maximax[0]))

    # Some plate inoculation patterns (e.g. alternate columns of fit and sick strains in miniQFA) break this ACF procedure (skip every second column)
    # However, the row ACF estimate is still robust.  First estimate: choose the smallest value
    dmin=min(dy,dx)
    dy,dx=dmin,dmin

    # Given shape of grid and size of tiles, search range for x0 and y0, assuming grid parallel to plate edges
    ry=arr.shape[0]-ny*dy
    rx=arr.shape[1]-nx*dx

    ### 2: Find x0,y0 that maximises contrast between grid points and intermediate points (e.g. aligns grid with peaks and troughs),
    ### in horizontal and vertical ACF.  Optimise with fixed dx, for a range of dx.

    dd=int(round(0.01*dx))
    xtest=range(dx-dd,min(int(round(arr.shape[1]/nx))-1,dx+dd))
    ytest=range(dy-dd,min(int(round(arr.shape[0]/ny))-1,dy+dd))
    xvals=[fitProjection(sumx,int(round(dxval)),nx,1.0,1.0,True) for dxval in xtest]
    yvals=[fitProjection(sumy,int(round(dyval)),ny,1.0,1.0,True) for dyval in ytest]
    xind=np.argmin([x[1] for x in xvals])
    yind=np.argmin([y[1] for y in yvals])

    xbest=xvals[xind][0]
    dx=xtest[xind]
    ybest=yvals[yind][0]
    dy=ytest[yind]

    grd,gp=makeGrid((ybest,xbest),ny,nx,dy=dy,dx=dx,theta=0,makeGaps=False)
    candy,candx=list(zip(*grd))
    if showPlt:
        plotAC(sumy,sumx,candy,candx,maximay,maximax,pdf=pdf,main="Projection Estimate")

    ### 3: Optimise positions, first just optimise x0,y0 then update, optimising x0,y0,dx and theta

    smarr=ndimage.filters.gaussian_filter(arr,dmin/10.0)
    #showIm(smarr)

    checkvecs=[range(ry),range(rx)]
    checkpos=list(itertools.product(*checkvecs))

    # Assume we can see the edges of the plate in the image (bright enough to make a peak in the smoothed intensities
    peaksy=np.where(np.diff(np.sign(np.diff(sumy)))==-2)[0]
    peaksx=np.where(np.diff(np.sign(np.diff(sumx)))==-2)[0]
    corner=[peaksy[0],peaksx[0]]

    com=ndimage.measurements.center_of_mass(arr)
    com=[int(round(x)) for x in com]

    #bounds=[(peaksy[0]+dy,ry),(peaksx[0]+dx,rx),(0.8*min(dy,dx),1.2*max(dy,dx)),(-5,5)]
    bounds=[(max(int(round(dy/2.0)),ybest-2.0*dy),min(int(round(arr.shape[0]-(ny-1)*dy)),ybest+2.0*dy)),
         (max(int(round(dx/2.0)),xbest-2.0*dx),min(int(round(arr.shape[1]-(nx-1)*dx)),xbest+2.0*dx)),
         (0.8*min(dy,dx),1.2*max(dy,dx)),
         (-5,5)]

    def makeOptAll(arr,ny,nx,bounds,sampfrac=0.35):
        def optfun(xvs):
            xrs=[b[0]+xv*(b[1]-b[0]) for b,xv in zip(bounds,xvs)]
            #res=-1*checkPos(arr,ny,nx,xrs[0:2],xrs[2],xrs[2],xrs[3],sampfrac=sampfrac)
            res=-1*checkPoints(arr,ny,nx,xrs[0:2],xrs[2],xrs[2],xrs[3])
            return (res)
        return optfun

    def makeOptPos(arr,ny,nx,dx_norm,theta_norm,bounds,sampfrac=0.35):
        dx=bounds[2][0]+dx_norm*(bounds[2][1]-bounds[2][0])
        theta=bounds[3][0]+theta_norm*(bounds[3][1]-bounds[3][0])
        def optfun(xvs):
            xrs=[b[0]+xv*(b[1]-b[0]) for b,xv in zip(bounds[0:2],xvs)]
            #res=-1*checkPos(arr,ny,nx,xrs,dx,dx,theta,sampfrac=sampfrac)
            res=-1*checkPoints(arr,ny,nx,xrs,dx,dx,theta)
            return (res)
        return optfun

    ### 3a: Optimise x0,y0 assuming grid parallel to edges of image and tile dimensions known (ignore theta, dx and dy)
    optmess=False

    dx_norm=(min(dx,dy)-bounds[2][0])/(bounds[2][1]-bounds[2][0])
    optpos=makeOptPos(smarr,ny,nx,dx_norm,0.5,bounds)

    # For even sampling: nsol = Nsamps**Ndim   
    x0s=[sobol.i4_sobol(2,i)[0] for i in range(nsol)]
    firstsol=[op.minimize(optpos,x0=x,method="L-BFGS-B",bounds=[(0.0,1.0) for b in bounds[0:2]],jac=False,options={'eps':0.005,'disp':optmess,'maxiter':5}) for x in x0s]
    solpos=firstsol[np.argmin([sol.fun for sol in firstsol])]

    solnpos=solpos.x
    solnpos=np.append(solnpos,[dx_norm,0.5])

    soln=[b[0]+xv*(b[1]-b[0]) for b,xv in zip(bounds,solnpos)]
    candy,candx=grid(soln,ny,nx)
    if showPlt:
        plotAC(sumy,sumx,candy,candx,maximay,maximax,pdf=pdf,main="First pass")
    ybest,xbest,dx,theta=soln

    ### 3b: Optimise all parameters, using above as initial guess
    optall=makeOptAll(smarr,ny,nx,bounds)

    sol=op.minimize(optall,x0=solnpos,method="L-BFGS-B",bounds=[(0.0,1.0) for b in bounds],jac=False,options={'eps':0.005,'disp':optmess})
    soln=[b[0]+xv*(b[1]-b[0]) for b,xv in zip(bounds,sol.x)]

    ##    x0s.append(xguess[0:2])
    ##    x0s.append(sol1.x[0:2])
    ##    firstpass=[optpos(x) for x in x0s]
    ##    firstguess=x0s[np.argmin(firstpass)]
    ##    sol2=op.minimize(optpos,x0=firstguess,method="L-BFGS-B",bounds=[(0.0,1.0) for b in bounds[0:2]],jac=False,options={'eps':0.005,'disp':optmess,'gtol':0.1})
    ##    soln2=sol2.x
    ##    soln2=np.append(soln2,sol1.x[2:])
    ##    sol=op.minimize(optall,x0=soln2,method="L-BFGS-B",bounds=[(0.0,1.0) for b in bounds],jac=False,options={'eps':0.005,'disp':optmess,'gtol':0.1})

    if verbose:
        print("Optimisation: "+sol.message+"\n\n")
        
    candy,candx=grid(soln,ny,nx)

    # Output some plots
    if showPlt:
        plotAC(sumy,sumx,candy,candx,maximay,maximax,pdf=pdf,main="Solution")

    xguess=list([0.5 for b in bounds])
    init=[b[0]+xv*(b[1]-b[0]) for b,xv in zip(bounds,xguess)]
    return((candx,candy,dx,dy,corner,com,init[0:2]))

def grid(soln,ny,nx):
    pos0=soln[0:2]
    dy,dx=soln[2],soln[2]    
    theta=soln[3]
    grid,gap=makeGrid(pos0,ny,nx,dy=dy,dx=dx,theta=theta,makeGaps=False)
    candy,candx=list(zip(*grid))
    return((candy,candx))

def plotAC(sumy,sumx,candy,candx,maximay,maximax,pdf=None,main=""):
    fig,ax=plt.subplots(2,2,figsize=(15,15))
    acx=autocor(sumx)
    acy=autocor(sumy)
    acxmax=max(len(acx),len(acy))

    ax[0,0].plot(sumx)
    for cand in candx:
        ax[0,0].axvline(x=cand,linestyle='--',linewidth=0.5,color="black")
    ax[0,0].set_xlabel('x coordinate (px)')
    ax[0,0].set_ylabel('Mean Intensity')
    ax[0,0].set_title(main)

    ax[0,1].plot(acx)
    for cand in maximax:
        ax[0,1].axvline(x=cand,linestyle='--',linewidth=0.5,color="black")
    ax[0,1].set_xlabel('Offset dx (px)')
    ax[0,1].set_ylabel('Autocorrelation')
    ax[0,1].set_xlim([0,acxmax])
    ax[0,1].set_title(main)
        
    ax[1,0].plot(sumy)
    for cand in candy:
        ax[1,0].axvline(x=cand,linestyle='--',linewidth=0.5,color="black")
    ax[1,0].set_xlabel('y coordinate (px)')
    ax[1,0].set_ylabel('Mean Intensity')
    ax[1,0].set_title(main)

    ax[1,1].plot(acy)
    for cand in maximay:
        ax[1,1].axvline(x=cand,linestyle='--',linewidth=0.5,color="black")
    ax[1,1].set_xlabel('Offset dy (px)')
    ax[1,1].set_ylabel('Autocorrelation')
    ax[1,1].set_xlim([0,acxmax])
    ax[1,1].set_title(main)
    
    if pdf==None:
        plt.show()
    else:
        pdf.savefig()
        plt.close()
    return()
    
def initialGuess(intensities,counts):
    '''Construct non-parametric guesses for distributions of two components and use these to estimate Gaussian parameters'''
    # Get all maxima
    maxima=1+np.where(np.diff(np.sign(np.diff(counts)))==-2)[0]
    maxima=maxima[counts[maxima]>0.01*np.max(counts)]
    # Get peak heights
    heights=counts[maxima]
    # Order maxima by peak heights
    maxima=maxima[heights.argsort()]
    # Take two biggest peaks as means of two components
    biggest=maxima[-1]
    dists=np.abs(biggest-maxima)
    nextbig_candidates=maxima[dists>len(intensities)/10.0]
    #nextbig_candidates=maxima[maxima>1.1*mu1]
    if(len(nextbig_candidates)>0):
        nextbig=nextbig_candidates[-1]
    else:
        # Assume that single peak dected is agar and second, brighter peak will be cells
        nextbig=int(round(1.5*biggest))
    mu1=min(biggest,nextbig)
    mu2=max(biggest,nextbig)
    
    # Mirror curve from 0...mu1 to estimate distribution of first component
    P1=np.zeros(len(intensities),dtype=np.int)
    halfpeak=counts[0:mu1]
    for i in range(0,mu1):
        P1[i]=halfpeak[i]
    for i in range(mu1,len(intensities)):
        P1[i]=halfpeak[min(len(halfpeak)-1,max(0,2*len(halfpeak)-i))]

    # Mirror curve for second peak also
    P2=np.zeros(len(intensities),dtype=np.int)
    halfpeak=counts[mu2:]
    for i in range(0,mu2):
        P2[i]=halfpeak[min(len(halfpeak)-1,mu2-i)]
    for i in range(mu2,len(intensities)):
        P2[i]=halfpeak[i-mu2]
    
    bindat=pd.DataFrame(intensities,columns=["intensities"])
    bindat["counts"]=counts
    bindat["P1"]=P1
    bindat["P2"]=P2
    # Calculate standard deviation of (binned) observations from first and second components
    sigma1=np.sqrt(np.sum(P1*(np.array(intensities-mu1,dtype=np.float)**2)/np.sum(P1)))
    sigma2=np.sqrt(np.sum(P2*(np.array(intensities-mu2,dtype=np.float)**2)/np.sum(P2)))
    # Estimate component weighting
    theta=float(np.sum(P1))/float(np.sum(P1)+np.sum(P2))
    # Discard empty bins
    bindat=bindat[bindat.counts>0]
    bindat["frac"]=np.array(np.cumsum(bindat.counts),dtype=np.float)/np.sum(bindat.counts)
    bindat["freq"]=np.array(bindat.counts,dtype=np.float)/np.sum(bindat.counts)
    #plotGuess(bindat)
    return((bindat,[theta,mu1,mu2,sigma1,sigma2]))

def totFunc(x,p):
    '''Probability density function for a 2-component mixed Gaussian model'''
    [theta,mu1,mu2,sigma1,sigma2]=p
    if mu2-mu1<2:
        candidate=1e-100
    else:
        candidate=theta*stats.norm.pdf(x,mu1,sigma1)+(1.0-theta)*stats.norm.pdf(x,mu2,sigma2)
    return(candidate)

def makeObjective(ints,cnts,PDF):
    '''Returns a function for (log likelihood)*-1 (suitable for minimisation), given a set of binned observations and a PDF'''
    ints=np.array(ints,dtype=np.int)
    cnts=np.array(cnts,dtype=np.int)
    def logL(p):
        modeldens=np.array([PDF(x,p) for x in ints],dtype=np.float)
        lik=np.sum(cnts*np.log(modeldens))
        return(-1*lik)
    return(logL)

def getRoot(p,ints):
    '''Get the point at which two component Gaussians intersect.  Specifically looking for root with highest probability.'''
    [theta,mu1,mu2,sigma1,sigma2]=p
    ints=np.array(ints,dtype=np.int)
    def diffFunc(x):
        return(theta*stats.norm.pdf(x,mu1,sigma1)-(1.0-theta)*stats.norm.pdf(x,mu2,sigma2))
    # Find pairs of points in truncated, filtered intensity list which bracket any roots
    diffs=[np.sign(diffFunc(x)) for x in ints]
    switches=[]
    for i in range(1,len(diffs)):
        if abs((diffs[i]-diffs[i-1]))==2:
            switches.append((i,i-1))
    # Fine-tune root locations
    threshlist=[]
    for switch in switches:
        thresh=optimize.brentq(diffFunc,ints[switch[0]],ints[switch[1]])
        threshlist.append(thresh)
    # Get root which gives the highest probability for peak 1 (or peak 2, either is fine)
    p1=[stats.norm.pdf(thresh,mu1,sigma1) for thresh in threshlist]
    thresh=threshlist[np.argmax(p1)]
    return(thresh)

def thresholdArr(arrim,thresh):
    '''Thresholding array representation of an image'''
    if thresh>0:
        arrim[arrim<thresh]=0
        arrim[arrim>=thresh]=255
    else:
        arrim=np.round(arrim)
    arrim=np.array(arrim,dtype=np.uint8)
    imnew=Image.fromarray(arrim, "L")
    return(imnew)

def plotGuess(bindat,label="",pdf=None):
    '''Plot intensity frequency histogram and non-parametric estimates of component distributions'''
    plt.figure()
    plt.plot(bindat.intensities,bindat.counts,color="black")
    plt.plot(bindat.intensities,bindat.P1,color="red")
    plt.plot(bindat.intensities,bindat.P2,color="blue")
    plt.xlabel('Intensity')
    plt.ylabel('Frequency')
    plt.suptitle(label)
    if pdf==None:
        plt.show()
    else:
        pdf.savefig()
        plt.close()

def plotModel(bindat,thresholds=(),label="",pdf=None):
    '''Plot intensity density histogram, modelled distribution, component distributions and threshold estimate.'''
    plt.figure()
    plt.plot(bindat.intensities,bindat.freq,color="black")
    plt.plot(bindat.intensities,bindat.gauss1,color="red")
    plt.plot(bindat.intensities,bindat.gauss2,color="blue")
    plt.plot(bindat.intensities,bindat.mixed,color="green")
    plt.xlabel('Intensity')
    plt.ylabel('Density')
    plt.suptitle(label)
    for thresh in thresholds:
        plt.axvline(x=thresh,linestyle='--',linewidth=0.5,color="darkorchid")
    if pdf==None:
        plt.show()
    else:
        pdf.savefig()
        plt.close()

def getEdges(arr,cutoff=0.9975):
    '''Sobel edge detection for 2d array using scipy functions'''
    sx = ndimage.sobel(arr, axis=0)
    sy = ndimage.sobel(arr, axis=1)
    sob = np.hypot(sx, sy)
    sob[sob<stats.mstats.mquantiles(sob,cutoff)[0]]=0
    sob[sob>0]=1
    return(np.array(sob,dtype=np.bool))  

def sizeSpots(locations,arr,thresharr,edge,background=0):
    '''Add intensity measures and other phenotypes to locations dataFrame'''
    intMax=255.0
    # http://en.wikipedia.org/wiki/Shape_factor_(image_analysis_and_microscopy)#Circularity
    # Calculate area, intensity and trimmed intensity for each spot
    sumInt,sumArea,trim,fMed,bMed,circ,fVar,perim=[],[],[],[],[],[],[],[]
    for i in range(0,len(locations.x.values)):
        x,y,rad=locations.x.values[i],locations.y.values[i],int(math.ceil(max(locations.Diameter.values)/2.0))
        tile=arr[max(0,y-rad):min(arr.shape[0],(y+rad+1)),max(0,x-rad):min(arr.shape[1],(x+rad+1))]-background
        threshtile=thresharr[max(0,y-rad):min(arr.shape[0],(y+rad+1)),max(0,x-rad):min(arr.shape[1],(x+rad+1))]
        edgetile=edge[max(0,y-rad):min(arr.shape[0],(y+rad+1)),max(0,x-rad):min(arr.shape[1],(x+rad+1))]
        perimeter=np.sum(edgetile)
        area=np.sum(threshtile)
        if perimeter>0:
            circularity=4*math.pi*area/(perimeter)**2
        else:
            circularity=0
        tthresh=tile[threshtile]
        if tthresh.size>1:
            featureMedian=np.median(tthresh/intMax)
            featureVariance=np.var(tthresh/intMax)
            allTrim=float(np.sum(tthresh))/(float(tile.size)*intMax)
        else:
            featureMedian=featureVariance=allTrim=0
        bkgrnd=tile[np.logical_not(threshtile)]
        if bkgrnd.size>1:
            backgroundMedian=np.median(bkgrnd/intMax)
        else:
            backgroundMedian=0
        sumInt.append(max(0,float(np.sum(tile))/(float(tile.size)*intMax)))
        sumArea.append(max(0,float(area)/float(tile.size)))
        trim.append(max(0,allTrim))
        fMed.append(featureMedian)
        bMed.append(backgroundMedian)
        circ.append(circularity)
        fVar.append(featureVariance)
        perim.append(float(perimeter)/float(tile.size))
    locations["Intensity"]=sumInt
    locations["Area"]=sumArea
    locations["Trimmed"]=trim
    locations["FeatureMedian"]=fMed
    locations["FeatureVariance"]=fVar
    locations["BackgroundMedian"]=bMed
    locations["Circularity"]=circ
    locations["Perimeter"]=perim
    return(locations)

def getColours(im,locations,thresharr):
    '''Extract feature and background mean and median Red Green and Blue channel values for a given 24 bit image'''
    (red,green,blue)=im.split()
    redarr,greenarr,bluearr=np.array(red,dtype=np.uint8),np.array(green,dtype=np.uint8),np.array(blue,dtype=np.uint8)
    r,g,b,rB,gB,bB,rm,gm,bm,rmB,gmB,bmB=[],[],[],[],[],[],[],[],[],[],[],[]
    store=np.zeros((len(locations.x.values),12),np.float)
    for i in range(0,len(locations.x.values)):
        x,y,rad=locations.x.values[i],locations.y.values[i],int(math.ceil(max(locations.Diameter.values)/2.0))
        redtile=redarr[y-rad:(y+rad+1),x-rad:(x+rad+1)]
        greentile=greenarr[y-rad:(y+rad+1),x-rad:(x+rad+1)]
        bluetile=bluearr[y-rad:(y+rad+1),x-rad:(x+rad+1)]
        threshtile=thresharr[y-rad:(y+rad+1),x-rad:(x+rad+1)]
        rMean,gMean,bMean=np.mean(redtile[threshtile]),np.mean(greentile[threshtile]),np.mean(bluetile[threshtile])
        rMed,gMed,bMed=np.median(redtile[threshtile]),np.median(greentile[threshtile]),np.median(bluetile[threshtile])
        rMeanBk,gMeanBk,bMeanBk=np.mean(redtile[np.logical_not(threshtile)]),np.mean(greentile[np.logical_not(threshtile)]),np.mean(bluetile[np.logical_not(threshtile)])
        rMedBk,gMedBk,bMedBk=np.median(redtile[np.logical_not(threshtile)]),np.median(greentile[np.logical_not(threshtile)]),np.median(bluetile[np.logical_not(threshtile)])
        store[i]=[rMean,gMean,bMean,rMeanBk,gMeanBk,bMeanBk,rMed,gMed,bMed,rMedBk,gMedBk,bMedBk]
    locations["redMean"]=store[:,0]
    locations["greenMean"]=store[:,1]
    locations["blueMean"]=store[:,2]
    locations["redMeanBack"]=store[:,3]
    locations["greenMeanBack"]=store[:,4]
    locations["blueMeanBack"]=store[:,5]
    locations["redMedian"]=store[:,6]
    locations["greenMedian"]=store[:,7]
    locations["blueMedian"]=store[:,8]
    locations["redMedianBack"]=store[:,9]
    locations["greenMedianBack"]=store[:,10]
    locations["blueMedianBack"]=store[:,11]
    return(locations)      

def saveColonyzer(filename,locs,thresh,dx,dy):
    '''Save output data in original Colonyzer format'''
    # FILENAME ROW COLUMN TOPLEFTX TOPLEFTY WHITEAREA(px) TRIMMED THRESHOLD INTENSITY EDGEPIXELS COLR COLG COLB BKR BKG BKB EDGELEN XDIM YDIM
    df={}
    df["FILENAME"]=locs["Filename"].values
    df["ROW"]=locs["Row"].values
    df["COLUMN"]=locs["Column"].values
    df["TOPLEFTX"]=locs["x"].values-locs["Diameter"].values/2.0
    df["TOPLEFTY"]=locs["y"].values-locs["Diameter"].values/2.0
    df["WHITEAREA"]=locs["Area"].values*dx*dy*255.0
    df["TRIMMED"]=locs["Trimmed"].values*dx*dy*255.0
    df["THRESHOLD"]=thresh
    df["INTENSITY"]=locs["Intensity"].values*dx*dy*255.0
    df["EDGEPIXELS"]=locs["FeatureMedian"].values ### NOTE LABEL INCORRECT!
    df["COLR"]=locs["redMedian"].values
    df["COLG"]=locs["greenMedian"].values
    df["COLB"]=locs["blueMedian"].values
    df["BKR"]=locs["redMedianBack"].values
    df["BKG"]=locs["greenMedianBack"].values
    df["BKB"]=locs["blueMedianBack"].values
    df["EDGELEN"]=locs["Perimeter"].values
    df["XDIM"]=dx
    df["YDIM"]=dy
    colorder=("FILENAME","ROW","COLUMN","TOPLEFTX","TOPLEFTY","WHITEAREA","TRIMMED","THRESHOLD","INTENSITY","EDGEPIXELS","COLR","COLG","COLB","BKR","BKG","BKB","EDGELEN","XDIM","YDIM")
    dataf=pd.DataFrame(df)
    dataf.reindex_axis(colorder, axis=1)
    dataf.to_csv(filename,"\t",index=False,header=False,columns=colorder,engine='python')
    return(dataf)

def setupDirectories(dictlist,verbose=True):
    '''Create output directories and return paths for writing/reading files'''
    if isinstance(dictlist,dict):
        # Flatten dictionary to list:
        # dictlist= [x for d in dictlist.itervalues() for x in d] # Python 2.7
        dictlist= [x for d in dictlist.values() for x in d] 
    # Else assume a list

    # Get unique set of directories in list:
    dirs=np.unique([os.path.dirname(fname) for fname in dictlist])
    newdirs=[]
    # Create directories for storing output data and preview images
    for directory in dirs:
        imdir=os.path.join(directory,"Output_Images")
        datdir=os.path.join(directory,"Output_Data")
        repdir=os.path.join(directory,"Output_Reports")
        try:
            os.mkdir(imdir)
            os.mkdir(datdir)
            os.mkdir(repdir)
            if verbose: print("Created "+imdir+" & "+datdir+" & "+repdir+".")
            newdirs.append(directory)
        except:
            continue
    return(newdirs)

def getImageNames(fullpath):
    '''Get filenames for all images which have not yet been analysed.'''
    allfiles=[]
    alldats=[]
    for dirname, dirnames, filenames in os.walk(fullpath):
        if not any([x in dirname for x in ['Output_Images','.git']]):
            for filename in filenames:
                if filename[-4:] in ('.jpg','.JPG','.tiff','.TIFF','.tif','.TIF'):
                    allfiles.append(os.path.join(dirname, filename))
                elif filename[-4:] == '.out':
                    alldats.append(os.path.join(dirname, filename))
    imsDone=list(np.unique([os.path.basename(dat).split(".")[0] for dat in alldats]))
    imList=[]
    for filename in allfiles:
        imname=os.path.basename(filename).split(".")[0]
        if imname not in imsDone:
            imList.append(filename)
    imList.sort(reverse=True)
    return(imList)

def checkAnalysisStarted(imname):
    '''Check if Colonyzer has already analysed or is currently analysing the photo imname'''
    base=os.path.basename(imname)
    baseroot=base.split(".")[0]
    dirname=os.path.dirname(imname)
    return(os.path.exists(os.path.join(dirname,"Output_Data",baseroot+".out")))

def getBarcodes(fullpath,barcRange=(0,15),checkDone=True,verbose=False):
    '''Get filenames for all images in current directory and all sub-directories.
    Return a dictionary of filenames, listed by barcode (plate ID)'''
    allfiles=[]
    alldats=[]
    for dirname, dirnames, filenames in os.walk(fullpath):
        if not any([x in dirname for x in ['Output_Images','.git']]):
            for filename in filenames:
                if filename[-4:] in ('.jpg','.JPG','.tiff','.TIFF','.tif','.TIF'):
                    allfiles.append(os.path.join(dirname, filename))
                elif filename[-4:] == '.out':
                    alldats.append(os.path.join(dirname, filename))
    imsDone=list(np.unique([os.path.basename(dat).split(".")[0] for dat in alldats]))
    if checkDone:
        barcsDone=list(np.unique([os.path.basename(dat)[barcRange[0]:barcRange[1]] for dat in alldats]))
    else:
        barcsDone=[]
    barcdict={}
    for filename in allfiles:
        fname=os.path.basename(filename)
        fbase=fname.split(".")[0]
        barc=fname[barcRange[0]:barcRange[1]]
        if fbase not in imsDone:
            if barc not in barcsDone:
                if barc not in barcdict:
                    barcdict[barc]=[filename]
                else:
                    barcdict[barc].append(filename)
    for b in barcdict:
        fnames=np.array([os.path.basename(x) for x in barcdict[b]])
        barcdict[b]=list(np.array(barcdict[b])[fnames.argsort()])[::-1]
        #barcdict[b].sort(reverse=True)
    if verbose and not barcdict: print("No new images to analyse found in "+fullpath+".")
    return(barcdict)

def merge_dols(dol1, dol2):
    '''Merge two dictionaries of lists'''
    # http://stackoverflow.com/questions/1495510/combining-dictionaries-of-lists-in-python
    keys = set(dol1).union(dol2)
    no = []
    return dict((k, dol1.get(k, no) + dol2.get(k, no)) for k in keys)

def merge_lodols(dolList):
    '''Merge a list of dictionaries of lists'''
    tmp={}
    for dol in dolList:
        tmp=merge_dols(tmp,dol)
    return(tmp)

def openImage(imName):
    '''Open an image, strip alpha channel, convert to array of floats.'''
    im=Image.open(imName)
    # Strip alpha channel if present
    im = im.convert("RGB")
    img=im.convert("F")
    arrN=np.array(img,dtype=np.float)
    return(im,arrN)

def locateCulturesScan(candx,candy,dx,dy,arrN,nx,ny,search=0.4,radFrac=1.0,mkPlots=False,update=True):
    '''Starting with initial guesses for culture locations (top left corner), optimise individual culture locations and return locations (centre of spots) data frame.'''
    # radius is half width of spot tile, rad is "radius" of area tested for brightness (0<radnum<=1.0), RAD is half width of search space
    colvals,rowvals=np.meshgrid(np.arange(1,nx+1),np.arange(1,ny+1))
    d={"Row":rowvals.flatten(),"Column":colvals.flatten(),"y":candy,"x":candx}
    locations=pd.DataFrame(d)
    locations["Diameter"]=min(dx,dy)
    if update:
        radius=float(min(dx,dy))/2.0
        rad=radFrac*radius
        delta=int(round((radius-rad)/2.0))
        rad=int(round(rad))
        RAD=int(round(search*radius))
        for i in range(0,len(locations.x)):
            (x,y)=optimiseSpot(arrN,locations.x[i]+delta,locations.y[i]+delta,rad,RAD,mkPlots)
            # Note this returns coordinates of CENTRE OF SPOT
            locations.x[i]=int(round(x-delta+dx/2.0))
            locations.y[i]=int(round(y-delta+dy/2.0))
        print("Cultures located")
    else:
        locations.x=locations.x+dx/2.0
        locations.y=locations.y+dy/2.0
    return(locations)

def edgeBrightness(tile):
    '''Sum up the values (brightnesses) along the edge of a 2D array (tile from a monochrome image).'''
    if tile.size<=0:
        return(99999999999)
    else:
        return(np.mean(np.concatenate((tile[0,:],tile[-1,:],tile[1:-1,0],tile[1:-1,-1]))))

def locateCultures(candx,candy,dx,dy,arr,nx,ny,update=True,maxupdates=5,fuzzy=0.01,samp=1.0):
    '''Recursively calculate centre of mass for each tile until it converges (or updates maxupdates times).'''
    cols,rows=np.meshgrid(np.arange(1,nx+1),np.arange(1,ny+1))
    cx=list(candx)
    cy=list(candy)
    dx=int(round(dx))
    dy=int(round(dy))

    def measure(pos):
        cy0,cx0=max(0,pos[0]),max(0,pos[1])
        yA,yB=cy0,min((cy0+dy),arr.shape[0]-1)
        xA,xB=cx0,min((cx0+dx),arr.shape[1]-1)
        tile=arr[yA:yB,xA:xB]
        COM=ndimage.measurements.center_of_mass(tile)
        if sum(np.isnan(COM))>0:
            COM=(dy/2.0,dx/2.0)
        cy=min(max(0,int(round(cy0+COM[0]-dy/2.0))),arr.shape[0]-1)
        cx=min(max(0,int(round(cx0+COM[1]-dx/2.0))),arr.shape[1]-1)
        return((pos,(cy,cx),edgeBrightness(tile),tile.sum()))
    
    def updateLocation(pos):
        # Recursive iteration looking for centre of mass, recording brightness and edge length
        res=[measure(pos)]
        new=measure(res[-1][1])
        while new!=res[-1] and len(res)<=maxupdates:
            res.append(new)
            new=measure(res[-1][1])
        # Accept latest solution in series that gives edge length within a factor of (1+fuzzy) of the minimum observed
        edges=[r[2] for r in res]
        edgeMin=min(edges)
        res_sort=[r for r in res if (r[2]<=(1+fuzzy)*edgeMin and np.linalg.norm(np.array(r[0])-np.array(r[1]))<=max(dx,dy)/2.0)]
        if len(res_sort)>0:
            sol=res_sort[-1]
            # Only update if brightness has increased by a factor of less than 2, otherwise likely an error
            if sol[3]<2*res[0][3]:
                return(sol[0])
        return(pos)

    if update:
        posnew=[updateLocation(p) for p in zip(cy,cx)]
    cy,cx=zip(*posnew)
    
    colvals,rowvals=np.meshgrid(np.arange(1,nx+1),np.arange(1,ny+1))
    d={"Row":rowvals.flatten(),"Column":colvals.flatten(),"y":[cyv+dy/2.0 for cyv in cy],"x":[cxv+dx/2.0 for cxv in cx]}
    locations=pd.DataFrame(d)
    locations["Diameter"]=min(dx,dy)
    return(locations)

def makeMask(arrN,thresh1,tol=5):
    '''Generate an agar mask and a pseudo-empty image from a plate with obvious cultures.  Cultures are identified by thresholding, cut out and filled using a Markov field update.'''
    # Tolerance for average pixel intensity difference between iterations to declare convergence of Markov update
    # Save final mask for cutting out all cell signal from earlier images
    finalMask=np.ones(arrN.shape,dtype=np.bool)
    finalMask[arrN<thresh1]=False
    cutout_arr=maskAndFill(arrN,finalMask,tol)
    return (finalMask,cutout_arr)

def maskAndFill(arrN,finalMask,tol=5):
    '''Cut out masked pixels from image and re-fill using a Markov field update.'''
    # Unmask edges to allow Markov field update
    finalMask[0,:]=False
    finalMask[-1,:]=False
    finalMask[:,0]=False
    finalMask[:,-1]=False
    cutout_arr=np.copy(arrN)
    cutout_arr[finalMask]=np.nan
    old=np.zeros(cutout_arr.shape,dtype=float)
    
    (y_list,x_list)=np.where(finalMask)
    print("Filling in gaps")
    diff=100*tol
    while diff>tol or np.isnan(diff):
        # Invert filling order at every pass to minimise bias towards a particular direction
        x_list=x_list[::-1]
        y_list=y_list[::-1]
        old=np.copy(cutout_arr)
        # Markov field update
        for i in range(0,len(x_list)):
            plist=[cutout_arr[y_list[i],x_list[i]+1],cutout_arr[y_list[i]+1,x_list[i]],cutout_arr[y_list[i],x_list[i]-1],cutout_arr[y_list[i]-1,x_list[i]]]
            cutout_arr[y_list[i],x_list[i]]=stats.nanmean(plist)
        diff=np.sum(np.abs(old-cutout_arr))/np.sum(finalMask)
        diff=np.sum(np.abs(old*finalMask-cutout_arr*finalMask))/np.sum(finalMask)
    return(cutout_arr)
    
def makeCorrectionMap(arr0,locations,smoothfactor=250,verbose=True):
    '''Smooth a (pseudo-)empty plate image to generate a correction map.'''
    dy,dx=locations.Diameter[0],locations.Diameter[0]
    smoothed_arr=ndimage.gaussian_filter(arr0,arr0.shape[1]/smoothfactor)
    average_back=np.median(smoothed_arr[np.min(locations.y):np.max(locations.y),np.min(locations.x):np.max(locations.x)])
    correction_map=average_back/smoothed_arr
    if verbose: print("Lighting correction map constructed.")
    return(correction_map,average_back)

def measureSizeAndColour(locations,arr,im,finalmask,average_back,barcode,filename):
    '''Generate culture size and colour estimates given pixel array, culture locations and an image mask.'''
    edge=getEdges(arr,0.925)
    locations=sizeSpots(locations,arr,finalmask,edge,average_back)
    locations=getColours(im,locations,finalmask)
    locations["Barcode"]=barcode
    locations["Filename"]=os.path.basename(filename).split(".")[0]
    return(locations)

def threshPreview(locations,arr,thresh1=None,linethick=10,circlerad=10):
    '''Generate a preview version of thresholded image with culture locations highlighted (coloured squares).  Suitable for checking that culture location algorithms are functioning.'''
    if arr.max()==1 and arr.min()==0: # if arr is a mask
        imthresh=thresholdArr(np.copy(arr)*255.0,127.5).convert("RGB")
    else:
        imthresh=thresholdArr(np.copy(arr),thresh1).convert("RGB")
    draw=ImageDraw.Draw(imthresh)
    colours=((255,0,0),(0,255,0),(0,0,255),(255,255,0),(0,255,255),(255,0,255))
    for i in range(0,len(locations.x)):
        x,y,r=int(round(locations.x[i])),int(round(locations.y[i])),int(round(float(locations.Diameter[i])/2.0))
        for delta in range(linethick):
            draw.rectangle((x-r+delta,y-r+delta,x+r-delta,y+r-delta),outline=colours[i%5])
            draw.ellipse((x-circlerad,y-circlerad,x+circlerad,y+circlerad),fill=colours[i%5])
    return(imthresh)

def automaticThreshold(arr,label="",pdf=None):
    '''Choose a threshold for segmenting pixel intensities by fitting two-component Gaussian mixture model'''
    # Initial guess for mixed model parameters for thresholding lighting corrected image
    (counts,intensities)=np.histogram(arr,bins=2**8,range=(0,2**8))
    intensities=np.array(intensities[0:-1],dtype=np.int)
    smoothcounts=ndimage.gaussian_filter1d(counts,1)
    (bindat,[theta,mu1,mu2,sigma1,sigma2])=initialGuess(intensities,smoothcounts)
    if(pdf!=None):
        plotGuess(bindat,label,pdf)

    # Maximise likelihood of 2-component mixed Gaussian model parameters given binned observations by constrained optimisation
    logL=makeObjective(bindat.intensities,bindat.counts,totFunc)
    b=[(0.0,1.0),(float(mu1)/5.0,5*float(mu1)),(float(mu2)/5.0,5.0*float(mu2)),(float(sigma1)/5.0,5.0*float(sigma1)),(float(sigma2)/5.0,5.0*float(sigma2))]
    opt=optimize.fmin_l_bfgs_b(logL,[theta,mu1,mu2,sigma1,sigma2],bounds=b,approx_grad=True)
    [theta_opt,mu1_opt,mu2_opt,sigma1_opt,sigma2_opt]=opt[0]

    thresh=getRoot(opt[0],intensities)
    # Best estimate for threshold is point of intersection of two fitted component Gaussians
    thresh1=int(round(thresh))

    # Make threshold as low as possible, for maximum sensitivity
    while smoothcounts[thresh1]>=smoothcounts[thresh1-1]:
        thresh1-=1

    # Modelled densities
    bindat["mixed"]=np.array([totFunc(x,opt[0]) for x in bindat.intensities],dtype=np.float)
    bindat["gauss1"]=np.array([theta_opt*stats.norm.pdf(x,mu1_opt,sigma1_opt) for x in bindat.intensities],dtype=np.float)
    bindat["gauss2"]=np.array([(1.0-theta_opt)*stats.norm.pdf(x,mu2_opt,sigma2_opt) for x in bindat.intensities],dtype=np.float)
    return((thresh1,bindat))

def openQFA(fname):
    '''Reads tab-delimited QFA data, processes it and returns dataframe'''
    res=pd.read_csv(fname,sep="\t")
    # Tidy up columns (maybe add these lines to a read-in-data function instead?)
    if "Treatments" in res.columns:
        res.rename(columns={"Treatments":"Treatment","X.Offset":"XOffset","Y.Offset":"YOffset","Tile.Dimensions.X":"TileX","Tile.Dimensions.Y":"TileY"},inplace=True)
    res["TreatMed"]=res["Treatment"].map(str)+"_"+res["Medium"].map(str)
    res=res.dropna(axis=0,how="all")
    res=res[pd.notnull(res["Treatment"])]
    return(res)

def pad(x,zeros=2):
    '''Pads an integer to a two-character string with leading zero or just return string'''
    try:
        return(("%0"+str(zeros)+"d")%x)
    except:
        if x!=x: # Check for nan
            x="missing"
        return(x)

def viewerSummary(res):
    '''Generate report to help building vertical and horizontal categories for image viewer'''   
    print ("Data summary")
    print ("~~~~~~~~~~~~")
    print ("Barcode: "+str(len(res["Barcode"].unique())))
    print ("Library Plates: "+str(len(res["MasterPlate.Number"].unique())))
    print ("SGA replicate plates: "+str(len(res["RepQuad"].unique())))
    print ("Screen identifiers: "+str(len(res["Screen.Name"].unique()))+"("+",".join([str(x) for x in res["Screen.Name"].unique()])+")")
    if "ScreenID" in res.columns:
        print ("Screen IDs: "+str(len(res["ScreenID"].unique()))+"("+",".join([str(x) for x in res["ScreenID"].unique()])+")")
    print ("Libraries: "+str(len(res["Library.Name"].unique()))+"("+",".join([str(x) for x in res["Library.Name"].unique()])+")")
    print ("Treatment: "+str(len(res["Treatment"].unique()))+"("+",".join([str(x) for x in res["Treatment"].unique()])+")")
    print ("Medium: "+str(len(res["Medium"].unique()))+"("+",".join([str(x) for x in res["Medium"].unique()])+")")
    print ("TreatMed: "+str(len(res["TreatMed"].unique()))+"("+",".join([str(x) for x in res["TreatMed"].unique()])+")")
    print ("")

def getDate(x,fmt="%Y-%m-%d_%H-%M-%S"):
    lenf=len(time.strftime(fmt))
    try:
        dt=datetime.strptime(x.split(".")[-2][(-1*lenf):],fmt)
    except:
        dt=datetime(9999,1,1)
    return(dt)    

def getNearest(barcs,exptTime=1.0):
    '''Find one image whose time captured is closest to exptTime for all barcodes in a dictionary of file paths (barcs)'''
    closestImage={}
    for b in barcs:
        dates=[getDate(x) for x in barcs[b]]
        first=min(dates)
        datediffs=[date-first for date in dates]
        diffs=[(x.total_seconds()/(60*60*24.0))-exptTime for x in datediffs]
        absdiffs=[abs(diff) for diff in diffs]
        bestind=np.argmin(absdiffs)
        closestImage[b]=barcs[b][bestind]
    return(closestImage)

def makeHoriz(res,horizontal):
    '''Make a list of horizontal identifiers based on the "horizontal" column in the data frame res'''
    horiz=res[horizontal].unique()
    horiz=[x for x in horiz if x==x] # Get rid of nans
    return(horiz)

def makeCSS(fname=None):
    cssString='''img {
    position:absolute;
    top: 250px;
    left: 0px;
    border: 0;
}

body{
    font-family: 'Open Sans', sans-serif;
    font-size: 1.5em;
}

h2 {
    margin: 0px;
    padding: 0px;
    color: black;
    font-size: 2.5em;
    font-weight: 600;
}

h3 {
    margin: 0px;
    padding: 0px;
    color: black;
    font-size: 1.5em;
    font-weight: 500;
}

a:link {	
    color: black;
    font-weight: 700; 
    text-decoration: none;
}

a:visited {
    color: black;
    font-weight: 700; 
    text-decoration: none;
    border-bottom:0px dashed
}

a:hover{background: black;
    color: white;
    font-weight: 700;
}
'''
    if fname is not None:
        f=open(fname,"w")
        f.write(cssString)
        f.close()
    else:
        return(cssString)

def makeSCRIPT(ims):
    assert isinstance(ims, (list,tuple))
    imlst=json.dumps(ims)
    script='''<script>
    var images = {0};
    var numim = images.length;
    var index = 0;
    function setPicture()
    {{
        document.getElementById('imagetochange').src=images[index];
        document.getElementById('texttochange').innerHTML=images[index];
    }}
    function updatePicture()
    {{
        index=(index+1)%numim
    }}
</script>
'''.format(imlst)
    return(script)

def makeHEAD(ims):
    head='''<head>
<meta charset=utf-8>
<title>QFA image browser generated by Colonyzer</title>
<link href='http://fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,800italic,400,300,600,700,800' rel='stylesheet' type='text/css'>
<link rel="STYLESHEET" href="imbrowse.css" type="text/css">
'''
    head+=makeSCRIPT(ims)
    head+='''</head>
<base target='_blank' />
'''
    return(head)
    
def makePage(res,closestImage,horizontal,htmlroot="index",title="",scl=1,smw=600,highlight={},hitPercentile=100, deadPercentile=-99,outPath="."):
    '''Make a html preview of images listed in res, columns by "horizonal", filename htmlroot, report title, genes to highlight colour:list.'''
    # List of possible identifiers, by which experiment can be separated
    # If we need to sort final image differently, sort this list appropriately
    hitThresh=np.percentile(res["fit"],hitPercentile)
    if deadPercentile>=0.0:
        deadThresh=np.percentile(res["fit"],deadPercentile)
    else:
        deadThresh=-9999999999
    All_IDs=["MasterPlate.Number","RepQuad","Screen.Name","Condition","Library.Name","Treatment","Medium","Inoc"]
    horiz=makeHoriz(res,horizontal)
    # Build an ID which doesn't include the horizontal identifier
    # or any identifier which is effectively the same as the horizontal identifier
    IDs=[x for x in All_IDs if x != horizontal and len((res[x].map(pad)+res[horizontal].map(pad)).unique())>len(horiz)]
    res["vertID"]=""
    for i in range(0,len(IDs)):
        res["vertID"]=res["vertID"]+res[IDs[i]].map(pad)+"_"

    # Use Screen.Name as title if none provided
    if title=="":
        title=res["Screen.Name"][0]
        
    # Construct a replicate ID
    # Split data by horizontal
    hSplit=[res[res[horizontal]==x] for x in horiz]
    for cno,h in enumerate(hSplit):
        print("Setting up data for column "+str(cno))
        barcRepDict={}
        h["Replicate"]=0
        # Give each barcode a technical replicate number
        vertIDs=h["vertID"].unique()
        for vID in vertIDs:
            vIDbarcs=h["Barcode"][h["vertID"]==vID].unique()
            for repno,barcval in enumerate(vIDbarcs):
                barcRepDict[barcval]=repno+1
        h["Replicate"]=h["Barcode"].replace(barcRepDict)
        h["vertID"]=h["vertID"]+h["Replicate"].map(pad)
        h.sort("vertID",inplace=True)
        
    res=hSplit[0]
    if len(hSplit)>1:
        for i in range(1,len(hSplit)):
            res=res.append(hSplit[i])

    All_IDs.append("Replicate")

    # We need to construct an identifier that is common to all elements of horizontal
    # but will be different for all rows.  Tricky part is if some elements of horizontal have
    # fewer rows than other (e.g. technical replicates for one expt., but not another)...

    # First, eliminate identifiers that are common to all plates in res
    diffIDs=[x for x in All_IDs if len(res[x].unique())>1 and x not in horizontal and len((res[x].map(pad)+res[horizontal].map(pad)).unique())>len(horiz)]
    res["vID"]=""
    for i in range(0,len(diffIDs)):
        res["vID"]=res["vID"]+res[diffIDs[i]].map(pad)+"#"

    horiz=makeHoriz(res,horizontal)
    imList=[res[res[horizontal]==x] for x in horiz]
    vIDlist=res["vID"].unique()
    vIDlist.sort()
    # For the moment, assume that all images have the same aspect ratio
    allBarcs=[]
    for im in imList:
        for b in im["Barcode"].unique():
            allBarcs.append(b)
    wset,hset,aset=set(),set(),set()
    print("Scanning through images for sizes and aspect ratios")
    for climind,clim in enumerate(closestImage):
      for barc in allBarcs:
          impath=clim[barc]
          imroot=os.path.basename(impath).split(".")[0]
          w,h=Image.open(impath).size
          wset.add(w)
          hset.add(h)
          aset.add(float(w)/float(h))
    (imw,imh)=(max(wset),max(hset))

    smh=int(round(float(imh)*float(smw)/float(imw)))

    # Scale factors for coordinates of preview images
    scalex=(float(smw)/float(imw))
    scaley=(float(smh)/float(imh))

    W=len(horiz)
    H=len(vIDlist)

    # Prepare blank slates for large preview images
    plateArr=[Image.new('RGB',(int(round(scl*smw*W)),int(round(scl*smh*H))),color=0) for x in range(len(closestImage))]
    platePosArr=[Image.new('RGB',(int(round(scl*smw*W)),int(round(scl*smh*H))),color=0) for x in range(len(closestImage))]
    plateOver=Image.new('RGBA',(int(round(scl*smw*W)),int(round(scl*smh*H))),color=0)

    if (len(htmlroot)==len(closestImage)):
        imnames=[h+'.jpeg' for h in htmlroot]
        posimnames=[h+'_POS.jpeg' for h in htmlroot]
    else:
        imnames=[htmlroot+'_%04d.jpeg'%i for i in range(len(closestImage))]
        posimnames=[htmlroot+'_%04dPOS.jpeg'%i for i in range(len(closestImage))]
    htmlname="index.html"
    nomapname="nomap.html"
    overlayname='OVERLAY.gif'
        
    draw = ImageDraw.Draw(plateOver)

    # Row and column counters for preview images
    xblank=0
    yblank=0

    # Start creating html files for building the image maps
    SGAString=makeHEAD(imnames+posimnames)
    SGAString+="""
    <body onload='setPicture();'>
    <img src="" title="Plate image array" id="imagetochange" width="%s"/>
    <img src="%s" usemap="#ImageMap" width="%s"/>
    """%(str(smw*W),overlayname,str(smw*W))

    for colour in highlight:
        highlight[colour]=[x.upper() for x in highlight[colour]]

    # Font definition is annoyingly platform-dependent
    # http://www.razorvine.net/blog/user/irmen/article/2008-08-02/127
    for path in ["arial.ttf", "/System/Library/Fonts/Helvetica.dfont",
        "/Library/Fonts/Arial.ttf", "c:/windows/fonts/arial.ttf",
	"/usr/share/fonts/truetype/ttf-dejavu/DejaVuSans.ttf" ]:
        try:
            font = ImageFont.truetype(path, int(round(25*scl)))
        except:
            pass

    mapString=''''''
    plateString=''''''
    for col,colDat in enumerate(imList):
        for row,vID in enumerate(vIDlist):
            dat=colDat[colDat["vID"]==vID]
            if dat.shape[0]>0:
                barc=dat["Barcode"].iloc[0]
                print("Drawing: "+barc)
                for climind,clim in enumerate(closestImage):
                    impath=clim[barc]
                    imroot=os.path.basename(impath).split(".")[0]
                    pospath=os.path.join(os.path.dirname(impath),"Output_Images",imroot+".png")
                    im=Image.open(impath)
                    imw,imh=Image.open(impath).size
                    smh=int(round(float(imh)*float(smw)/float(imw)))
                    # Scale factors for coordinates of preview images
                    scalex=(float(smw)/float(imw))
                    scaley=(float(smh)/float(imh))
                    im=im.resize((int(round(scl*smw)),int(round(scl*smh))),Image.ANTIALIAS)
                    if(os.path.exists(pospath)):
                        posim=Image.open(pospath).resize((int(round(scl*smw)),int(round(scl*smh))),Image.ANTIALIAS)
                    else:
                        print("Preview file missing.  Analysis not complete?")
                        print(pospath)
                        posim=im
                    plateArr[climind].paste(im,(int(round(col*smw*scl)),int(round(row*smh*scl))))
                    platePosArr[climind].paste(posim,(int(round(col*smw*scl)),int(round(row*smh*scl))))
                #dat=res[(res["Barcode"]==barc)&(res["Timeseries.order"]==1)]
		dat=res[(res["Barcode"]==barc)]
                dat["tlx"]=col*smw+scalex*dat["XOffset"]
                dat["tly"]=row*smh+scaley*dat["YOffset"]
                dat["brx"]=dat["tlx"]+scalex*dat["TileX"]
                dat["bry"]=dat["tly"]+scaley*dat["TileY"]
                for ind,dRow in dat.iterrows():
                    inputs=(int(round(dRow["tlx"])),int(round(dRow["tly"])),int(round(dRow["brx"])),int(round(dRow["bry"])),dRow["Gene"],dRow["ORF"])
                    if dRow["ORF"][0:2]=="SP" and "." in dRow["ORF"]:
                        mapString+='<area shape="rect" coords="%i,%i,%i,%i" title="%s" href="http://www.pombase.org/spombe/result/%s" alt=""/>\n'%inputs
                    else:
                        mapString+='<area shape="rect" coords="%i,%i,%i,%i" title="%s" href="http://www.yeastgenome.org/cgi-bin/locus.fpl?locus=%s" alt=""/>\n'%inputs
                    if dRow["fit"]>hitThresh:
                            for d in range(0,1):
                                draw.rectangle((int(round(dRow["tlx"]*scl))-d,int(round(dRow["tly"]*scl))-d,int(round(dRow["brx"]*scl))+d,int(round(dRow["bry"]*scl))+d),outline="white")
                    if dRow["fit"]<deadThresh:
                            cx=int(round((dRow["tlx"]+dRow["brx"])*scl/2.0))
                            cy=int(round((dRow["tly"]+dRow["bry"])*scl/2.0))
                            r=1
                            draw.rectangle((cx-r,cy-r,cx+r,cy+r),fill="white")
                    for colour in highlight:
                        if str(dRow["Gene"]).upper() in highlight[colour]:
                            for d in range(0,3):
                                draw.rectangle((int(round(dRow["tlx"]*scl))-d,int(round(dRow["tly"]*scl))-d,int(round(dRow["brx"]*scl))+d,int(round(dRow["bry"]*scl))+d),outline=colour)
                                
                inputs=(col*smw,row*smh,(col+1)*smw,(row+1)*smh,str(horiz[col])+"#"+str(dat["vID"].iloc[0])+str(dat["Barcode"].iloc[0]))
                plateString+='<area shape="rect" coords="%i,%i,%i,%i" title="%s"/>\n'%inputs
                # Text indicating plate numbers
                draw.text((int(round(col*smw*scl)), int(round(smh*scl*(row+1)-25*scl))), "P"+pad(dat["MasterPlate.Number"].iloc[0]),fill="yellow",font=font)

    KeyString='''
    <h2><a onclick='updatePicture(); setPicture();' onmouseover="" style="cursor: pointer; text-align:left;float:left;" title="Toggle between background images">%s</a></h2>
    <p id="texttochange" style="ext-align:right;float:right;"></p>
    <h3 style="clear:both;"><a href="http://research.ncl.ac.uk/qfa/">QFA</a> image browser tool, generated by <a href="http://research.ncl.ac.uk/colonyzer/">Colonyzer</a></h3>
    <p><i>Plate hover key:</i> %s</p>
    '''%(str(title),horizontal+"#"+"#".join(diffIDs)+"#Barcode")
    for col,h in enumerate(horiz):
        KeyString+='''<div style="position:absolute; top: 210px; left: %ipx; width: %ipx;">%s</div>
'''%(smw*col,smw,str(h))

    for i in range(len(closestImage)):
        plateArr[i].save(os.path.join(outPath,imnames[i]),quality=100)
        platePosArr[i].save(os.path.join(outPath,posimnames[i]),quality=100)
    plateOver.save(os.path.join(outPath,overlayname),transparency=0)
    fout=open(os.path.join(outPath,htmlname),'w')
    fout.write(SGAString+KeyString+'<map name="ImageMap">'+mapString+plateString+"</map></body></html>")
    fout.close()
    makeCSS(fname=os.path.join(outPath,"imbrowse.css"))

    fout=open(os.path.join(outPath,nomapname),'w')
    fout.write(SGAString+KeyString+'<map name="ImageMap">'+plateString+"</map></body></html>")
    fout.close()

def parseColonyzer(fname,fmt="%Y-%m-%d_%H-%M-%S"):
    '''Read in Colonyzer .out file and parse some info from filename.'''
    lenf=len(time.strftime(fmt))
    froot=os.path.basename(fname).split(".")[0]
    barc=froot[0:(-1*(lenf+1))]
    datetime=froot[(-1*lenf):]
    data=pd.read_csv(fname,sep="\t",header=0)
    data["Barcode"]=barc
    data["DateTime"]=datetime
    data["FileName"]=froot
    return(data)

def getORF(libs,Library,Plate,Row,Column):
    '''Get ORF at a given row, column and plate in a particular QFA library, found in dataframe libs describing all available libraries.'''
    filt=libs.ORF[(libs.Library==Library)&(libs.Plate==Plate)&(libs.Row==Row)&(libs.Column==Column)]
    return(filt.get_values()[0])

def parseAndCombine(imOutDir=".",metaDir=".",exptDesc="ExptDescription.txt",libDesc="LibraryDescription.txt",geneToORF="ORF2GENE.txt",fout="ColonyzerOutput.txt",fmt="%Y-%m-%d_%H-%M-%S"):
    '''Read in a list of colonyzer output files, together with optional metadata files describing inoculation times, strains, treatments applied etc. to produce one summary output file.'''
    # Read in and combine all .out files
    imListROOT=os.listdir(imOutDir)
    # Check any OutputData directory if it exists
    imListOD=[os.path.join("Output_Data",x) for x in os.listdir(os.path.join(imOutDir,"Output_Data"))]
    imList=imListROOT+imListOD
    outs=[parseColonyzer(os.path.join(imOutDir,out),fmt) for out in imList if ".out" in out]
    ims=pd.concat(outs)
    
    try:
        # Read in experimental metadata
        expt=pd.read_csv(os.path.join(metaDir,exptDesc),sep="\t",header=0)
    except:
        print(exptDesc+" not found, carrying on...")
        expt=None
        
    try:
        # Read in library description file
        libs=pd.read_csv(os.path.join(metaDir,libDesc),sep="\t",header=0)
        libs.columns=[x.rstrip() for x in libs.columns]
    except:
        print(libDesc+" not found, carrying on...")
        libs=None

    try:
        # Read in file describing link between standard gene name and systematic gene name (ORF) as python dictionary orf2g
        g2orf_df=pd.read_csv(os.path.join(metaDir,geneToORF),sep="\t",header=None)
        orf2g=dict(zip(g2orf_df[0],g2orf_df[1]))
    except:
        print(geneToORF+" not found, carrying on...")
        orf2g=None

    if expt is not None:
        # Add metadata from expt to ims
        ims["Start.Time"]=[expt["Start.Time"][expt.Barcode==barc].get_values()[0] for barc in ims["Barcode"]]
        ims["Treatment"]=[expt["Treatment"][expt.Barcode==barc].get_values()[0] for barc in ims["Barcode"]]
        ims["Medium"]=[expt["Medium"][expt.Barcode==barc].get_values()[0] for barc in ims["Barcode"]]
        ims["Screen"]=[expt["Screen"][expt.Barcode==barc].get_values()[0] for barc in ims["Barcode"]]
        ims["Library"]=[expt["Library"][expt.Barcode==barc].get_values()[0] for barc in ims["Barcode"]]
        ims["Plate"]=[expt["Plate"][expt.Barcode==barc].get_values()[0] for barc in ims["Barcode"]]
        ims["RepQuad"]=[expt["RepQuad"][expt.Barcode==barc].get_values()[0] for barc in ims["Barcode"]]

        # Calculate time since inoculation date time (from expt metadata) that image was taken
        ims["ExptTime"]=(pd.to_datetime(ims["DateTime"],format=fmt)-pd.to_datetime(ims["Start.Time"],format=fmt))/np.timedelta64(1,"D")

        if libs is not None:
            # Get ORFs at each position from relevant library description
            ims["ORF"]=[getORF(libs,l,p,r,c) for l,p,r,c in zip(ims.Library,ims.Plate,ims.Row,ims.Column)]

        if orf2g is not None and libs is not None:
            # Get standard gene names for each ORF
            ims["Gene"]=[orf2g[orf] for orf in ims.ORF]

    # Write data, metadata and newly calulated times to file
    ims.to_csv(fout,sep="\t",index=False)
    return(ims)
