
#	      ___    __                          __         
#	     /   |  / /___  _________  _________/ /__  _____
#	    / /| | / / __ `/ ___/ __ \/ ___/ __  / _ \/ ___/
#	   / ___ |/ / /_/ / /__/ /_/ / /  / /_/ /  __/ /    
#	  /_/  |_/_/\__,_/\___/\____/_/   \__,_/\___/_/     
#
#
#		ALACORDER beta 7.4.8
#		by Sam Robson
#
#

import os
import sys
import glob
import re
import math
import numexpr
import bottleneck
import numpy as np
import pandas as pd
import xlrd
import openpyxl
import datetime
import time
import warnings
import PyPDF2 as pypdf
from io import StringIO

# CONFIG

pd.options.display.float_format = '${:,.2f}'.format

def config(in_path: str, out_path: str, flags="", print_log=True, warn=False, save_archive=False, set_batch=0): 

	# Get extensions
	out_ext: str = out_path.split(".")[-1].strip()
	in_ext: str = in_path.split(".")[-1].strip() if len(in_path.split(".")[-1])<5 else "directory" 

	if out_path == None or out_path == "":
		out_ext = "no_export"

	# Check if input path is valid
	if in_ext != "directory" and in_ext != "pkl" and in_ext != "csv" and in_ext != "xls" and in_ext != "json" and in_ext != "xz" and in_ext != "pdf":
		raise Exception("Input path must be to/pdf/directory/, (archive).csv, (archive).xls, (archive).json, (archive).pkl, or (archive).pkl.xz")
	if os.path.exists(in_path) == False:
		raise Exception("Input path does not exist!")

	# Check if output path is valid
	if out_ext != "pkl" and out_ext != "txt" and out_ext != "csv" and out_ext != "xls" and out_ext != "json" and out_ext != "dta" and out_ext != "xz" and out_ext != "no_export":
		raise Exception("Output path must be .csv, .xls, .json, or .pkl.xz! (.pkl.xz only for full text archives)")

	if out_ext == "txt":
		print("WARNING: Text files cannot be reimported to alacorder!")

	fromArchive = False

	# Set read, write modes, contents
	if in_ext == "directory" and bool(out_ext == "pkl" or out_ext == "xz" or out_ext == "txt"): 
		make = "archive"
		origin = "directory"
		contents = glob.glob(in_path + '**/*.pdf', recursive=True)
	if in_ext == "pdf":
		in_ext == "directory"
		contents = [in_path]
	elif in_ext == "directory" and bool(out_ext == "xls" or out_ext == "json" or out_ext == "csv" or out_ext == "txt" or out_ext == "dta"):
		make = "table"
		origin = "directory"
		contents = glob.glob(in_path + '**/*.pdf', recursive=True)
	elif in_ext == "pkl":
		make = "table"
		origin = "archive"
		contents = pd.read_pickle(in_path)['AllPagesText'].tolist()
		fromArchive = True
	elif in_ext == "xz":
		make = "table"
		origin = "archive"
		contents = pd.read_pickle(in_path,compression="xz")['AllPagesText'].tolist()
		fromArchive = True
	elif in_ext == "csv":
		make = "table"
		origin = "archive"
		contents = pd.read_csv(in_path).tolist()
	elif in_ext == "json": 
		make = "table"
		origin = "archive"
		contents = pd.read_json(in_path)

	if len(contents)==0:
		raise Exception("No cases found in input path! (" + in_path + ")")

	if origin == "archive" and set_batch == 0: # set batch
		batchsize = 1000
	if origin == "directory" and set_batch == 0:
		batchsize = 500
	if set_batch > 0:
		batchsize = set_batch
		

	case_max = len(contents)
	tot_batches = math.ceil(case_max / batchsize)

	batches = np.array_split(contents, tot_batches)

	if print_log == True:
		print(f"\nInitial configuration succeeded!\n\n{in_path} ---->\n{out_path}\n\n{case_max} cases in {tot_batches} batches")

	conf = pd.Series({
		'in_path': in_path,
		'out_path': out_path,
		'in_ext': in_ext,
		'out_ext': out_ext,
		'archive': fromArchive,
		'origin': origin,
		'make': make,
		'contents': contents,
		'batches': batches,
		'case_max': case_max,
		'tot_batches': tot_batches,
		'batchsize': batchsize,
		'print_log': print_log,
		'warnings': warn,
		'flags': flags,
		'save_archive': save_archive
	})
	
	return conf

# BATCH METHODS

def writeArchive(conf):
	path_in = conf['in_path']
	path_out = conf['out_path']
	case_max = conf['case_max']
	tot_batches = conf['tot_batches']
	batchsize = conf['batchsize']
	batches = conf['batches']
	contents = conf['contents']
	in_ext = conf['in_ext']
	out_ext = conf['out_ext']
	print_log = conf['print_log']
	warn = conf['warnings']

	if warn == False:
		warnings.filterwarnings("ignore")

	start_time = time.time()
	outputs = pd.DataFrame()
	on_batch = 0

	exptime = time.time()
	paths = pd.Series(contents)
	allpagestext = contents.map(lambda x: getPDFText(x))
	timestamp = time.time()

	c = pd.DataFrame({
		'Path': paths,
		'AllPagesText': allpagestext,
		'Timestamp': timestamp
		})

	outputs = pd.concat([outputs, c],ignore_index=True)
	on_batch += 1
	outputs.fillna('',inplace=True)

	if out_ext == "pkl":
		outputs.to_pickle(path_out+".xz",compression="xz")
	if out_ext == "xz":
		outputs.to_pickle(path_out,compression="xz")
	elif out_ext == "json":
		outputs.to_json(path_out)
	elif out_ext == "csv":
		outputs.to_csv(path_out,escapechar='\\')
	elif out_ext == "md":
		outputs.to_markdown(path_out)
	elif out_ext == "txt":
		outputs.to_string(path_out)
	elif out_ext == "dta":
		outputs.to_stata(path_out)
	elif out_ext == "no_export" or print_log == True:
		print(outputs.to_string())
	log_complete(conf, start_time)
	on_batch = 0

def writeTables(conf):
	batches = conf['batches']
	path_in = conf['in_path']
	path_out = conf['out_path']
	case_max = conf['case_max']
	tot_batches = conf['tot_batches']
	batchsize = conf['batchsize']
	in_ext = conf['in_ext']
	out_ext = conf['out_ext']
	print_log = conf['print_log']
	warn = conf['warnings']
	contents = conf['contents']
	batches = conf['batches']
	save_archive = conf['save_archive']
	from_archive = conf['archive']
	if warn == False:
		warnings.filterwarnings("ignore")
	start_time = time.time()
	on_batch = 0
	outputs = pd.DataFrame()

	fees = pd.DataFrame({'CaseNumber': '', 'Code': '', 'Payor': '', 'AmtDue': '', 'AmtPaid': '', 'Balance': '', 'AmtHold': ''},index=[0])
	charges = pd.DataFrame({'CaseNumber': '', 'Num': '', 'Code': '', 'Felony': '', 'Conviction': '', 'CERV': '', 'Pardon': '', 'Permanent': '', 'Disposition': '', 'CourtActionDate': '', 'CourtAction': '', 'Cite': '', 'TypeDescription': '', 'Category': '', 'Description': ''},index=[0]) 
	arch = pd.DataFrame({'Path':'','AllPagesText':'','Timestamp':''},index=[0])

	b = pd.DataFrame()
	exptime = time.time()

	if from_archive == True:
		b['AllPagesText'] = contents
	else:
		b['AllPagesText'] = pd.Series(contents).map(lambda x: getPDFText(x))

	b['CaseInfoOutputs'] = b['AllPagesText'].map(lambda x: getCaseInfo(x))
	b['CaseNumber'] = b['CaseInfoOutputs'].map(lambda x: x[0])
	b['Name'] = b['CaseInfoOutputs'].map(lambda x: x[1])
	b['Alias'] = b['CaseInfoOutputs'].map(lambda x: x[2])
	b['DOB'] = b['CaseInfoOutputs'].map(lambda x: x[3])
	b['Race'] = b['CaseInfoOutputs'].map(lambda x: x[4])
	b['Sex'] = b['CaseInfoOutputs'].map(lambda x: x[5])
	b['Address'] = b['CaseInfoOutputs'].map(lambda x: x[6])
	b['Phone'] = b['CaseInfoOutputs'].map(lambda x: x[7])
	b['ChargesOutputs'] = b.index.map(lambda x: getCharges(b.loc[x].AllPagesText, b.loc[x].CaseNumber))
	b['Convictions'] = b['ChargesOutputs'].map(lambda x: x[0])
	b['DispositionCharges'] = b['ChargesOutputs'].map(lambda x: x[1])
	b['FilingCharges'] = b['ChargesOutputs'].map(lambda x: x[2])
	b['CERVConvictions'] = b['ChargesOutputs'].map(lambda x: x[3])
	b['PardonConvictions'] = b['ChargesOutputs'].map(lambda x: x[4])
	b['PermanentConvictions'] = b['ChargesOutputs'].map(lambda x: x[5])
	b['ConvictionCount'] = b['ChargesOutputs'].map(lambda x: x[6])
	b['ChargeCount'] = b['ChargesOutputs'].map(lambda x: x[7])
	b['CERVChargeCount'] = b['ChargesOutputs'].map(lambda x: x[8])
	b['PardonChargeCount'] = b['ChargesOutputs'].map(lambda x: x[9])
	b['PermanentChargeCount'] = b['ChargesOutputs'].map(lambda x: x[10])
	b['CERVConvictionCount'] = b['ChargesOutputs'].map(lambda x: x[11])
	b['PardonConvictionCount'] = b['ChargesOutputs'].map(lambda x: x[12])
	b['PermanentConvictionCount'] = b['ChargesOutputs'].map(lambda x: x[13])
	b['ChargeCodes'] = b['ChargesOutputs'].map(lambda x: x[14])
	b['ConvictionCodes'] = b['ChargesOutputs'].map(lambda x: x[15])
	b['FeeOutputs'] = b.index.map(lambda x: getFeeSheet(b.loc[x].AllPagesText, b.loc[x].CaseNumber))
	b['TotalAmtDue'] = b['FeeOutputs'].map(lambda x: x[0])
	b['TotalBalance'] = b['FeeOutputs'].map(lambda x: x[1])
	b['TotalD999'] = b['FeeOutputs'].map(lambda x: x[2])
	b['FeeCodesOwed'] = b['FeeOutputs'].map(lambda x: x[3])
	b['FeeCodes'] = b['FeeOutputs'].map(lambda x: x[4])
	b['FeeSheet'] = b['FeeOutputs'].map(lambda x: x[5])

	fees['AmtDue'] = fees['AmtDue'].map(lambda x: pd.to_numeric(x,'ignore'))
	fees['AmtPaid'] = fees['AmtPaid'].map(lambda x: pd.to_numeric(x,'ignore'))
	fees['Balance'] = fees['Balance'].map(lambda x: pd.to_numeric(x,'ignore'))
	fees['AmtHold'] = fees['AmtHold'].map(lambda x: pd.to_numeric(x,'ignore'))
	charges['Num'] = charges['Num'].map(lambda x: pd.to_numeric(x,'ignore'))

	feesheet = b['FeeOutputs'].map(lambda x: x[6]) 

	feesheet= feesheet.dropna() 
	fees=fees.dropna()
	feesheet = feesheet.tolist() # -> [df, df, df]
	try:
		feesheet = pd.concat(feesheet,axis=0,ignore_index=True) #  -> batch df
	except ValueError:
		pass
	try:
		fees = fees.append(feesheet, ignore_index=True) # -> all fees df
	except ValueError:
		pass

	if print_log == True:
		print(fees)

	chargetabs = b['ChargesOutputs'].map(lambda x: x[17])
	chargetabs = chargetabs.dropna()
	charges = charges.dropna()
	chargetabs = chargetabs.tolist()
	try:
		chargetabs = pd.concat(chargetabs,axis=0,ignore_index=True)
	except ValueError:
		pass
	try:
		charges = charges.append(chargetabs,ignore_index=True)
	except ValueError:
		pass

	if print_log == True:
		print(charges)

	on_batch += 1

	b['ChargesTable'] = b['ChargesOutputs'].map(lambda x: x[-1])
	b['TotalD999'] = b['TotalD999'].map(lambda x: pd.to_numeric(x,'ignore'))
	b['Phone'] =  b['Phone'].map(lambda x: pd.to_numeric(x,'ignore'))
	b['TotalAmtDue'] = b['TotalAmtDue'].map(lambda x: pd.to_numeric(x,'ignore'))
	b['TotalBalance'] = b['TotalBalance'].map(lambda x: pd.to_numeric(x,'ignore'))


	if save_archive == True:
		timestamp = exptime
		ar = pd.DataFrame({
			'Path': pd.Series(contents),
			'AllPagesText': b['AllPagesText'],
			'Timestamp': timestamp
			},index=range(0,pd.Series(contents).shape[0]))
		arch = pd.concat([arch, ar],ignore_index=True)
		arch.fillna('',inplace=True)
		arch.dropna(inplace=True)
		arch.to_pickle(path_out+".pkl.xz",compression="xz")


	b.drop(columns=['AllPagesText','CaseInfoOutputs','ChargesOutputs','FeeOutputs','TotalD999','ChargesTable','FeeSheet'],inplace=True)
	outputs = pd.concat([outputs, b],ignore_index=True)
	
	outputs.fillna('',inplace=True)
	charges.fillna('',inplace=True)
	fees.fillna('',inplace=True)

	# write 
	if out_ext == "xls":
		with pd.ExcelWriter(path_out) as writer:
			outputs.to_excel(writer, sheet_name="cases-table")
			fees.to_excel(writer, sheet_name="fees-table")
			charges.to_excel(writer, sheet_name="charges-table")
	elif out_ext == "pkl":
		outputs.to_pickle(path_out+".xz",compression="xz")
	elif out_ext == "xz":
		outputs.to_pickle(path_out,compression="xz")
	elif out_ext == "json":
		outputs.to_json(path_out)
	elif out_ext == "csv":
		outputs.to_csv(path_out,escapechar='\\')
	elif out_ext == "md":
		outputs.to_markdown(path_out)
	elif out_ext == "txt":
		outputs.to_string(path_out)
	elif out_ext == "dta":
		outputs.to_stata(path_out)
	elif out_ext == "no_export" or print_log == True:
		print(outputs.to_string())
	else:
		raise Exception("Output file extension not supported! Please output to .xls, .pkl, .json, or .csv")
		
	log_complete(conf, start_time)
	on_batch = 0

def writeFees(conf):
	batches = conf['batches']
	path_in = conf['in_path']
	path_out = conf['out_path']
	case_max = conf['case_max']
	tot_batches = conf['tot_batches']
	batchsize = conf['batchsize']
	in_ext = conf['in_ext']
	out_ext = conf['out_ext']
	print_log = conf['print_log']
	warn = conf['warnings']
	contents = conf['contents']
	batches = conf['batches']
	if warn == False:
		warnings.filterwarnings("ignore")
	start_time = time.time()
	on_batch = 0
	outputs = pd.DataFrame()

	fees = pd.DataFrame({'CaseNumber': '', 'Code': '', 'Payor': '', 'AmtDue': '', 'AmtPaid': '', 'Balance': '', 'AmtHold': ''},index=[0])
	for i, c in enumerate(batches):
		exptime = time.time()
		b = pd.DataFrame()
		b['AllPagesText'] = pd.Series(c).map(lambda x: getPDFText(x))
		b['CaseInfoOutputs'] = b['AllPagesText'].map(lambda x: getCaseInfo(x))
		b['CaseNumber'] = b['CaseInfoOutputs'].map(lambda x: x[0])
		b['FeeOutputs'] = b.index.map(lambda x: getFeeSheet(b.loc[x].AllPagesText, b.loc[x].CaseNumber))

		fees['AmtDue'] = fees['AmtDue'].map(lambda x: pd.to_numeric(x,'ignore'))
		fees['AmtPaid'] = fees['AmtPaid'].map(lambda x: pd.to_numeric(x,'ignore'))
		fees['Balance'] = fees['Balance'].map(lambda x: pd.to_numeric(x,'ignore'))
		fees['AmtHold'] = fees['AmtHold'].map(lambda x: pd.to_numeric(x,'ignore'))

		feesheet = b['FeeOutputs'].map(lambda x: x[6]) # -> pd.Series(df, df, df)
		# print(feesheet)

		feesheet= feesheet.dropna() # drop empty 
		fees=fees.dropna()
		feesheet = feesheet.tolist() # convert to list -> [df, df, df]
		feesheet = pd.concat(feesheet,axis=0,ignore_index=True) # add all dfs in batch -> df
		fees = fees.append(feesheet, ignore_index=True) #pd.concat([fees, feesheet],axis=0,ignore_index=True)


		fees.fillna('',inplace=True)

		# write 
		if out_ext == "xls":
			with pd.ExcelWriter(path_out) as writer:
				fees.to_excel(writer, sheet_name="fees-table")
		elif out_ext == "pkl":
			fees.to_pickle(path_out+".xz",compression="xz")
		elif out_ext == "xz":
			fees.to_pickle(path_out,compression="xz")
		elif out_ext == "json":
			fees.to_json(path_out)
		elif out_ext == "csv":
			fees.to_csv(path_out,escapechar='\\')
		elif out_ext == "md":
			fees.to_markdown(path_out)
		elif out_ext == "txt":
			fees.to_string(path_out)
		elif out_ext == "dta":
			fees.to_stata(path_out)
		elif out_ext == "no_export" or print_log == True:
			print(fees.to_string())
		else:
			raise Exception("Output file extension not supported! Please output to .xls, .json, or .csv")
		on_batch += 1
	log_complete(conf, start_time)
	on_batch = 0

def writeCharges(conf):
	batches = conf['batches']
	path_in = conf['in_path']
	path_out = conf['out_path']
	case_max = conf['case_max']
	tot_batches = conf['tot_batches']
	batchsize = conf['batchsize']
	in_ext = conf['in_ext']
	out_ext = conf['out_ext']
	print_log = conf['print_log']
	warn = conf['warnings']
	contents = conf['contents']
	batches = conf['batches']
	if warn == False:
		warnings.filterwarnings("ignore")
	start_time = time.time()
	on_batch = 0
	outputs = pd.DataFrame()
	charges = pd.DataFrame({'CaseNumber': '', 'Num': '', 'Code': '', 'Felony': '', 'Conviction': '', 'CERV': '', 'Pardon': '', 'Permanent': '', 'Disposition': '', 'CourtActionDate': '', 'CourtAction': '', 'Cite': '', 'TypeDescription': '', 'Category': '', 'Description': ''},index=[0]) 
	for i, c in enumerate(batches):
		exptime = time.time()
		b = pd.DataFrame()
		b['AllPagesText'] = pd.Series(c).map(lambda x: getPDFText(x))
		b['CaseInfoOutputs'] = b['AllPagesText'].map(lambda x: getCaseInfo(x))
		b['CaseNumber'] = b['CaseInfoOutputs'].map(lambda x: x[0])
		b['ChargesOutputs'] = b.index.map(lambda x: getCharges(b.loc[x].AllPagesText, b.loc[x].CaseNumber))
		charges['Num'] = charges['Num'].map(lambda x: pd.to_numeric(x,'ignore'))

		chargetabs = b['ChargesOutputs'].map(lambda x: x[17])
		chargetabs = chargetabs.dropna()
		charges = charges.dropna()
		chargetabs = chargetabs.tolist()
		chargetabs = pd.concat(chargetabs,axis=0,ignore_index=True)
		charges = charges.append(chargetabs,ignore_index=True)

		
		charges.fillna('',inplace=True)

		# write 
		if out_ext == "xls":
			with pd.ExcelWriter(path_out) as writer:
				charges.to_excel(writer, sheet_name="charges-table")
		elif out_ext == "pkl":
			charges.to_pickle(path_out+".xz",compression="xz")
		elif out_ext == "xz":
			charges.to_pickle(path_out,compression="xz")
		elif out_ext == "json":
			charges.to_json(path_out)
		elif out_ext == "csv":
			charges.to_csv(path_out,escapechar='\\')
		elif out_ext == "md":
			charges.to_markdown(path_out)
		elif out_ext == "txt":
			charges.to_string(path_out)
		elif out_ext == "dta":
			charges.to_stata(path_out)
		elif out_ext == "no_export" or print_log == True:
			print(charges.to_string())
		else:
			raise Exception("Output file extension not supported! Please output to .xls, .json, or .csv")
		on_batch += 1
	log_complete(conf, start_time)
	on_batch = 0

def search(conf, method, status=''): 
	batches = conf['batches']
	path_in = conf['in_path']
	path_out = conf['out_path']
	case_max = conf['case_max']
	tot_batches = conf['tot_batches']
	batchsize = conf['batchsize']
	in_ext = conf['in_ext']
	out_ext = conf['out_ext']
	print_log = conf['print_log']
	warn = conf['warnings']
	contents = conf['contents']
	batches = conf['batches']
	if warn == False:
		warnings.filterwarnings("ignore")
	start_time = time.time()
	on_batch = 0
	allCustomOutputs = pd.Series()
	uselist = False
	for i, c in enumerate(batches):
		exptime = time.time()
		b = pd.DataFrame()
		allpagestext = pd.Series(c).map(lambda x: getPDFText(x))
		customoutputs = allpagestext.map(lambda x: method(x))
		allCustomOutputs = allCustomOutputs.append(customoutputs)
		if print_log == True:
			print(allCustomOutputs)
		on_batch += 1
	log_complete(conf, start_time)
	return allCustomOutputs

def write(conf, method, status=''): 
	batches = conf['batches']
	path_in = conf['in_path']
	path_out = conf['out_path']
	case_max = conf['case_max']
	tot_batches = conf['tot_batches']
	batchsize = conf['batchsize']
	in_ext = conf['in_ext']
	out_ext = conf['out_ext']
	print_log = conf['print_log']
	warn = conf['warnings']
	contents = conf['contents']
	batches = conf['batches']
	if warn == False:
		warnings.filterwarnings("ignore")
	start_time = time.time()
	on_batch = 0
	alloutputs = pd.Series()
	uselist = False
	for i, c in enumerate(batches):
		exptime = time.time()
		b = pd.DataFrame()
		allpagestext = pd.Series(c).map(lambda x: getPDFText(x))
		customoutputs = allpagestext.map(lambda x: method(x))
		alloutputs = alloutputs.append(customoutputs)
		if print_log == True and out_ext != "no_export":
			print(alloutputs)
		on_batch += 1

		if out_ext == "xls":
			with pd.ExcelWriter(path_out) as writer:
				alloutputs.to_excel(writer, sheet_name="output-table")
		elif out_ext == "pkl":
			alloutputs.to_pickle(path_out+".xz",compression="xz")
		elif out_ext == "xz":
			alloutputs.to_pickle(path_out,compression="xz")
		elif out_ext == "json":
			alloutputs.to_json(path_out)
		elif out_ext == "csv":
			alloutputs.to_csv(path_out,escapechar='\\')
		elif out_ext == "md":
			alloutputs.to_markdown(path_out)
		elif out_ext == "txt":
			alloutputs.to_string(path_out)
		elif out_ext == "dta":
			alloutputs.to_stata(path_out)
		elif out_ext == "no_export" or print_log == True:
			print(alloutputs.to_string())
		else:
			raise Warning("Batch export failed!")
		on_batch += 1
	log_complete(conf, start_time)
	on_batch = 0

# CASE METHODS 

def getPDFText(path: str) -> str:
	text = ""
	pdf = pypdf.PdfReader(path)
	for pg in pdf.pages:
		text += pg.extract_text()
	return text

def getCaseNumber(text: str):
	try:
		county: str = re.search(r'(?:County\: )(\d{2})(?:Case)', str(text)).group(1).strip()
		case_num: str = county + "-" + re.search(r'(\w{2}\-\d{4}-\d{6}.\d{2})', str(text)).group(1).strip() 
		return county + "-" + case_num
	except (IndexError, AttributeError):
		return ""

def getName(text: str):
	name = ""
	if bool(re.search(r'(?a)(VS\.|V\.{1})(.+)(Case)*', text, re.MULTILINE)) == True:
		name = re.search(r'(?a)(VS\.|V\.{1})(.+)(Case)*', text, re.MULTILINE).group(2).replace("Case Number:","").strip()
	else:
		if bool(re.search(r'(?:DOB)(.+)(?:Name)', text, re.MULTILINE)) == True:
			name = re.search(r'(?:DOB)(.+)(?:Name)', text, re.MULTILINE).group(1).replace(":","").replace("Case Number:","").strip()
	return name

def getDOB(text: str):
	dob = ""
	if bool(re.search(r'(\d{2}/\d{2}/\d{4})(?:.{0,5}DOB\:)', str(text), re.DOTALL)):
		dob: str = re.search(r'(\d{2}/\d{2}/\d{4})(?:.{0,5}DOB\:)', str(text), re.DOTALL).group(1)
	return dob

def getFeeTotals(text: str):
	try:
		trowraw = re.findall(r'(Total.*\$.*)', str(text), re.MULTILINE)[0]
		totalrow = re.sub(r'[^0-9|\.|\s|\$]', "", trowraw)
		if len(totalrow.split("$")[-1])>5:
			totalrow = totalrow.split(" . ")[0]
		tbal = totalrow.split("$")[3].strip().replace("$","").replace(",","").replace(" ","")
		tdue = totalrow.split("$")[1].strip().replace("$","").replace(",","").replace(" ","")
		tpaid = totalrow.split("$")[2].strip().replace("$","").replace(",","").replace(" ","")
		thold = totalrow.split("$")[4].strip().replace("$","").replace(",","").replace(" ","")
	except IndexError:
		totalrow = ""
		tbal = ""
		tdue = ""
		tpaid = ""
		thold = ""
	return [totalrow,tdue,tpaid,tdue,thold]

def getCaseInfo(text: str):
	case_num = ""
	name = ""
	alias = ""
	race = ""
	sex = ""
	address = ""
	dob = ""
	phone = ""

	try:
		county: str = re.search(r'(?:County\: )(\d{2})(?:Case)', str(text)).group(1).strip()
		case_num: str = county + "-" + re.search(r'(\w{2}\-\d{4}-\d{6}.\d{2})', str(text)).group(1).strip() 
	except (IndexError, AttributeError):
		pass
 
	if bool(re.search(r'(?a)(VS\.|V\.{1})(.{5,100})(Case)*', text, re.MULTILINE)) == True:
		name = re.search(r'(?a)(VS\.|V\.{1})(.{5,100})(Case)*', text, re.MULTILINE).group(2).replace("Case Number:","").strip()
	else:
		if bool(re.search(r'(?:DOB)(.{5,100})(?:Name)', text, re.MULTILINE)) == True:
			name = re.search(r'(?:DOB)(.{5,100})(?:Name)', text, re.MULTILINE).group(1).replace(":","").replace("Case Number:","").strip()
	try:
		alias = re.search(r'(SSN)(.{5,75})(Alias)', text, re.MULTILINE).group(2).replace(":","").replace("Alias 1","").strip()
	except (IndexError, AttributeError):
		pass
	else:
		pass
	try:
		dob: str = re.search(r'(\d{2}/\d{2}/\d{4})(?:.{0,5}DOB\:)', str(text), re.DOTALL).group(1)
		phone: str = re.search(r'(?:Phone\:)(.*?)(?:Country)', str(text), re.DOTALL).group(1).strip()
		if len(phone) < 7:
			phone = ""
		if len(phone) > 10 and phone[-3:] == "000":
			phone = phone[0:9]
	except (IndexError, AttributeError):
		dob = ""
		phone = ""
	try:
		racesex = re.search(r'(B|W|H|A)\/(F|M)(?:Alias|XXX)', str(text))
		race = racesex.group(1).strip()
		sex = racesex.group(2).strip()
	except (IndexError, AttributeError):
		pass
	try:
		street_addr = re.search(r'(Address 1\:)(.+)', str(text), re.MULTILINE).group(2).strip()
	except (IndexError, AttributeError):
		street_addr = ""
	try:
		zip_code = re.search(r'(Zip\: )(.+)', str(text), re.MULTILINE).group(2).strip()	
	except (IndexError, AttributeError):
		zip_code = ""
	try:
		city = re.search(r'(City\: )(.*)(State\: )(.*)', str(text), re.MULTILINE).group(2).strip()
	except (IndexError, AttributeError):
		city = ""
	try:
		state = re.search(r'(?:City\: ).*(?:State\: ).*', str(text), re.MULTILINE).group(4).strip()
	except (IndexError, AttributeError):
		state = ""
	
	address = street_addr + " " + city + ", " + state + " " + zip_code
	case = [case_num, name, alias, dob, race, sex, address, phone]
	return case

def getFeeSheet(text: str, cnum: str):
	actives = re.findall(r'(ACTIVE.*\$.*)', str(text))
	if len(actives) == 0:
		return [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan]
	else:
		rind = range(0, len(actives)+1)
		try:
			trowraw = re.findall(r'(Total.*\$.*)', str(text), re.MULTILINE)[0]
			totalrow = re.sub(r'[^0-9|\.|\s|\$]', "", trowraw)
			if len(totalrow.split("$")[-1])>5:
				totalrow = totalrow.split(" . ")[0]
			tbal = totalrow.split("$")[3].strip().replace("$","").replace(",","").replace(" ","")
			tdue = totalrow.split("$")[1].strip().replace("$","").replace(",","").replace(" ","")
			tpaid = totalrow.split("$")[2].strip().replace("$","").replace(",","").replace(" ","")
			thold = totalrow.split("$")[4].strip().replace("$","").replace(",","").replace(" ","")
		except IndexError:
			totalrow = ""
			tbal = ""
			tdue = ""
			tpaid = ""
			thold = ""
		fees = pd.Series(actives,dtype=str)
		fees_noalpha = fees.map(lambda x: re.sub(r'[^0-9|\.|\s|\$]', "", x))
		srows = fees.map(lambda x: x.strip().split(" "))
		drows = fees_noalpha.map(lambda x: x.replace(",","").split("$"))
		coderows = srows.map(lambda x: str(x[5]).strip() if len(x)>5 else "")
		payorrows = srows.map(lambda x: str(x[6]).strip() if len(x)>6 else "")
		amtduerows = drows.map(lambda x: str(x[1]).strip() if len(x)>1 else "")
		amtpaidrows = drows.map(lambda x: str(x[2]).strip() if len(x)>2 else "")
		balancerows = drows.map(lambda x: str(x[-1]).strip() if len(x)>5 else "")
		amtholdrows = drows.map(lambda x: str(x[3]).strip() if len(x)>5 else "")
		amtholdrows = amtholdrows.map(lambda x: x.split(" ")[0].strip() if " " in x else x)
		istotalrow = fees.map(lambda x: False if bool(re.search(r'(ACTIVE)',x)) else True)

		feesheet = pd.DataFrame({
			'CaseNumber': cnum,
			'Total': '',
			'Code': coderows.tolist(),
			'Payor': payorrows.tolist(),
			'AmtDue': amtduerows.tolist(),
			'AmtPaid': amtpaidrows.tolist(),
			'Balance': balancerows.tolist(),
			'AmtHold': amtholdrows.tolist()
			})

		totalrdf = {
			'CaseNumber': cnum,
			'Total': 'TOTAL',
			'Code': '',
			'Payor': '',
			'AmtDue': tdue,
			'AmtPaid': tpaid,
			'Balance': tbal,
			'AmtHold': thold
		}


		feesheet = feesheet.append(totalrdf, ignore_index=True)
		feesheet['Code'] = feesheet['Code'].astype("category")
		feesheet['Payor'] = feesheet['Payor'].astype("category")

		try:
			d999 = feesheet[feesheet['Code']=='D999']['Balance']
		except (TypeError, IndexError):
			d999 = ""

		owe_codes = " ".join(feesheet['Code'][feesheet.Balance.str.len() > 0])
		codes = " ".join(feesheet['Code'])
		allrows = actives
		allrows.append(totalrow)
		allrowstr = "\n".join(allrows)
		return [tdue, tbal, d999, owe_codes, codes, allrowstr, feesheet]

def getCharges(text: str, cnum: str):
	# get all charges matches
	c = re.findall(r'(\d{3}\s{1}.{1,100}?.{3}-.{3}-.{3}.{10,75})', text, re.MULTILINE)
	# print(c)
	cind = range(0, len(c))
	charges = pd.DataFrame({ 'Charges': c,'parentheses':'','decimals':''},index=cind)
	charges['CaseNumber'] = charges.index.map(lambda x: cnum)
	# find table fields
	split_charges = charges['Charges'].map(lambda x: x.split(" "))
	charges['Num'] = split_charges.map(lambda x: x[0].strip())
	charges['Code'] = split_charges.map(lambda x: x[1].strip()[0:4])
	charges['Felony'] = charges['Charges'].map(lambda x: bool(re.search(r'FELONY',x)))
	charges['Conviction'] = charges['Charges'].map(lambda x: bool(re.search(r'GUILTY|CONVICTED',x)))
	charges['VRRexception'] = charges['Charges'].map(lambda x: bool(re.search(r'(A ATT|ATTEMPT|S SOLICIT|CONSP)',x)))
	charges['CERVCode'] = charges['Code'].map(lambda x: bool(re.search(r'(OSUA|EGUA|MAN1|MAN2|MANS|ASS1|ASS2|KID1|KID2|HUT1|HUT2|BUR1|BUR2|TOP1|TOP2|TPCS|TPCD|TPC1|TET2|TOD2|ROB1|ROB2|ROB3|FOR1|FOR2|FR2D|MIOB|TRAK|TRAG|VDRU|VDRY|TRAO|TRFT|TRMA|TROP|CHAB|WABC|ACHA|ACAL)', x)))
	charges['PardonCode'] = charges['Code'].map(lambda x: bool(re.search(r'(RAP1|RAP2|SOD1|SOD2|STSA|SXA1|SXA2|ECHI|SX12|CSSC|FTCS|MURD|MRDI|MURR|FMUR|PMIO|POBM|MIPR|POMA|INCE)', x)))
	charges['PermanentCode'] = charges['Code'].map(lambda x: bool(re.search(r'(CM\d\d|CMUR)', x)))
	charges['CERV'] = charges.index.map(lambda x: charges['CERVCode'][x] == True and charges['VRRexception'][x] == False and charges['Felony'][x] == True)
	charges['Pardon'] = charges.index.map(lambda x: charges['PardonCode'][x] == True and charges['VRRexception'][x] == False and charges['Felony'][x] == True)
	charges['Permanent'] = charges.index.map(lambda x: charges['PermanentCode'][x] == True and charges['VRRexception'][x] == False and charges['Felony'][x] == True)
	charges['Disposition'] = charges['Charges'].map(lambda x: bool(re.search(r'\d{2}/\d{2}/\d{4}', x)))
	charges['CourtActionDate'] = charges['Charges'].map(lambda x: re.search(r'(\d{2}/\d{2}/\d{4})', x).group() if bool(re.search(r'(\d{2}/\d{2}/\d{4})', x)) else "")
	charges['CourtAction'] = charges['Charges'].map(lambda x: re.search(r'(BOUND|GUILTY PLEA|PROBATION|WAIVED|DISMISSED|TIME LAPSED|NOL PROSS|CONVICTED|INDICTED|OTHER|DISMISSED|FORFEITURE|TRANSFER|REMANDED|PROBATION|ACQUITTED|WITHDRAWN|PETITION|PRETRIAL|COND\. FORF\.)', x).group() if bool(re.search(r'(BOUND|GUILTY PLEA|PROBATION|WAIVED|DISMISSED|TIME LAPSED|NOL PROSS|CONVICTED|INDICTED|OTHER|DISMISSED|FORFEITURE|TRANSFER|REMANDED|PROBATION|ACQUITTED|WITHDRAWN|PETITION|PRETRIAL|COND\. FORF\.)', x)) else "")
	# print(charges)
	try:
		charges['Cite'] = charges['Charges'].map(lambda x: re.search(r'([^a-z]{1,2}?.{1}-[^\s]{3}-[^\s]{3})', x).group())
	except (AttributeError, IndexError):
		pass	
		try:
			charges['Cite'] = charges['Charges'].map(lambda x: re.search(r'([0-9]{1,2}.{1}-.{3}-.{3})',x).group()) # TEST
		except (AttributeError, IndexError):
			charges['Cite'] = ""
	charges['Cite'] = charges['Cite'].astype(str)
	try:
		charges['decimals'] = charges['Charges'].map(lambda x: re.search(r'(\.[0-9])', x).group())
		charges['Cite'] = charges['Cite'] + charges['decimals']
	except (AttributeError, IndexError):
		charges['Cite'] = charges['Cite']
	try:
		charges['parentheses'] = charges['Charges'].map(lambda x: re.search(r'(\([A-Z]\))', x).group())
		charges['Cite'] = charges['Cite'] + charges['parentheses']
		charges['Cite'] = charges['Cite'].map(lambda x: x[1:-1] if bool(x[0]=="R" or x[0]=="Y" or x[0]=="C") else x)
	except (AttributeError, IndexError):
		pass

	charges['TypeDescription'] = charges['Charges'].map(lambda x: re.search(r'(BOND|FELONY|MISDEMEANOR|OTHER|TRAFFIC|VIOLATION)', x).group() if bool(re.search(r'(BOND|FELONY|MISDEMEANOR|OTHER|TRAFFIC|VIOLATION)', x)) else "")
	charges['Category'] = charges['Charges'].map(lambda x: re.search(r'(ALCOHOL|BOND|CONSERVATION|DOCKET|DRUG|GOVERNMENT|HEALTH|MUNICIPAL|OTHER|PERSONAL|PROPERTY|SEX|TRAFFIC)', x).group() if bool(re.search(r'(ALCOHOL|BOND|CONSERVATION|DOCKET|DRUG|GOVERNMENT|HEALTH|MUNICIPAL|OTHER|PERSONAL|PROPERTY|SEX|TRAFFIC)', x)) else "")
	try:
		charges['Description'] = charges['Charges'].map(lambda x: re.search(r'(\d{3}\s[\w\d]{4}\s)(.{5,75}?)(.{3}-.{3}-.{3})',x, re.MULTILINE).group(2).strip() if bool(re.search(r'(\d{3}\s[\w\d]{4}\s)(.{5,75}?)(.{3}-.{3}-.{3})',x, re.MULTILINE).group(2).replace("\n"," ").strip()) else x)
	except (AttributeError, IndexError):
		charges['Description'] = charges['Charges']
	charges['Charges'] = charges['Charges'].map(lambda x: x.replace("SentencesSentence","").replace("Sentence","").strip())
	charges.drop(columns=['PardonCode','PermanentCode','CERVCode','VRRexception','parentheses','decimals'], inplace=True)
	# print(charges['Description'])
	charges['Description'] = charges['Description'].map(lambda x: x.replace("\'","").strip())
	charges['Description'] = charges.index.map(lambda x: charges['Description'][x].replace(charges['CourtActionDate'][x],"").replace(charges['CourtAction'][x],"").strip())
	charges['Category'] = charges['Category'].astype("category")
	charges['TypeDescription'] = charges['TypeDescription'].astype("category")
	charges['Code'] = charges['Code'].astype("category")
	charges['CourtAction'] = charges['CourtAction'].astype("category")

	# counts
	conviction_ct = charges[charges.Conviction == True].shape[0]
	charge_ct = charges.shape[0]
	cerv_ct = charges[charges.CERV == True].shape[0]
	pardon_ct = charges[charges.Pardon == True].shape[0]
	perm_ct = charges[charges.Permanent == True].shape[0]
	conv_cerv_ct = charges[charges.CERV == True][charges.Conviction == True].shape[0]
	conv_pardon_ct = charges[charges.Pardon == True][charges.Conviction == True].shape[0]
	conv_perm_ct = charges[charges.Permanent == True][charges.Conviction == True].shape[0]

	# summary strings
	convictions = "; ".join(charges[charges.Conviction == True]['Charges'].tolist())
	conv_codes = " ".join(charges[charges.Conviction == True]['Code'].tolist())
	charge_codes = " ".join(charges[charges.Disposition == True]['Code'].tolist())
	dcharges = "; ".join(charges[charges.Disposition == True]['Charges'].tolist())
	fcharges = "; ".join(charges[charges.Disposition == False]['Charges'].tolist())
	cerv_convictions = "; ".join(charges[charges.CERV == True][charges.Conviction == True]['Charges'].tolist())
	pardon_convictions = "; ".join(charges[charges.Pardon == True][charges.Conviction == True]['Charges'].tolist())
	perm_convictions = "; ".join(charges[charges.Permanent == True][charges.Conviction == True]['Charges'].tolist())

	allcharge = "; ".join(charges['Charges'])
	if charges.shape[0] == 0:
		charges = np.nan

	return [convictions, dcharges, fcharges, cerv_convictions, pardon_convictions, perm_convictions, conviction_ct, charge_ct, cerv_ct, pardon_ct, perm_ct, conv_cerv_ct, conv_pardon_ct, conv_perm_ct, charge_codes, conv_codes, allcharge, charges]

# LOG METHODS

def log_complete(conf, start_time):
	path_in = conf['in_path']
	path_out = conf['out_path']
	path_out = "Wrote to " + conf['out_path'] if conf['out_path'] != "" else "No export path provided - did not export!"
	case_max = conf['case_max']
	bsize = conf['batchsize']
	completion_time = time.time()
	elapsed = completion_time - start_time
	cases_per_sec = case_max/elapsed
	print(f'''
    ___    __                          __         
   /   |  / /___  _________  _________/ /__  _____
  / /| | / / __ `/ ___/ __ \\/ ___/ __  / _ \\/ ___/
 / ___ |/ / /_/ / /__/ /_/ / /  / /_/ /  __/ /    
/_/  |_/_/\\__,_/\\___/\\____/_/   \\__,_/\\___/_/     
																																										
	
	ALACORDER beta 7.4.8
	by Sam Robson	

	Searched {path_in} 
	{path_out} 

	TASK SUCCEEDED ({case_max}/{case_max} cases)
	Completed export in {elapsed:.2f} seconds ({cases_per_sec:.2f}/sec)

''') 

