# conversion json and command line parameters to functional input variables
# 	add more context to key definition
# Jaesub Hong

import json
# import pyjson5 as json5
import hjson   as hjson

import re
import sys
# import math
# import glob
# import hashlib
import textwrap3  # we modify one of its routines below: len

# import numpy  as np

from os		import path, getenv
from pathlib	import Path
# from socket		import gethostname
from collections  import OrderedDict
from datetime     import datetime
# from importlib    import import_module
# from IPython	import embed

# from rich import box
from rich.console import Console
from rich.table import Table

# there must be a better way to do this...
if 'clise' in __file__:
	STARTUP  = 'CLISE_STARTUP'
	AUTO_GEN = 'clise.rabbit.multiply_by_file'
	MANU_GEN = 'clise.rabbit.multiply_by_id'
else:
	STARTUP  = 'HERON_STARTUP'
	AUTO_GEN = 'rabbit.multiply_by_file'
	MANU_GEN = 'rabbit.multiply_by_id'

# ----------------------------------------------------------------------------
class cc:
	""" color codes
	"""
	orange	 = "\033[38;2;190;165;0m"
	green		 = "\033[38;2;46;145;36m"
	pink		 = "\033[38;2;185;145;175m"
	red		 = "\033[38;2;230;0;0m"
	# steelblue	 = "\033[38;2;176;196;222m"
	steelblue	 = "\033[38;2;156;176;202m"
	darkgrey	 = "\033[38;2;90;90;90m"
	reset		 = "\033[38;2;160;160;160m"

	key   = orange
	type  = green
	reset = reset
	none  = darkgrey
	hl    = pink
	err   = red
	defs  = steelblue
	""" """

class tk:
	eval   = '#'
	float  = '##'
	list   = r'\*'
	sep    = ','
	dict   = r'\*\*'
	json   = '='
	nest   = ':'
	neg    = '^'
	key    = r'[#\*^]'
	vstart = '{'
	vstop  = '}'

def str_to_list(input):
	if type(input) is str: return [input]
	else: return input

def extract(dict, key='-main', level='-top'):
	""" recursively extract the value for a key
	"""

	ans    = OrderedDict()
	unique = OrderedDict()
	if key in dict.keys():
		ans[level] = dict[key]
		unique[ans[level]] = ''

	for k, v in dict.items():
		if type(v) is not dict and type(v) is not OrderedDict: continue
		if level == '-top': cans, _ = extract(v, key=key, level=k)
		else:               cans, _ = extract(v, key=key, level=level + ':' + k)

		for k_, v_ in cans.items():
			ans[k_] = v_
			unique[v_] = ''

	unique = list(unique.keys())

	return ans, unique

	""" """

def simplify(value):
	if   value > 1099511627776:	return "{:.1f}".format(value / 1099511627776) + 'T'
	elif value > 1073741824:	return "{:.1f}".format(value / 1073741824)    + 'G'
	elif value > 1048576:		return "{:.1f}".format(value / 11048576)      + 'M'
	elif value > 1024:		return "{:.1f}".format(value / 1024)          + 'k'
	else:					return str(value)
# ----------------------------------------------------------------------------
def truekey(key, tk=tk.key):
	if type(key) is str: return  re.sub(tk, '', key)
	else:			   return [re.sub(tk, '', k) for k in key]

def merge(base, add, onlynew=False):
	# this doesn't deal with
	# 	sub dict parameters:  only when given with the special key: e.g., -k1:k2...
	# 	value substitute:	    this could be messy
	# 	special keys:	    easy? make sure the order

	if add is None: return base
	if len(add) == 0: return base
	add_keys      = list(add.keys())
	true_add_keys = truekey(add_keys)

	if not onlynew:
		for each in true_add_keys:
			idx       = true_add_keys.index(each)
			key       = add_keys[idx]
			base[key] = add[key]

		return base

	base_keys      = list(base.keys())
	true_base_keys = truekey(base_keys)

	for each in true_add_keys:
		if each not in true_base_keys:
			idx       = true_add_keys.index(each)
			key       = add_keys[idx]
			base[key] = add[key]

	return base

def merge_before(base, add):
	# this doesn't deal with
	# 	sub dict parameters:  only when given with the special key: e.g., -k1:k2...
	# 	value substitute:	    this could be messy
	# 	special keys:	    easy? make sure the order

	accept = str_to_list(base.get('-accept', True))  # bool or list of key names or matching reg expr
	reject = str_to_list(base.get('-reject', []))    # list of key names or matching reg expr

	add_keys      = list(add.keys())
	true_add_keys = truekey(add_keys)

	base_keys      = list(base.keys())
	true_base_keys = truekey(base_keys)

	if type(accept) is list:
		if '--listed' in accept:
			accept.remove('--listed')
			accept = accept + list(base.keys())

		for each in accept:
			if each in true_add_keys:
				idx       = true_add_keys.index(each)
				key       = add_keys[idx]
				if key not in true_base_keys: base[key] = add[key]
			else:
				for each2 in true_add_keys:
					if bool(re.search(each, each2)):
						idx       = true_add_keys.index(each2)
						key       = add_keys[idx]
						if key not in true_base_keys: base[key] = add[key]
						break
		return base

	if '--listed' in reject:
		reject.remove('--listed')
		reject = reject + list(base.keys())

	if accept:
		for each2 in true_add_keys:
			acceptable = True
			for each in reject:
				if bool(re.search(each, each2)):
					acceptable = False
					break
			if acceptable:
				idx       = true_add_keys.index(each2)
				key       = add_keys[idx]
				if key not in true_base_keys: base[key] = add[key]
	return base

def merge_after(base, add):
	# this doesn't deal with
	# 	sub dict parameters:  only when given with the special key: e.g., -k1:k2...
	# 	value substitute:	    this could be messy
	# 	special keys:	    easy? make sure the order

	accept = str_to_list(base.get('-accept', True))  # bool or list of key names or matching reg expr
	reject = str_to_list(base.get('-reject', []))    # list of key names or matching reg expr

	add_keys      = list(add.keys())
	true_add_keys = truekey(add_keys)

	base_keys      = list(base.keys())
	true_base_keys = truekey(base_keys)

	if type(accept) is list:
		if '--listed' in accept:
			accept.remove('--listed')
			accept = accept + list(base.keys())

		for each in accept:
			if each in true_add_keys:
				idx       = true_add_keys.index(each)
				key       = add_keys[idx]
				base[key] = add[key]
				try:
					idx      = true_base_keys.index(each)
					okey	   = base_keys[idx]
					if okey != key: base.pop(okey)
				except Exception:
					pass

			else:
				for each2 in true_add_keys:
					if bool(re.search(each, each2)):
						idx       = true_add_keys.index(each2)
						key       = add_keys[idx]
						base[key] = add[key]
						try:
							idx      = true_base_keys.index(each2)
							okey	   = base_keys[idx]
							if okey != key: base.pop(okey)
						except Exception:
							pass
						break
		return base

	if '--listed' in reject:
		reject.remove('--listed')
		reject = reject + list(base.keys())

	if accept:
		for each2 in true_add_keys:
			acceptable = True
			for each in reject:
				if bool(re.search(each, each2)):
					acceptable = False
					break
			if acceptable:
				idx       = true_add_keys.index(each2)
				key       = add_keys[idx]
				base[key] = add[key]
				try:
					idx      = true_base_keys.index(each2)
					okey	   = base_keys[idx]
					if okey != key: base.pop(okey)
				except Exception:
					pass
	return base

# dict routines
def substitute_local(inp, pre=None, post='}', level=0):
	""" parameters starting with ^ and . are used to replace variables of the same names
	"""

	if pre is None:
		if   level == -2: pre = '{<<'
		elif level == -1: pre = '{<'
		elif level == 0:  pre = '{'
		elif level == 1:  pre = '{>'
		else:		      pre = '{>>'

	text = json.dumps(inp)

	keys  = list(inp.keys())
	tkeys = truekey(keys)

	for key, tkey in zip(keys, tkeys):
		if tkey == '': continue

		# to avoid the infinite loop, skip the self reference
		val = str(inp[key])
		if bool(re.search(pre + tkey + post, val)): continue

		new_text = text
		while True:
			text = new_text
			new_text = text.replace(pre + tkey + post, val)
			if new_text == text: break

	out = json.loads(text, object_pairs_hook=OrderedDict)
	return out
	""" """

def substitute_global_truekeys(inp, pre='{', post='}'):
	""" parameters starting with ^ and . are used to replace variables of the same names
	"""
	text = json.dumps(inp)

	merged = inp[0].copy()

	keys = []
	for each in inp:
		keys  = keys + list(each.keys())
		merged = merge(merged, each)
	tkeys = truekey(keys)

	for key, tkey in zip(keys, tkeys):
		if tkey == '': continue

		# to avoid the infinite loop, skip the self reference
		val = str(merged[key])
		if bool(re.search(pre + tkey + post, val)): continue

		new_text = text
		while True:
			text = new_text
			new_text = text.replace(pre + tkey + post, val)
			if new_text == text: break

	out = json.loads(text, object_pairs_hook=OrderedDict)
	return out
	""" """

def substitute_global(inp, pre='{', post='}'):
	""" parameters starting with ^ and . are used to replace variables of the same names
	"""
	text = json.dumps(inp)

	merged = inp[0].copy()

	keys = []
	for each in inp:
		try:
			keys  = keys + list(each.keys())
			merged = merge(merged, each)
		except Exception:
			import plottool
			plottool.embed()

	for key in keys:

		# to avoid the infinite loop, skip the self reference
		val  = str(merged[key])
		key_ = key.replace('*', '\*')  # to handle str -> dict conversion with **key
		if bool(re.search(pre + key_ + post, val)): continue

		new_text = text
		while True:
			text = new_text
			new_text = text.replace(pre + key + post, val)
			if new_text == text: break

	out = json.loads(text, object_pairs_hook=OrderedDict)
	return out
	""" """

# ----------------------------------------------------------------------------
def nested_key(cfg, key, val=None, sep=':', pop=True):
	""" simple par into a nested variable

	"""
	if val is None: val = cfg[key]

	# itype = type(cfg)
	mat = re.search("^(.+)" + sep + "(.+)$", key)
	if not bool(mat):
		cfg[key] = val
		return cfg

	if pop: cfg.pop(key, None)
	key, rest = mat.group(1, 2)
	if key not in cfg.keys(): cfg[key] = OrderedDict()

	cfg[key] = nested_key(cfg[key], rest, val=val, sep=sep)
	return cfg

def expand_nested(cfg, sep=":", pop=True):
	if sep == ''   : return cfg

	new = cfg.copy()
	for k, v in cfg.items():
		new = nested_key(new, k, sep=sep, pop=pop)
	return new

def correct_json_par(cfg, pre="=", pop=True):
	if pre == ''   : return cfg

	new = cfg.copy()
	for k, v in cfg.items():
		if type(v) is dict or type(v) is OrderedDict:
			new[k] = correct_json_par(cfg[k], pre=pre, pop=pop)
			continue

		if k[0] == "=":
			new[k[1:]] = v
			if pop: del new[k]

	return new

def expand_dict(cfg, pre=r"\*\*", pop=True):
	if pre == ''   : return cfg

	new = cfg.copy()
	for k, v in cfg.items():
		if type(v) is dict or type(v) is OrderedDict:
			new[k] = expand_dict(cfg[k], pre=pre, pop=pop)
			continue

		# this may leave some weird key, parameter pair
		# since it doesn't pop
		if type(v) is not str: continue

		mat = re.search("^([-]*)" + pre + "(.+)$", k)
		if not bool(mat): continue

		newk = mat.group(1) + mat.group(2)

		if not bool(re.search(r'^[\n\s]*{', v)): v = '{' + v + '}'
		vals = hjson.loads(v, object_pairs_hook=OrderedDict)

		new[newk] = vals
		if pop: del new[k]

	return new

def expand_list(cfg, pre=r"\*", sep=",", pop=True):
	if pre == ''   : return cfg

	new = cfg.copy()
	for k, v in cfg.items():
		if type(v) is dict or type(v) is OrderedDict:
			new[k] = expand_list(cfg[k], pre=pre, pop=pop)
			continue

		if type(v) is not str: continue

		mat = re.search("^([-]*)" + pre + "(.+)$", k)
		if not bool(mat): continue
		newk = mat.group(1) + mat.group(2)

		vals = v.split(sep)
		new[newk] = vals
		if pop: del new[k]

	return new

def to_float(k, v, pre='##'):
	mat = re.search(r"^([-]*)" + pre + "(.+)$", k)
	if not bool(mat): return k, v

	newk = mat.group(1) + mat.group(2)

	if type(v) is str:
		try:
			return newk, float(v)
		except ValueError:
			print(cc.key + 'warning: enforcing floats failed for:', k, v, cc.reset)
			return newk, v
	elif type(v) is list:
		newv = []
		for ev in v:
			if type(ev) is not str:
				newv.append(ev)
				continue
			try:
				newv.append(float(ev))
			except ValueError:
				print(cc.key + 'warning: enforcing floats failed for:', k, ev, 'in', v, cc.reset)
				newv.append(ev)
		return newk, newv
	else:
		print('warning: enforcing floats failed for:', k, v)
		return newk, v

def to_numeric_(k, v, pre='#'):

	mat = re.search(r"^([-]*)" + pre + "(.+)$", k)
	if not bool(mat): return k, v

	newk = mat.group(1) + mat.group(2)
	if type(v) is str:
		try:
			return newk, int(v)
		except ValueError:
			try:
				return newk, float(v)
			except ValueError:
				print(cc.key + 'warning: enforcing numeric failed for:', k, v, cc.reset)
				return newk, v
	elif type(v) is list:
		newv = []
		for ev in v:
			if type(ev) is not str:
				newv.append(ev)
				continue
			try:
				newv.append(int(ev))
			except ValueError:
				try:
					newv.append(float(ev))
				except ValueError:
					print(cc.key + 'warning: enforcing numeric failed for:', k, ev, 'in', v, cc.reset)
					newv.append(ev)
		return newk, newv
	else:
		print(cc.key + 'warning: enforcing numeric failed for:', k, v, cc.reset)
		return newk, v

def to_numeric(k, v, pre='#', tofloat=False):

	mat = re.search(r"^([-]*)" + pre + "(.+)$", k)
	if not bool(mat): return k, v

	newk = mat.group(1) + mat.group(2)
	if type(v) is str:
		try:
			if tofloat: v = v + "*1.0"
			return newk, eval(v)
		except ValueError:
			print(cc.key + 'warning: enforcing numeric failed for:', k, v, cc.reset)
			import plottool as pt
			pt.embed()
			return newk, v
	elif type(v) is list:
		newv = []
		for ev in v:
			if type(ev) is not str:
				newv.append(ev)
				continue
			try:
				if tofloat: ev = ev + "*1.0"
				newv.append(eval(ev))
			except ValueError:
				print(cc.key + 'warning: enforcing numeric failed for:', k, ev, 'in', v, cc.reset)
				newv.append(ev)
			except NameError:
				newv.append(ev)
		return newk, newv
	else:
		print(cc.key + 'warning: enforcing numeric failed for:', k, v, cc.reset)
		import plottool as pt
		pt.embed()
		return newk, v

def enforce_numeric(cfg,
		pre4n="#",   # prefix for number (integer and floats)
		pre4f='##',  # prefix for floats
		pop=True):
	if pre4n is False and pre4n is None or pre4n == '': return cfg

	new = cfg.copy()
	for k, v in cfg.items():
		if type(v) is dict or type(v) is OrderedDict:
			new[k] = enforce_numeric(cfg[k], pre4n=pre4n, pre4f=pre4f, pop=pop)
			continue

		if pre4f is not None and pre4f != "":
			newk, newv = to_numeric(k, v, pre=pre4f, tofloat=True)
			if k != newk:
				new[newk] = newv
				if pop: del new[k]
				continue

		newk, newv = to_numeric(k, v, pre=pre4n)
		if k == newk: continue
		new[newk] = newv
		if pop: del new[k]
		continue

	return new

def enforce_format(cfg, enforcer=None):

	if enforcer is None: return cfg
	new = cfg.copy()
	for k, v in enforcer.items():
		if k not in cfg.keys(): continue

		if type(v) is dict or type(v) is OrderedDict:
			new[k] = enforce_format(cfg[k], enforcer[k])
			continue

		if type(cfg[k]) is list:
			try:
				if   v ==     'int': new[k] = [    int(ev) for ev in cfg[k]]
				elif v ==   'float': new[k] = [  float(ev) for ev in cfg[k]]
				elif v ==    'bool': new[k] = [   bool(ev) for ev in cfg[k]]
				elif v == 'complex': new[k] = [complex(ev) for ev in cfg[k]]
				elif v ==     'str': new[k] = [    str(ev) for ev in cfg[k]]
			except ValueError:
				print(cc.key + 'enforcing type', v, 'failed for', k + ':', cfg[k], cc.reset)
				exit()
		else:
			try:
				if   v ==     'int': new[k] =     int(cfg[k])
				elif v ==   'float': new[k] =   float(cfg[k])
				elif v ==    'bool': new[k] =    bool(cfg[k])
				elif v == 'complex': new[k] = complex(cfg[k])
				elif v ==     'str': new[k] =     str(cfg[k])
			except ValueError:
				print(cc.key + 'enforcing type', v, 'failed for', k + ':', cfg[k], cc.reset)
				exit()

	return new

def update_specific(cfg, only):
	new = cfg.copy()
	if only is not None:
		for k, v in only.items():
			if k not in cfg.keys(): new[k] = only[k]
			elif type(v) is dict or type(v) is OrderedDict:
				newv = new[k] if k in new.keys() else OrderedDict()
				for ek in v.keys() - newv.keys(): newv[ek] = v[ek]
				new[k] = newv
	return new

def routine_specific(cfg):

	if '-main' not in cfg: return cfg
	routine = cfg['-main']

	# ignore this if multiple mains are called, for now
	if type(routine) is not str: return cfg
	new = cfg.copy()

	for k, v in cfg.items():
		if not bool(re.search('^--only for ', k)): continue

		levels = routine
		while levels != '':
			if bool(re.search('^' + k, '--only for ' + levels)):
				new = update_specific(new, cfg[k].copy())
				break
			levels = '.'.join(levels.split('.')[:-1])

		del new[k]

	return new

def discard(cfg, pop=True):
	discard = cfg.get('-discard', [])
	if type(discard) is not list: discard = [discard]
	for each in discard: cfg.pop(each, None)
	if pop: cfg.pop('-discard', None)
	return cfg

def nonify(cfg, pop=True):
	nonify = cfg.get('-nonify', [])
	if type(nonify) is not list: nonify = [nonify]
	for each in nonify: cfg[each] = None
	if pop: cfg.pop('-nonify', None)
	return cfg

def process_cfg(cfg, level=0, ignore_main=False):

	cfg = cfg.copy()

	# order 3
	# load default parameters
	if '-load' in cfg.keys():
		if type(cfg['-load']) is str: cfg['-load'] = cfg['-load'].split(':')
		# note: this should be reversed in order
		for each in cfg['-load'][::-1]:
			each = path.expanduser(each)
			if not path.isfile(each):
				print('file not found for -load:', each)
				continue
			cfg_ = load_file(each)
			for each2 in cfg_:
				cfg = merge(cfg, each2, onlynew=True)

	# order 2
	# startup file or other load file with
	# "--only for ..." key
	cfg = routine_specific(cfg)

	# start up file
	# order 1
	startup = getenv(STARTUP)
	if startup is not None:
		startup = startup.split(':')
		# note: this should be reversed in order
		for each in startup[::-1]:
			each = path.expanduser(each)
			if not path.isfile(each):
				print('file not found for', STARTUP + ':', each)
				continue
			cfg_ = load_file(each)
			for each2 in cfg_:
				cfg = merge(cfg, each2, onlynew=True)

	if not ignore_main:
		if "-main" not in cfg: return cfg

	pop = cfg.get('-pop_modifier', True)
	# order 4
	# expand a string value to a dict, useful for cmdline or json file
	# -*key '"val1":1,"val2":2,... => "key": {"val1":1,"val2":2,..]
	cfg = expand_dict(cfg, pop=pop,
			pre=cfg.get('-dict_pre', tk.dict))

	# order 5
	# expand a key to a nested dict or OrderedDict from cmdline or json file
	# useful for cmdline, and adding parameters to an existing dict or OrderedDict
	# -k1:k2 val => "k1": { "k2": val}
	cfg = expand_nested(cfg, pop=pop,
			sep=cfg.get('-expand_nested', tk.nest))

	# order 6
	# expand a string value to a list, useful for cmdline or json file
	# -*key val1,val2,... => "key": [val1,val2]
	cfg = expand_list(cfg, pop=pop,
			pre=cfg.get('-list_pre', tk.list),
			sep=cfg.get('-list_sep', tk.sep))

	cfg = correct_json_par(cfg, pop=pop,
			pre=cfg.get('-json_pre', tk.json))

	# order 7
	# substitution?
	cfg = substitute_local(cfg, level=level)

	# '{<<parameter}' can be substituted by the command line options later
	# this requires a caution when used with a loop 
	cfg = substitute_local(cfg, level=-2) 

	# order 8
	# enforce numeric parameter from cmdline or json file
	# -#key "12.34" => "key": 12.34
	cfg = enforce_numeric(cfg, pop=pop,
			pre4n=cfg.get('-enforce_numeric', tk.eval),
			pre4f=cfg.get('-enforce_floats', tk.float))

	# order 9
	cfg = enforce_format(cfg,
			enforcer=cfg.get('-enforce_format', None))

	# order 10
	# either delete variables or set them None
	cfg = discard(cfg, pop=False)
	cfg = nonify (cfg, pop=False)

	return cfg

# ----------------------------------------------------------------------------
# reconfig a single task
def reconfig_to_taskgen(cfg, id=''):

	if type(id) is str: auto = True
	else:			  auto = False

	temp = cfg.get('-loop', OrderedDict()).copy()
	cfg.pop('-loop', None)
	temp[cfg.get('-seed_key', 'seed')] = cfg.copy()
	cfg = temp

	if '-main' not in cfg:
		if auto: cfg['-main'] = AUTO_GEN
		else:	   cfg['-main'] = MANU_GEN

	# cfg['-rabbit']=cfg.get('-rabbit',True)
	cfg['-block'] = cfg.get('-block', 'rabbit')
	return cfg

# reconfig all task
def reconfig_to_alltaskgen_(cfgs, id=''):
	inloop = False
	looped = []
	for each in cfgs[:]:
		# if each.get('-rabbit',False) > True:
		if 'release' in each.get('-block', ''):
			new = each.copy()
			new['-block'] = 'rabbit'  # clean up now
			seed   = []
			inloop = True
			looped.append(new)
		elif inloop:
			# if each.get('-rabbit',False) < False:
			if 'capture' in each.get('-block', ''):
				ceach = each.copy()
				ceach['-block'] = ''  # clean up now
				seed.append(ceach)
				new['seed'] = seed
				inloop = False
			else:
				seed.append(each.copy())
		else:
			looped.append(each)

	return looped

def reconfig_to_alltaskgen(cfgs, id=''):
	inloop = False
	looped = []
	# need to be more natural and systematic in future
	k1 = '-apply until -block'
	k2 = '>> -block'
	for each in cfgs[:]:
		# if each.get('-rabbit',False) > True:
		# if 'rabbit' in each.get('-block','') or \
		# if '-main' in each and bool(re.search('(-apply until|>>)( -block)', each)):
		if '-main' in each and (k1 in each or k2 in each):
			new = each.copy()
			new['-block'] = 'rabbit'  # clean up now
			if k1 in each: apply_until = new[k1]
			else:		   apply_until = new[k2]
			seed = []
			inloop = True
			looped.append(new)
		elif inloop:
			# if each.get('-rabbit',False) < False:
			if apply_until in each.get('-block', ''):
				ceach = each.copy()
				ceach['-block'] = ''  # clean up now
				seed.append(ceach)
				new['seed'] = seed
				inloop = False
			else:
				seed.append(each.copy())
		else:
			looped.append(each)

	return looped

def matchID(cid, cfg, matkey='-id'):
	""" case match for a given parameter, and reset other parameters accordingly
	This is targeted for CLI only.
	"""

	if cid == '': return cfg

	ans = cfg.copy()
	regex = '(==|!=|~=|~!=)'

	for k, v in cfg.items():
		mat = re.search(matkey + regex + '(.*)', k)
		if not bool(mat): continue

		matype = mat[1]

		mat_ = re.search('(.*):(.*)', mat[2])
		if bool(mat_):
			# this is likely cli
			phrase = mat_[1]
			newkey = mat_[2]

			if matype == '==':
				if cid == phrase: ans[newkey] = ans[k]
			elif matype == '!=':
				if cid != phrase: ans[newkey] = ans[k]
			elif matype == '~=':
				if bool(re.search(phrase, cid)): ans[newkey] = ans[k]
			elif matype == '~!=':
				if not bool(re.search(phrase, cid)): ans[newkey] = ans[k]

		else:
			# this is likely from a file, v should be dictionary
			phrase = mat[2]

			feedit = False
			if matype == '==':
				if cid == phrase: feedit = True
			elif matype == '!=':
				if cid != phrase: feedit = True
			elif matype == '~=':
				if bool(re.search(phrase, cid)): feedit = True
			elif matype == '~!=':
				if not bool(re.search(phrase, cid)): feedit = True

			if feedit:
				for each in v: ans[each] = v[each]

		ans.pop(k)

	return ans

def matchblock(source, target, tk_word='apply to', tk_symbol='>>', ifnone=True):
	merge = False
	apply = []
	# perhaps natural and systematic enough?
	for skey in source:
		mat = re.search(r'(' + tk_word + '|' + tk_symbol + ') ([^ ]+)(| except| contains| without| ==| !=| ~=| ~!=)$', skey)
		if not bool(mat): continue

		apply.append(skey)
		tkey = mat.group(2)
		# print('here', skey, tkey)
		if tkey not in target: continue

		# print('her2', target[tkey], mat.group(3))
		# import plottool as pt
		# pt.embed()
		# source[sky] can be string or string list
		# if bool(re.search(r"(==|)", mat.group(3))):
		if mat.group(3) == "" or mat.group(3) == " ==":
			if target[tkey] in source[skey]: merge = True
		# if bool(re.search(r"(except|!=)", mat.group(3))):
		if mat.group(3) == " except" or mat.group(3) == " !=":
			if target[tkey] not in source[skey]: merge = True

		# source[sky] is assumed to be string
		if mat.group(3) == " contains" or mat.group(3) == " ~=":
			# print(source[skey])
			if type(source[skey]) is not str:
				print(cc.err + 'expect a regex string for', skey, source[skey], cc.reset)
				continue
			if bool(re.search(source[skey], target[tkey])):
				merge = True
			# print('here', merge)

		if mat.group(3) == " without" or mat.group(3) == " ~!=":
			if type(source[skey]) is not str:
				print(cc.err + 'expect a regex string for', skey, source[skey], cc.reset)
				continue
			if not bool(re.search(source[skey], target[tkey])): merge = True

	if ifnone:
		if len(apply) == 0: merge = True
	return merge, apply

def defaults(cfg):
	cfg['-verbose'] = cfg.get('-verbose', 0)
	cfg['-clobber'] = cfg.get('-clobber', False)
	return cfg

def get_task_ready(task, before, after,
			matval=None, matkey=['-id', '-main', '-nid', '-fid'], doprocess=True):

	if matval is None:
		matval = []
		for each in matkey: matval.append(task.get(each, ''))

	filtered_before = OrderedDict()
	for ckey, cmat in zip(matkey, matval):
		filtered_before = merge(filtered_before, matchID(cmat, before, matkey=ckey))

	filtered_after = OrderedDict()
	for ckey, cmat in zip(matkey, matval):
		filtered_after = merge(filtered_after, matchID(cmat, after, matkey=ckey))

	task   = merge_before(task, filtered_before)
	task   = merge_after (task, filtered_after )

	task   = defaults(task)
	if doprocess: return process_cfg(task, level=doprocess - 1)
	else:         return task

def set_id(short, counter, pref=''):
	nid = f"{counter:02}"
	#   short id: there can be duplicate for this
	# numeric id: no duplicate is possible
	#    full id:
	return short, pref + nid, pref + nid + ':' + short

# ----------------------------------------------------------------------------
json_counter = 0

def load_file(file, astext=False):
	""" read json file, add proper beginning and brackets, and remove trailing ","
	"""
	out = OrderedDict()
	try:
		jsonfile = open(file)
		text = jsonfile.read()
		jsonfile.close()
	except FileNotFoundError:
		print(cc.err + 'file not found:', file, cc.reset)

	# add beginning and ending {} if missing
	# if it has {}, it's either multiple sets or single complete set
	# ready = True

	# outer most brackets {} or [] can be omitted
	if not bool(re.match(r'[\n\s]*[{\[]', text)): text = '{' + text + '}'
	if not bool(re.match(r'[\n\s]*[\[]',  text)): text = '[' + text + ']'

	# now get rid of trailing ","
	text = re.sub(r',([\n\s]*)([}\]])', r'\1\2', text)
	if astext: return text

	try:
		out = hjson.loads(text, object_pairs_hook=OrderedDict)
		out = substitute_global(out)
	except ValueError as ve:
		print(cc.err + 'The file', file, 'failed parsing!')
		print(ve, cc.reset)
		exit()

	# if not ready:
		# if '-main' in out[0]: ready=True

	return out  # , ready
	""" """

def group_cli(cli=sys.argv[1:]):
	""" group the cli sets
	the main function, enables a series of runs
	"""

	# cli=sys.argv[1:]

	if type(cli) is not list: cli = cli.split()

	# lower priority: set with --with
	mergefile = []  # common json file set
	merge_cli = []  # common command line

	# normal priority: supercede mergefile, and merge_cli
	stackfile = []  # json file set
	stack_cli = []  # command line for each json

	# high priority: supercede stackfile, stack_cli, set with --WITH
	overwfile = []  # json file set
	overw_cli = []  # command line for each json

	# order in CLISE
	# clise stack* .... --with merge* .... --WITH overw* ...

	parse = True
	if '-^-parse'    in cli: parse = False
	if '--^parse'    in cli: parse = False

	ext  = '(json|json5|hjson|jsonc)'
	expr = r"^[^-{].*\." + ext + "$"

	prev_par = "^-=[a-zA-Z]"

	# The extended JSON format is used to feed the input parameters
	# since it is easy to convert to a python dictionary.
	# Check for JSON, JSON5, JSONC, HJSON files for now:
	# The accepted format allows a simple math expr using python's
	# eval function with '<', '>'.
	# Since JSON is not exactly optimized for configuration files,
	# in future perhaps other configuration file formats will are accepted as
	# well: e.g., yaml?
	# Note the format without the outer most brackets '{',''}' like in JSON5 is
	# allowed to simulate a more config-like file, but this violates the JSON
	# syntax set by LSP, which can be annoying for some editors.

	# check common variables

	if '--WITH' in cli:
		idx = cli.index('--WITH')
		for par in cli[idx + 1:]:
			if bool(re.search(expr, par)): overwfile.append(par)
			else:                          overw_cli.append(par)
		cli = cli[0:idx]

	if '--with' in cli:
		idx = cli.index('--with')
		for par in cli[idx + 1:]:
			if bool(re.search(expr, par)): mergefile.append(par)
			else:                          merge_cli.append(par)
		cli = cli[0:idx]

	# run one by one
	# need to fix this for command line version
	orphan = []
	expect_jsfile = False
	for par in cli:
		if bool(re.search(expr, par)) and parse and not expect_jsfile:
			stackfile.append(par)
			new = []
			stack_cli.append(new)
		else:
			try:
				new.append(par)
			except NameError:
				orphan.append(par)

		if bool(re.search(prev_par, par)): expect_jsfile = True

	if len(stackfile) == 0:
		return [orphan + mergefile + merge_cli + stack_cli], overwfile + overw_cli

	cli_sets = []
	for eachfile, each_cli in zip(stackfile, stack_cli):
		cli_sets.append(orphan + mergefile + [eachfile] + merge_cli + each_cli)
		# cli_sets.append(orphan + merge_cli + mergefile + [eachfile] +  each_cli)
		# now eachfile can have multiple tasks

	return cli_sets, overwfile + overw_cli
	""" """

def analyze_cli(cli):
	""" for a given cli set, get the cfg sets
	since the main cjson file can have multiple sets
	"""

	cfg_cli = OrderedDict()  # current cli cfg

	potential = False  # could be potentially not a boolean parameter
	jsonpar = []

	for par in cli:
		# dela with negation operator '^'
		mat = re.search("^-([^0-9].*)", par)
		if bool(mat):
			key = mat.group(1)
			# first deal with -^key or -^-key
			matneg = re.search(r"^\^(.+)", key)
			if bool(matneg):
				key = matneg.group(1)
				cfg_cli[key] = False
			else:
				# second deal with --^key, which is the same as -^-key
				matneg2 = re.search(r"^-\^(.+)", key)
				if bool(matneg2):
					key = '-' + matneg2.group(1)
					cfg_cli[key] = False
				else:
					cfg_cli[key] = True
					potential = True
		else:
			# now we assume everything else is value for a key or json (either par or file)
			# which include a negative number
			# so keys like -5a are not allowed, which may cause an error or
			# taken as value of a key
			if potential:
				cfg_cli[key] = par
				potential = False
			else:
				# these are json
				jsonpar.append(par)

	# handle json file or json format parameter
	before = OrderedDict()  # before base
	after  = OrderedDict()  # after base
	base   = []	        # these are the base sets
	shared = []	        # these are blocks shared for some
	# jsonfiles =[]
	global json_counter
	for par in jsonpar:
		mat = re.search("^{(.+)", par)
		if bool(mat):
			cfg_ = hjson.loads(par, object_pairs_hook=OrderedDict)
			for key in cfg_: cfg_cli[key] = cfg_[key]
		else:
			# cfg_file, ready=load_file(par)
			# jsonfiles.append(par)
			# ncfg=len(cfg_file)
			# if not ready:
			# 	if len(base) == 0: before=merge(before, cfg_file[0])
			# 	else:			 after =merge(after , cfg_file[0])
			# else:
			# 	base = base + cfg_file
			json_counter = json_counter + 1
			cfg_file = load_file(par)
			cfg_file = reconfig_to_alltaskgen(cfg_file)
			for each in cfg_file:
				# need to be more natural and systematic?
				# esp need to absorp seed block for task gen,
				# then we don't need this rabbit part
				# the challenge is this part might not be ready for reconfig...
				# we can force one file contains the full list of task gen
				# if '-main' not in each and 'rabbit' not in each.get('-block',''):
				# downside is CLI options may not easily go in here...
				if '-main' not in each:
					# the latter is common block for rabbit's task gen,
					# so don't separate out
					shared = shared + [each]
				else:
					base = base + [each]

	after = merge(after, cfg_cli)
	# feed cfg_cli to task_gen seed if requested
	# this should have some option: use accept and reject?
	k1 = '-apply until -block'
	k2 = '>> -block'
	for each in base:
		if '-main' in each and (k1 in each or k2 in each):
			seed = each.get('seed', [OrderedDict()])
			seed = seed + [cfg_cli]
			each['seed'] = seed

	# -main is given by the cli
	if len(base) == 0:
		if '-main' in after:
			base = [after]
			after = OrderedDict()
		elif '-main' in before:
			base = [before]
			before = OrderedDict()
		else:
			after = merge(after, before)
			base = [after]  # this should be a simple task run

	return base, before, after, shared

def load_cli(cli=sys.argv[1:]):
	cli_sets, cli_ovw = group_cli(cli)
	if len(cli_ovw) > 0:
		ovw_base, *_ = analyze_cli(cli_ovw)
		ovw_base = ovw_base[0]
	else: ovw_base = OrderedDict()
	cfgs    = []
	befores = []
	afters  = []
	itask   = 0
	for cli in cli_sets:
		base, before, after, shared = analyze_cli(cli)  # a bit of redundancy
		# base=reconfig_to_alltaskgen(base)
		for task in base:

			# assign -id
			if '-main' in task:
				# if '-loop' in task or type(id) is not str:

				# this is to parse --loop:... in cli
				task = expand_nested(task)

				if '-loop' in task:
					# task generator routine
					id = task.get('-id', '')

					task = reconfig_to_taskgen(task, id=id)

					# set basic id
					name = task['-main'].split('.')[-1]
					task['-id'], task['-nid'], task['-fid'] = set_id(name, itask + 1)

					# need to handle -id match
					task = get_task_ready(task, before, after, doprocess=False)

					# feed cli to generated tasks
					# some task generator will do actual io in addition to pushing around
					# the variables, the parameters have to be real: i.e., doprocess=True
					task['seed'] = get_task_ready(task['seed'], before, after,
								matval=task['-id'], doprocess=True)

					task = process_cfg(task)
				else:

					# now check shared ones for common
					for share in shared:
						merge, apply = matchblock(share, task)
						if merge:
							# clean up keys with -apply to ...
							share_ = share.copy()
							for each in apply:
								if each in share_: del share_[each]
							task = merge_before(task, share_)
							# from IPython import embed; embed()
							# if not after.get('-holdsub', False):
							task = substitute_local(task, level=-1)
							# from IPython import embed; embed()

					# this is a regular routine
					id = task.get('-id', '')
					if id == '': name = task['-main'].split('.')[-1]
					else:		 name = task['-id']
					task['-id'], task['-nid'], task['-fid'] = set_id(name, itask + 1)

					# need to handle -id match
					task = get_task_ready(task, before, after, doprocess=True)
			else:
				if len(base) == 1:
					# this is a (only) regular routine
					task = defaults(task)
					task = process_cfg(task, ignore_main=True)
				else:
					# now this is a partial block that needs to be distributed
					# perhaps right now it never gets here?
					pass
					# embed()

			skip = False
			for share in shared:
				skip, apply = matchblock(share, task, tk_word='skip', tk_symbol='<<', ifnone=False)
				if skip: break

			if skip: continue

			cfgs    .append(task)
			befores .append(before)
			afters  .append(after)

			itask = itask + 1

	return cfgs, befores, afters, ovw_base

# ----------------------------------------------------------------------------
def show_(dict, ref=None, tab=0, cc=cc, full=False,
		notype=False, hidden='-', exceptions=['-prep', '-post']):
	""" show input parameters
	"""

	do = False
	if len(dict) == 0: do = True

	if do:
		keys   = dict.keys()
		if len(keys) == 0: return
		types  = [type(each).__name__ for each in dict.values()]
		values = dict.values()
		strval = [str(each) for each in dict.values()
				if type(each) is not dict and type(each) is not OrderedDict]

		maxkey   = len(max(keys,   key=len))
		maxtype  = len(max(types,  key=len))
		maxvalue = len(max(strval, key=len))
		if maxvalue > 50: maxvalue = 50
	else:
		maxtype = 6
		maxvalue = 6

	doref = False
	if ref is not None:
		lenref = len(ref)
		if lenref > 0: doref = True

	# nothing to do
	if not do and not doref: return

	if doref:
		reftypes   = [type(each).__name__ for each in ref.values()]
		maxreftype = len(max(reftypes, key=len))
		if not do:
			maxkey = len(max(ref.keys(), key=len))

	tab_ = ''.rjust(tab)
	if notype: nextab = tab + maxkey + 5
	else:	     nextab = tab + maxkey + maxtype + 5

	if do:
		for key, typ, value in zip(keys, types, values):
			cckey = cc.key
			if key[0:1] == hidden:
				cckey = cc.defs
				if not full:
					if key not in exceptions: continue

			key_ = tab_ + cckey + key.rjust(maxkey + 1)

			if typ is None:
				typ_   = cc.none + typ.ljust(maxtype)
				value_ = cc.none + str(value).ljust(maxvalue)
			else:
				typ_   = cc.type  + typ.ljust(maxtype)
				value_ = cc.reset + str(value).ljust(maxvalue)

			if typ == 'dict' or typ == 'OrderedDict':
				if notype:
					print(key_, cc.reset)
					maxtype = 0
				else:		print(key_, typ_, cc.reset)
				cref = None
				if doref:
					if key in ref.keys(): cref = ref[key]
				show(value, tab=nextab, full=full, ref=cref, notype=notype)
				continue

			if typ == 'list':
				if len(value) > 0:
					if type(value[0]).__name__ == 'dict' or type(value[0]).__name__ == 'OrderedDict':
						if notype: print(key_, cc.reset, '[')
						else:      print(key_, typ_, cc.reset, '[')
						for each in value:
							cref = None
							if doref:
								if key in ref.keys(): cref = ref[key]
							show(each, tab=nextab, full=full, ref=cref, notype=notype)
							if each != value[-1]:
								print(tab_ + ','.rjust(maxkey + 3 + maxtype * (1 - notype)))
						print(tab_ + ']'.rjust(maxkey + 3 + maxtype * (1 - notype)))
						continue

			if doref:
				if key in ref.keys():
					reftype = type(ref[key]).__name__
					if reftype == 'NoneType':
						reftype = cc.none  + reftype.ljust(maxreftype)
						refval  = cc.none  + str(ref[key])
						feed    = cc.none + '<<'
					else:
						if reftype != typ and typ != 'NoneType':
							reftype = cc.hl   + reftype.ljust(maxreftype)
						else:
							reftype = cc.type + reftype.ljust(maxreftype)
						feed   = cc.reset + '<<'
						refval = cc.reset + str(ref[key])

					if notype:
						print(key_, value_, feed, refval, cc.reset)
					else:
						print(key_, typ_, value_, feed, reftype, refval, cc.reset)
			else:
				if notype: print(key_, value_, cc.reset)
				else:      print(key_, typ_,  value_, cc.reset)

	if doref:
		typ_   = cc.err + '?'.rjust(maxtype)
		value_ = cc.err + '?'.ljust(maxvalue)
		if do: keys = ref.keys() - dict.keys()  # why does this keep changing the order???
		else:  keys = ref.keys()
		for key in keys:
			key_ = tab_ + cc.key + key.rjust(maxkey + 1)
			reftype = type(ref[key]).__name__

			if reftype == 'dict' or reftype == 'OrderedDict':
				print(key, reftype, cc.reset)
				cref = None
				show(OrderedDict(), tab=nextab, full=full, ref=ref[key], notype=notype)
				continue

			if reftype == 'NoneType':
				reftype = cc.none  + reftype.ljust(maxreftype)
				refval  = cc.none  + str(ref[key])
				feed    = cc.none + '<<'
			else:
				reftype = cc.hl   + reftype.ljust(maxreftype)
				feed   = cc.reset + '<<'
				refval = cc.reset + str(ref[key])
			if notype:
				print(key_, value_, feed, refval, cc.reset)
			else:
				print(key_, typ_, value_, feed, reftype, refval, cc.reset)

	""" """

console = Console()
def show(input_, ref=None, hidden='-', level=0, tab=0, notype=False, maxkey=1, maxtype=1, full=False):
	if len(input_) == 0: return

	if not full: input_ = {k: v for k, v in input_.items() if k[0] != '-'}
	input = input_.copy()

	keys    = list(input.keys())
	if len(keys) == 0: return
	types   = [type(each).__name__ for each in input.values()]
	types   = ['odict' if v == 'OrderedDict' else v for v in types]

	maxkey_  = len(max(keys,   key=len))
	maxtype_ = len(max(types,  key=len))
	if maxkey  < maxkey_  : maxkey  = maxkey_
	if maxtype < maxtype_ : maxtype = maxtype_

	space = ' ' * tab

	table = Table(show_footer=False, show_header=False, show_edge=False)
	table.add_column('depth' ,  no_wrap=True)

	table.add_column("key"   ,  no_wrap=True)
	table.columns[1].width = maxkey
	table.columns[1].style = "bold dim yellow"
	table.columns[1].justify = "right"

	if notype:
		table.add_column("value" , no_wrap=False)
		table.columns[2].style = "dim white"
	else:
		table.add_column("type"  ,  no_wrap=True)
		table.columns[2].style = "green"
		table.columns[2].width = maxtype

		table.add_column("value" , no_wrap=False)
		table.columns[3].style = "dim white"

	table.border_style = None
	table.box          = None
	table.pad_edge     = False

	child = None
	# for k, v in input_.items():
	for k, t, v in zip(keys, types, input_.values()):

		if k[0] == hidden: k_ = '[cyan]' + k + '[/cyan]'
		else:              k_ = k

		if t == 'dict' or t == 'odict':
			if notype: table.add_row(space, k_, '')
			else:      table.add_row(space, k_, t, '')
			child = v.copy()
			isdict = True
			del input[k]
			break
		elif t == 'list':
			if len(v) > 0:
				if type(v[0]).__name__ == 'dict' or type(v[0]).__name__ == 'OrderedDict':
					if notype: table.add_row(space, k_, ']')
					else:      table.add_row(space, k_, t, '[')
					child = [ev.copy() for ev in v]
					isdict = False
					del input[k]
					break

		if notype: table.add_row(space, k_, str(v))
		else:	     table.add_row(space, k_, t, str(v))
		del input[k]

	# table.width        = console.measure(table).maximum
	console.print(table)

	if child is not None:
		level_ = level + 1
		tab_   = tab + maxkey + 2 + (maxtype + 2) * (1 - notype)
		if isdict: show(child, ref=ref, hidden=hidden, level=level_, tab=tab_, notype=notype, full=full)
		else:
			tab_ = tab_ + 3
			for v in child:
				show(v, ref=ref, hidden=hidden, level=level_, tab=tab_, notype=notype, full=full)
				if v != child[-1]: print(','.rjust(tab_))
				else:			 print(']'.rjust(tab_))
	if len(input) > 0: show(input, ref=ref, hidden=hidden, level=level, maxkey=maxkey, maxtype=maxtype,
							tab=tab, notype=notype, full=full)

def show_feed(args, kwargs, name, inargs=[], inkwargs=OrderedDict(), cc=cc):

	# print(' main:',cc.key+name+cc.reset)
	print(' main:', cc.key + name + cc.reset)
	ninargs   = len(inargs)
	# ninkwargs = len(inkwargs)

	if len(args) > 0:
		if 'self' in args: args = args[1:]
		if len(args) > 0:
			maxargs = len(max(args, key=len))

	if ninargs > 0:
		types   = [type(each).__name__ for each in inargs]
		maxtype = len(max(types, key=len))

	for i, each in enumerate(args):
		arg_ = cc.hl + each.rjust(maxargs + 1)

		if i >= ninargs:
			print(arg_, cc.reset)
			continue
		if types[i] == 'NoneType':
			feed   = cc.none + '<<'
			type_  = cc.none + types[i].ljust(maxtype)
			inarg_ = cc.none + str(inargs[i])
		else:
			feed   = cc.reset + '<<'
			type_  = cc.type  + types[i].ljust(maxtype)
			inarg_ = cc.reset + str(inargs[i])

		print(arg_ + cc.reset, feed, type_, inarg_, cc.reset)

	show_(kwargs, ref=inkwargs, cc=cc)

def show_files(input_, cc=cc):
	""" show files
	"""

	input = input_.copy()

	checkin  = input.get('-checkin',  [])
	checkout = input.get('-checkout', [])
	if type(checkin)  is str: checkin  = [checkin]
	if type(checkout) is str: checkout = [checkout]
	check    = checkin + checkout
	lcheck   = len(check)
	if lcheck == 0:
		# print(cc.key+'No files to check',cc.reset)
		return

	for each in input:
		if input[each] is None: input[each] = ''
	
	maxlen = max([len(input.get(v, '')) for v in check])

	# this is not a regular parameter for heron or jsontool any more
	if input.get('-basedir', None) is not None:
		print('-basedir'.rjust(maxlen) + ':', input['-basedir'])

	for k in check:
		if k not in input: print(cc.key + 'No key named', k, 'in inputs', cc.reset)

		# k_ = k.rjust(maxlen)
		k_ = k
		if k not in input:
			print(cc.key + k, 'is not in input', cc.reset)
			continue
		if path.isfile(Path(input[k]).expanduser()):
			modtime = Path(input[k]).expanduser().stat().st_mtime
			size    = Path(input[k]).expanduser().stat().st_size
			regtime = str(datetime.fromtimestamp(modtime))[2:19:]
			if k in checkin:
				print(cc.key + k_ + ':', regtime, cc.reset + simplify(size).rjust(6), input[k])
			else:
				if input.get('-clobber', True):
					print(cc.err + 'overwriting', k_ + ':', regtime, cc.reset + simplify(size).rjust(6), input[k])
				else:
					print(cc.key + k_ + ':', regtime, cc.reset + simplify(size).rjust(6), input[k])

		else:
			if k in checkin:
				print(k_ + ':'.ljust(7) + cc.err + 'missing'.ljust(19), input[k] + cc.reset)
			else:
				print(k_ + ':'.ljust(7) + cc.key + 'missing'.ljust(19), input[k] + cc.reset)

	""" """
# -------------------------------------------------------------------
# extracted from Jonathaneunice/ansiwrap
# to eliminate ansi_terminate_lines, which seems to have a bug
# string_types = basestring if sys.version_info[0] == 2 else str
def ansilen(s):
	"""
	Return the length of a string as it would be without common
	ANSI control codes. The check of string type not needed for
	pure string operations, but remembering we are using this to
	monkey-patch len(), needed because textwrap code can and does
	use len() for non-string measures.
	"""
	if isinstance(s, str):
		s_without_ansi = re.compile('\x1b\\[(K|.*?m)').sub('', s)
		return len(s_without_ansi)
	else:
		return len(s)

# now replace the standard len of textwrap3
textwrap3.len = ansilen
