
from __future__ import unicode_literals

import os
import sys
import re
import codecs
import inspect
import uuid
import datetime
import json
from json import JSONEncoder
from collections import OrderedDict
from collections import namedtuple
from pyld import jsonld
try:
	from rdflib import Dataset as CG
except:
	from rdflib import ConjunctiveGraph as CG

KEY_ORDER_DEFAULT = 10000
LINKED_ART_CONTEXT_URI = "https://linked.art/ns/v1/linked-art.json"

try:
	STR_TYPES = [str, unicode] #Py2.7
	FILE_STREAM_CLASS = file
except:
	import io
	STR_TYPES = [bytes, str] #Py3.x
	FILE_STREAM_CLASS = io.TextIOBase


pyld_proc = jsonld.JsonLdProcessor()
min_context = {
	"crm": "http://www.cidoc-crm.org/cidoc-crm/",
    "sci": "http://www.ics.forth.gr/isl/CRMsci/",
    "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
    "rdfs": "http://www.w3.org/2000/01/rdf-schema#",
    "dc": "http://purl.org/dc/elements/1.1/",
    "dcterms": "http://purl.org/dc/terms/",
    "schema": "http://schema.org/",
    "skos": "http://www.w3.org/2004/02/skos/core#",
    "foaf": "http://xmlns.com/foaf/0.1/",
    "xsd": "http://www.w3.org/2001/XMLSchema#",
    "dig": "http://www.ics.forth.gr/isl/CRMdig/",
    "la": "https://linked.art/ns/terms/",
    "id": "@id",
    "type": "@type"
}
re_bnodes = re.compile("^_:b([0-9]+) ", re.M)
re_bnodeo = re.compile("> _:b([0-9]+) <", re.M)
re_quad = re.compile(" <[^<]+?> .$", re.M)

PropInfo = namedtuple("PropInfo", [
	'property', # the name of the property, eg 'identified_by'
	'predicate', # the name from the ontology, eg 'crm:P1_is_identified_by'
	'range', # the class which is the range of this predicate, eg model.Identifier
	'inverse_property', # the name of the property which is the inverse of this one, eg 'identifies'
	'inverse_predicate', # the predicate for the inverse of this one, eg 'crm:P1i_idenfifies'
	'multiple_okay', # can there be multiple values, eg True
	'profile_okay' # is this property okay according to the profile, eg 0 (no), 1 (yes), 2 (warn)
	])


class CromulentError(Exception):
	"""Base exception class"""

	resource = None

	def __init__(self, msg, resource=None):
		"""Initialize CidocError."""
		self.args = [msg]
		self.resource = resource

class ConfigurationError(CromulentError):
	"""Raised when an object (likely the factory) isn't configured properly for the current operation."""
	pass

class MetadataError(CromulentError):
	"""Base metadata exception."""
	pass

class RequirementError(MetadataError):
	"""Raised when schema/profile/metadata requirements not met."""
	pass

class DataError(MetadataError):
	"""Raised when data is not valid/allowed."""
	pass

class ProfileError(MetadataError):
	"""Raised when a class or property not in the configured profile is used"""
	pass

class CromJsonEncoder(JSONEncoder):

	def default(self, o):
		if isinstance(o, BaseResource):
			# print("Saw %r" % o)
			return o._minToJSON()
		else:
			return JSONEncoder.default(self, o)			

class CromulentFactory(object):

	def __init__(self, base_url="", base_dir="", lang="", full_names=False, 
		context="", context_file={}, load_context=True):
		self.base_url = base_url
		self.base_dir = base_dir

		self.debug_level = "warn"
		self.log_stream = sys.stderr

		self.materialize_inverses = False # Create the inverse relationship on the object to the subject
		self.full_names = False # Use full property and class names in output
		self.pipe_scoped_contexts = False # Serialize to "name|Pnn_x_y" for scoped properties (documentation)
		self.validate_properties = True # validate properties are even CRM
		self.validate_profile = True # Validate linked_art specific stuff
		self.validate_range = True # Raise if attempt to set prop to invalid class
		self.validate_multiplicity = True  # Raise if attempt to set n:1 to []
		self.auto_assign_id = True # Automatially assign a URI
		self.process_multiplicity = True # Return multiple with single value as [value]
		self.multiple_instances_per_property = "drop"
		self.allow_highlight = False # Allow the JSON to include a _highlight flag for re-rendering
		self.allow_elide = False

		self.pair_tree_levels = 0

		self.auto_id_type = "int-per-segment" #  "int", "int-per-type", "int-per-segment", "uuid", "uuid-segment"
		# self.default_lang = lang  # NOT USED
		self.filename_extension = ".json"  # some people like .jsonld
		self.context_uri = context # Might be a list, or a context value as a dict
		self.context_json = {}

		self.prefixes = {}
		self.prefixes_rev = {}
		self.context_rev = {}
		# Maybe load it up for prefixes
		if load_context:
			# Leave this as a map for future extensions
			context_filemap = {
				LINKED_ART_CONTEXT_URI: 
					os.path.join(os.path.dirname(__file__), 'data', 'linked-art.json')
			}
			context_filemap.update(context_file)
			self.load_context(context, context_filemap)

		self.elasticsearch_compatible = False # return {'id': 'uri'} instead of string
		self.linked_art_boundaries = False # break on linked art API boundaries between classes
		self.id_type_label = True # references are id, type and _label, not just id.

		# if sorting is unimportant, use fast. If sorting is important, and python >= 3.6, use fast.
		# fast is approximately half the time for serializing
		self.json_serializer = "normal" # "normal" or "fast"
		self.json_indent = 2
		self.order_json = True
		self.key_order_hash = {"@context": 0, "id": 1, "type": 2, 
			"_label": 5, "value": 6}
		self.full_key_order_hash = {"@context": 0, "@id": 1, "rdf:type": 2, "@type": 2,
			"rdfs:label": 5, "rdf:value": 6}
		self.key_order_default = 10000

		self.underscore_properties = ["_label"]

		self._auto_id_types = {}
		self._auto_id_segments = {}
		self._auto_id_int = -1
		self._all_classes = {}

	def load_context(self, context, context_filemap):
		if not context or not context_filemap:
			raise ConfigurationError("No context provided, and load_context not False")

		if type(context) is not list:
			context = [context]

		js = {'@context': {}}
		for ct in context:
			fn = context_filemap.get(ct, "")
			if fn:
				try:
					fh = open(fn)
					data = fh.read()
					fh.close()
				except IOError:
					raise ConfigurationError("Provided context file does not exist")
			else:
				# XXX Fetch from web
				data = "{}"

			try:
				ctx = json.loads(data)
				js['@context'].update(ctx['@context'])
			except:
				raise ConfigurationError("Provided context does not have valid JSON")				
		# this is the merged context information, not any single one
		self.context_json = js
		self.process_context()

	def process_context(self):
		# Filter context looking for prefixes
		# And make reverse mapping
		# Note that this does not process scoped contexts for member_of / part_of
		for (k,v) in self.context_json['@context'].items():
			if type(v) in STR_TYPES and v[-1] in ['/', '#']:
				self.prefixes[k] = v
				self.prefixes_rev[v] = k
			elif k == "@version":
				continue
			else:
				if type(v) in STR_TYPES:
					rdf = v
				else:
					rdf = v['@id']
				self.context_rev[rdf] = k

	def __getstate__(self):
		# Make a copy of current object state
		d = self.__dict__.copy()
		# try to flush the stream
		try:
			self.log_stream.flush()
		except:
			# stream instance may not support flush()
			pass
		# Now reify the log stream
		strm = d['log_stream']
		if strm is sys.stdout:
			d['log_stream'] = ("sys.stdout", "stream")
		elif strm is sys.stderr:
			d['log_stream'] = ("sys.stderr", "stream")
		elif isinstance(strm, FILE_STREAM_CLASS):
			d['log_stream'] = (strm.name, "file")
		else:
			d['log_stream'] = None
		return d

	def __setstate__(self, state):
		# State is __dict__ with a reified log_stream as above
		self.__dict__.update(state)
		if self.log_stream:
			if self.log_stream[1] == "stream":
				if self.log_stream[0] == "sys.stdout":
					self.log_stream = sys.stdout
				elif self.log_stream[0] == "sys.stderr":
					self.log_stream = sys.stderr
			elif self.log_stream[1] == "file":
				try:
					self.log_stream = open(self.log_stream[0], 'a') 
				except:
					self.log_stream = None


	def set_debug_stream(self, strm):
		"""Set debug level."""
		self.log_stream = strm

	def set_debug(self, typ):
		"""Set behavior on errors and warnings.

		error = squash warnings
		warn = display warnings
		error_on_warning = raise exception for a warning rather than continuing
		"""
		if typ in ['error', 'warn', 'error_on_warning']:
			self.debug_level = typ
		else:
			raise ConfigurationError("Only levels are 'error', 'warn' and 'error_on_warning'")

	def maybe_warn(self, msg):
		"""warn method that respects debug_level property."""
		if self.log_stream and self.debug_level == "warn":
			self.log_stream.write(msg + "\n")
			try:	
				self.log_stream.flush()
			except:
				pass
		elif self.debug_level == "error_on_warning":
			# We don't know the type, just raise a MetadataError
			raise MetadataError(msg)

	def _is_uri(self, what):
		uri_schemes = ['urn:uuid:', 'tag:', 'data:', 'mailto:', 'info:', 'ftp:/', 'sftp:/'] 
		for u in uri_schemes:
			if what.startswith(u):
				return True
		return False

	def generate_id(self, what, auto_type=None):
		if auto_type == None:
			auto_type = self.auto_id_type

		if auto_type == "int":
			# increment and return
			self._auto_id_int += 1
			slug = self._auto_id_int
		elif auto_type == "int-per-segment":
			curr = self._auto_id_segments.get(what._uri_segment, -1)
			curr += 1
			self._auto_id_segments[what._uri_segment] = curr
			slug = self._auto_id_segments[what._uri_segment]
		elif auto_type == "int-per-type":
			t = type(what).__name__
			curr = self._auto_id_types.get(t, -1)
			curr += 1
			self._auto_id_types[t] = curr
			slug = self._auto_id_types[t]
		elif auto_type == "uuid":
			return "urn:uuid:%s" % uuid.uuid4()
		elif auto_type == "uuid-segment":
			slug = uuid.uuid4()
		else:
			raise ConfigurationError("Unknown auto-id type")

		if what.__class__._uri_segment:
			seg = what.__class__._uri_segment + "/"
		else:
			seg = ""

		return self.base_url + seg + str(slug)	

	def find_serializable(self, what):

		if not self.linked_art_boundaries:
			raise ConfigurationError("Factory doesn't have any boundaries to distinguish between entities")

		found = []
		props = what.list_my_props()
		for p in props:
			if p in ['id', 'type', '_label', 'content', 'value', 'begin_of_the_begin', 'end_of_the_end']:
				continue
			val = getattr(what, p)
			if isinstance(val, ExternalResource):
				val = [val]
			if type(val) is list:
				for v in val:
					if isinstance(v, ExternalResource):
						if not v in found and v.id and not v._linked_art_boundary_okay(what, p, v) and set(v.list_my_props()).difference(set(["_label", "id"])):
							found.append(v)
						downstream = self.find_serializable(v)
						for d in downstream:
							if not d in found:
								found.append(d)
		return found

	def toJSON(self, what, done=None):
		""" Serialize what, making sure of no infinite loops """
		if not done:
			done = {}
		if self.json_serializer == "fast":
			out = what._toJSON_fast(top=what, done=done)
		else:
			out = what._toJSON(top=what, done=done)
		return out

	def _collapse_json(self, text, collapse):
		js_indent = self.json_indent
		lines = text.splitlines()
		out = [lines[0]]
		while lines:
			l = lines.pop(0)
			indent = len(re.split('[^ \t\n\r]', l, 1)[0])
			if indent and l.rstrip()[-1] in ['[', '{']:
				curr = indent
				temp = []
				stemp = []
				while lines and curr <= indent:
					if temp and curr == indent:
						break
					temp.append(l[curr:])
					stemp.append(l.strip())
					l = lines.pop(0)
					indent = len(re.split('[^ \t\n\r]', l, 1)[0])
				temp.append(l[curr:])
				stemp.append(l.lstrip())

				short = " " * curr + ''.join(stemp)
				if len(short) < collapse:
					out.append(short)
				else:
					ntext = '\n'.join(temp)
					nout = self._collapse_json(ntext, collapse)					
					for no in nout:
						out.append(" " * curr + no)
			elif indent:
				out.append(l)
		out.append(l)
		return out
 
	def collapse_json(self, text, collapse):
		return '\n'.join(self._collapse_json(text, collapse))

	def _buildString(self, js, compact=True, collapse=0):
		"""Build string from JSON."""
		try:
			if compact:
				out = json.dumps(js, separators=(',',':'), ensure_ascii=False)
			else:
				out = json.dumps(js, indent=self.json_indent, ensure_ascii=False)
		except:
			out = ""
			self.maybe_warn("Can't decode %r" % js)
			raise
		if collapse:
			out = self.collapse_json(out, collapse)
		return out 		

	def toString(self, what, compact=True, collapse=0, done=None):
		"""Return JSON setialization as string."""
		if not done:
			done = {}
		js = self.toJSON(what, done=done)
		return self._buildString(js, compact, collapse)


	def toHtml(self, what, done=None):
		enc = JSONEncoder(indent=self.json_indent, ensure_ascii=False)
		js = self.toJSON(what, done=done)
		res = ['<pre><span>']
		lineno = 0
		for y in enc.iterencode(js):
			# split newlines
			if '\n' in y:
				sp = y.split('\n')
				x2 = []
				for z in sp[:-1]:
					if z:
						x2.append(z)
					x2.append('\n')
				if sp[-1]:
					x2.append(sp[-1])
			else:
				x2 = [y]

			for x in x2:
				if x[0] == '"':
					s = x.strip()[1:-1]
					clss = "str"
					if s.startswith('http') or self._is_uri(s):
						x = '"<a href="%s">%s</a>"' % (s, s)
					elif s in ["@context"]:
						clss = "str context"
					res.append('<span class="%s">%s</span>' % (clss, x))
				elif x[0].isdigit():
					res.append('<span class="int">%s</span>' % x)						
				elif x[0] in [":", ","]:
					res.append('<span class="sep">%s</span>' % x)	
				elif x[0]  == "{":
					res.append('<span class="bsep">%s</span><span class="block">' % x)
				elif x[0]  == "[":
					res.append('<span class="asep">%s</span><span class="ablock">' % x)
				elif x[0] == "}":
					res.append('</span><span class="bsep">%s</span>' % x)
				elif x[0] == "]":
					res.append('</span><span class="asep">%s</span>' % x)
				elif x == '\n':
					res.append('\n<span class="line" data-lno="%s"></span>' % lineno)
					lineno += 1
				elif x.isspace():
					res.append(x)
				else:
					print("*** Unhandled: %r" % x)
					res.append(x)
		res.append('</span></pre>')
		return ''.join(res)

	def toRDF(self, what, format="nq", bnode_prefix=""):
		# Format can be:  xml, pretty-xml, turtle, n3, nt, trix, trig, nquads
		# ttl = turtle; nq, n-quads == nquads

		# Need to ensure we generate the full form of predicates
		# otherwise context processing takes AGES
		# So set serializer to normal, and full_names to True
		srlz = self.json_serializer
		fn = self.full_names
		self.json_serializer = "normal"
		self.full_names = True
		js  = self.toJSON(what)
		# And put them back
		self.json_serializer = srlz
		self.full_names = fn

		# Substitute in a minimal context that defines only prefixes
		js['@context'] = min_context
		src = {'@id': js['@id'], '@graph':js}
		data = pyld_proc.to_rdf(src, options={"format": "application/nquads"})

		# Here replace all the bnodes with a unique id
		# This works so long as PyLD continues with incrementing integer bnode ids
		if bnode_prefix:
			data = re_bnodes.subn("_:b{bnode_prefix}_\\1 ".format(bnode_prefix=bnode_prefix), data)[0]
			data = re_bnodeo.subn("> _:b{bnode_prefix}_\\1 <".format(bnode_prefix=bnode_prefix), data)[0]				

		if format in ['nq', 'nquads', 'n-quads', 'application/nquads']:
			return data
		elif format in ['nt', 'ntriples', 'n-triples', 'application/ntriples']:
			data = re_quad.subn(" .", data)[0]
			return data
		else:
			# Need to pass over to rdflib
			g = CG()
			for (k,v) in min_context.items():
				if v[0] != "@":
					g.bind(k, v)
			g.parse(data=data, format="nquads")
			out = g.serialize(format=format)
			if type(out) == bytes:
				return out.decode('utf-8')
			else:
				return out

	def get_filename(self, whatid, extension=""):

		mdb = self.base_url
		if not whatid.startswith(mdb):
			raise ConfigurationError("The id of that object is not the base URI (factory.base_url) in the Factory")
		mdd = self.base_dir
		if not mdd:
			raise ConfigurationError("Directory (factory.base_dir) on Factory must be set to generate a file name")
		fp = whatid[len(mdb):]	

		# This will always be /, as it's from the URI
		bits = fp.split('/')
		fn = bits[-1]
		dirs = bits[:-1]

		if self.pair_tree_levels:
			for d in range(self.pair_tree_levels):
				if len(fn) > 2*d+1:
					dirs.append(fn[2*d:2*d+2])

		if len(dirs):
			mydir = os.path.join(mdd, *dirs)		
			try:
				os.makedirs(mydir)
			except OSError:
				pass

		# Allow passing in an override
		if extension:
			fn = fn + extension
		elif self.filename_extension:
			fn = fn + self.filename_extension
		dirs.append(fn)
		filename = os.path.join(mdd, *dirs)		
		return filename

	def toFile(self, what, compact=True, filename="", done=None, format=None, bnode_prefix="", extension=""):
		"""Write to local file.

		Creates directories as necessary based on URI, if filename is not supplied
		"""

		if not done:
			done = {}

		if not format:
			if not filename:
				filename = self.get_filename(what.id, extension=extension)
			js = self.toJSON(what, done=done)
			out = self._buildString(js, compact)
		else:
			if not filename:
				if extension:
					ext = extension
				elif format == "pretty-xml":
					ext = "xml"
				else:
					ext = format
				filename = self.get_filename(what.id, extension=ext)
			out = self.toRDF(what, format=format, bnode_prefix=bnode_prefix)

		fh = open(filename, 'w')
		try:
			fh.write(out)
		except:
			# Could be 2.x unicode issue
			fh.write(out.encode('utf-8'))
		fh.close()
		return out

	def production_mode(self, state=True):
		if state:
			self.cache_hierarchy()
			self.validate_profile = False
			self.validate_properties = False
			self.validate_range = False
			self.validate_multiplicity = False
			return True
		else:
			self.validate_profile = True
			self.validate_properties = True
			self.validate_range = True
			self.validate_multiplicity = True
			# no way to undo the hierarchy cache
			return False
			
	def cache_hierarchy(self):
		""" For each class, walk up the hierarchy and cache the terms """
		# This will work with the existing code, as it will find it in the first
		# test of props on classhier[0], the loop will just terminate straight away

		for c in self._all_classes.values():
			new_hash = c._all_properties.copy()
			if len(c._classhier) > 1:
				for p in c._classhier[1:]:
					for (prop, info) in p._all_properties.items():
						if not prop in new_hash:
							new_hash[prop] = info
			c._all_properties = new_hash


class ExternalResource(object):
	"""Base class for all resources, including external references"""
	
	_factory = None
	_uri_segment = ""
	id = ""
	_full_id = ""
	_all_properties = {}
	_type = ""
	_embed = True
	_property_name_map = {}
	_highlight = False
	_elide = False


	def __init__(self, ident=None):
		self._factory = factory
		if ident is not None:
			if self._factory._is_uri(ident):
				self.id = ident
			elif ident.startswith('http'):
				# Try to find prefixable term
				hashed = ident.rsplit('#', 1)
				if len(hashed) == 1:
					(pref, rest) = ident.rsplit('/', 1)
					pref += "/"
				else:
					(pref, rest) = hashed
					pref += "#"

				if pref in self._factory.prefixes_rev:
					self._full_id = ident
					ident = "%s:%s" % (self._factory.prefixes_rev[pref], rest)

				self.id = ident
			elif ident == "":
				# Allow explicit setting of empty string
				self.id = ""
			elif ident.startswith("auto "):
				# Allow override of auto setting for this resource
				# eg:  model.Person(ident="auto int") to assign an int, rather than the default
				self.id = factory.generate_id(self, auto_type=ident[5:].strip())
			else:
				# Allow for prefixed term that isn't ambiguously a URI
				curied = ident.split(':', 1)
				if len(curied) == 2 and curied[0] in self._factory.prefixes:
					self.id = ident
					self._full_id = self._factory.prefixes[curied[0]] + curied[1]	
				elif not self.__class__._uri_segment:
					self.id = factory.base_url + ident
				else:
					self.id = factory.base_url + self.__class__._uri_segment + "/" + ident
		elif factory.auto_assign_id:
			self.id = factory.generate_id(self)
		else:
			# Not auto assigning, and not submitted = blank node
			self.id = ""

	def _toJSON(self, done, top=None):
		if self._factory.elasticsearch_compatible:
			return {'id': self.id}
		else:
			return self.id

class BaseResource(ExternalResource):
	"""Base class for all resources with classes"""

	_integer_properties = []
	_object_properties = []
	_required_properties = []
	_warn_properties = []
	_classification = ""
	_classhier = []

	def __init__(self, ident=None, label="", value="", content="", **kw):
		"""Initialize BaseObject."""
		super(BaseResource, self).__init__(ident)

		if self._factory.validate_profile and hasattr(self, '_okayToUse'): 
			if not self._okayToUse:
				raise ProfileError("Class '%s' is configured to not be used" % self.__class__._type)
			elif self._okayToUse == 2:
				self._factory.maybe_warn("Class '%s' is configured to warn on use" % self.__class__._type)

		# Set label and value/content
		if label:
			self._label = label
		# this might raise an exception if value is not allowed on the object
		# but easier to do it in the main init than on many generated subclasses

		is_sym = isinstance(self, SymbolicObject)
		is_dim = isinstance(self, Dimension)

		if value and is_dim:
			self.value = value
		elif content and is_sym:
			self.content = content
		elif value and is_sym:
			self.content = value # not the right param, but not ambiguous
		elif content and is_dim:
			self.value = content # ditto
		elif value or content: 
			raise ProfileError("Class '%s' does not hold values" % self.__class__._type)
		# Custom post initialization function for autoconstructed classes
		self._post_init(**kw)

	def __dir__(self):
		d = dir(self.__class__)
		d.extend(self.list_all_props())
		return sorted(d)

	def __eq__(a, b):
		if id(a) == id(b):
			return True
		if not a or not b:
			return False
		if not isinstance(b, BaseResource) or not isinstance(a, BaseResource):
			return False
		ap = a.list_my_props()
		bp = b.list_my_props()
		if ap != bp:
			return False
		for p in ap:
			av = getattr(a, p)
			bv = getattr(b, p)
			if av != bv:
				return False
		return True

	@property
	def type(self):
		for c in self._classhier:
			if c._type:
				return c.__name__

	@type.setter
	def type(self, value):
		raise AttributeError("Must not set 'type' on resources directly")
	
	def set_context(self, value):	
		raise DataError("Must not set the JSON LD context directly", self)

	def _post_init(self, **kw):
		# Expect this to be overridden / replaced
		pass

	def __setattr__(self, which, value):
		"""Attribute setting magic for error checking and resource/literal handling."""

		if which[0] == "_" or not value:
			# _label goes through here, but it would below anyway, as it takes a Literal
			object.__setattr__(self, which, value)			
		else:
			# Allow per-class setters
			if hasattr(self, 'set_%s' % which):
				fn = getattr(self, 'set_%s' % which)
				return fn(value)

			if self._factory.validate_properties or self._factory.validate_profile or self._factory.validate_range:
				ok = self._check_prop(which, value)
			elif isinstance(value, ExternalResource):
				ok = 2
			else:
				ok = 1

			if ok == 2:
				self._set_magic_resource(which, value)
			else:			
				object.__setattr__(self, which, value)				
		 
	def _check_prop(self, which, value):
		val_props = self._factory.validate_properties
		val_profile = self._factory.validate_profile and getattr(self, '_validate_profile', True)
		val_range = self._factory.validate_range
		for c in self._classhier:
			if which in c._all_properties:
				pinfo = c._all_properties[which]

				if val_profile:
					okay = pinfo.profile_okay					
					rdf = pinfo.predicate
					if not okay:
						raise ProfileError("Property '%s' / '%s' is configured to not be used" % (which, rdf), self)
					elif okay == 2:
						self._factory.maybe_warn("Property '%s' / '%s' is configured to warn on use" % (which, rdf))

				if val_range:
					rng = pinfo.range
					if rng is str:					
						return 1
					elif type(value) is BaseResource:
						# Allow direct instances of base resource anywhere
						# this is an override for external URIs
						return 2
					elif isinstance(value, rng):
						return 2
					else:
						raise DataError("Can't set '%s' on resource of type '%s' to '%r'; must be '%s'" % (which, self._type, value, rng), self)
				# Found it, but not validating range and either okay or not validating profile
				return 1
		if val_props:
			raise DataError("Can't set unknown field '%s' on resource of type '%s'" % (which, self._type), self)
		else:
			# Not validating ANYTHING
			return 1

	def _check_reference(self, data):
		"""True if data is a resource or reference to a resource"""
		# "http://..."
		# {"@id": "http://..."}
		# or list of above
		if type(data) in STR_TYPES:
			return data.startswith('http')
		elif type(data) is dict:
			return 'id' in data
		elif isinstance(data, BaseResource):
			return True
		elif type(data) is list:
			for d in data:
				if type(d) in STR_TYPES and not d.startswith('http'):
					return False
				elif type(d) is dict and not 'id' in d:
					return False
			return True
		else:
			self._factory.maybe_warn("expecing a resource, got: %r" % (data))
			return True
				
	def _set_magic_resource(self, which, value, inversed=False):
		"""Set resource property.
		allow: string/object/dict, and magically generate list thereof
		"""

		if self._factory.materialize_inverses or self._factory.process_multiplicity or \
			self._factory.validate_multiplicity:
			inverse = None
			multiple = 1
			for c in self._classhier:
				if which in c._all_properties:
					v = c._all_properties[which]
					multiple = v.multiple_okay
					if v.inverse_property:
						inverse = v.inverse_property
					break

		try:
			current = getattr(self, which)
		except:
			current = None
		if not current:
			object.__setattr__(self, which, value)
		elif type(current) is list:
			# check value not in list
			if self._factory.multiple_instances_per_property == "error" and isinstance(value, BaseResource) and value in current:
				raise DataError("""Cannot add the same resource in the same property more than once:
change factory.multiple_instances_per_property to 'drop' or 'allow'""")
			current.append(value)
		else:
			if self._factory.validate_multiplicity and not multiple:
				raise ProfileError("Cannot append to %s on %s as multiplicity is 1" % (which, self._type))
			nvalue = [current, value]
			object.__setattr__(self, which, nvalue)

		if self._factory.materialize_inverses and not inversed and inverse:
			# set the backwards ref		
			value._set_magic_resource(inverse, self, True)
		if self._factory.process_multiplicity and type(current) is not list and multiple:
			object.__setattr__(self, which, [getattr(self, which)])


	def _toJSON(self, done, top=None):
		"""Serialize as JSON."""
		# If we're already in the graph, return our URI only
		# This should only be called from the factory!

		d = self.__dict__.copy()
		del d['_factory']

		# Can't pass in self as a param
		if top is None:
			top = self

		# id, type, _label is the default.
		if not factory.id_type_label and id(self) in done:
			if self._factory.elasticsearch_compatible:
				return {'id': self.id}
			else:
				return self.id

		# In case of local contexts, not at the root
		# Shouldn't ever happen, but worth testing for
		if 'context' in d:
			d['@context'] = d['context']
			del d['context']

		# Check mandatory properties
		if self._factory.validate_profile:
			for e in self._required_properties:
				if e not in d:
					raise RequirementError("Resource type '%s' requires '%s' to be set" % (self._type, e), self)

		debug = self._factory.debug_level
		if debug.find("warn") > -1:
			for e in self._warn_properties:
				if e not in d:
					msg = "Resource type '%s' should have '%s' set" % (self._type, e)
					self._factory.maybe_warn(msg)

		# Add back context at the top, if set
		if top is self and self._factory.context_uri: 
			d['@context'] = self._factory.context_uri

		if (self._factory.id_type_label and id(self) in done) or (top is not self and not self._embed):
			# limit to only id, type, label
			nd = {}
			nd['id'] = d['id']
			if self.type:
				nd['type'] = self.type
			try:
				nd['_label'] = d['_label']
			except:
				pass
			d = nd
		else:	
			# otherwise, we're about to serialize the resource completely
			done[id(self)] = 1			

		# Need to do in order now to get done correctly ordered
		if self._factory.order_json:
			KOH = self._factory.key_order_hash
			kodflt = self._factory.key_order_default
			kvs = sorted(d.items(), key=lambda x: KOH.get(x[0], kodflt))
		else:
			kvs = list(d.items())

		tbd = []
		for (k, v) in kvs:
			# some _foo might be carried through, eg _label or _comment
			k = self._property_name_map.get(k, k)
			if (not v and v is not False) or (k[0] == "_" and not k in self._factory.underscore_properties):
				del d[k]
			else:
				if isinstance(v, ExternalResource):
					if self._factory.linked_art_boundaries and \
						not self._linked_art_boundary_okay(top, k, v):
						# never follow, so just add to done
						done[id(v)] = 1
					else:
						tbd.append(id(v))
				elif type(v) is list:
					for ni in v:
						if isinstance(ni, ExternalResource):
							if self._factory.linked_art_boundaries and \
								not self._linked_art_boundary_okay(top, k, ni):
								# never follow, so just add to done
								done[id(ni)] = 1							
							else:
								tbd.append(id(ni))
					# For completeness should check list-of-datetime here too
				elif isinstance(v, datetime.datetime):
					# replace with string
					kvs[k] = v.strftime("%Y-%m-%dT%H:%M:%SZ")

		for t in tbd:
			if not t in done:
				done[t] = id(self)
			
		for (k,v) in kvs:
			nk = self._property_name_map.get(k, k)
			if nk != k:
				del d[k]
				d[nk] = v
				k = nk
			if v and (k[0] != "_" and not k in self._factory.underscore_properties):
				if isinstance(v, ExternalResource):
					if done[id(v)] == id(self):
						del done[id(v)]
					d[k] = v._toJSON(done=done, top=top)
				elif type(v) is list:
					newl = []
					uniq = set()
					for ni in v:
						if self._factory.multiple_instances_per_property == "drop":
							if id(ni) in uniq:
								continue
							else:
								uniq.add(id(ni))
						if isinstance(ni, ExternalResource):
							if done[id(ni)] == id(self):
								del done[id(ni)]
							newl.append(ni._toJSON(done=done, top=top))
						else:
							# A number or string
							newl.append(ni)
					d[k] = newl				

		if self._factory.full_names:
			nd = {}
			# @context gets ganked by this renaming
			# so add it back in first.
			if top is self:
				nd['@context'] = self._factory.context_uri

			for (k,v) in d.items():
				# look up the rdf predicate in _properties
				for c in self._classhier:
					if k in c._all_properties:
						nk = c._all_properties[k].predicate
						nd[nk] = v
						break

			# Ensure full version uses basic @type
			if "rdf:type" in nd:
				nd['@type'] = nd['rdf:type']
				del nd['rdf:type']

			# And type gets ganked for overlay classes (Painting)
			# plus for stupidity classes (DestructionActivity)
			# so add this back too
			if not "@type" in nd or not nd['@type']:
				# find class up that has a type and use its name
				for c in reversed(self._classhier):
					if c._type:
						nd['@type'] = c._type

			d = nd
			KOH = self._factory.full_key_order_hash
		else:
			# Use existing programmer-friendly names for classes too
			# find class up that has a type and use its name
			# almost certainly the first one
			if self.type:
				d['type'] = self.type

			if self._factory.pipe_scoped_contexts:
				# XXX TODO This should be configurable not hard coded
				if 'part' in d:
					# Calculate which part
					for c in reversed(self._classhier):
						if 'part' in c._all_properties:
							nk = c._all_properties['part'].predicate
							d['part|%s' % nk]  = d['part']
							del d['part']
							break
				if 'part_of' in d:
					# Calculate which part
					for c in reversed(self._classhier):
						if 'part_of' in c._all_properties:
							nk = c._all_properties['part_of'].predicate
							d['part_of|%s' % nk]  = d['part_of']
							del d['part_of']
							break

		if self._highlight and self._factory.allow_highlight:
			d['_highlight'] = True
		if self._elide and self._factory.allow_elide:
			d['_elide'] = True

		if self._factory.order_json:
			return OrderedDict(sorted(d.items(), key=lambda x: KOH.get(x[0], 1000)))
		else:
			return d

	def _toJSON_fast(self, done, top=None):
		"""Serialize as JSON."""
		# If we're already in the graph, return our URI only
		# This should only be called from the factory!

		# This should ONLY be used in Python3.6+
		# as it relies on a stable order to a regular dict
		# which was not the case before 3.6

		# id, type, _label is the default.
		if not self._factory.id_type_label and id(self) in done:
			return self.id

		# Can't pass in self as a param
		if top is None:
			top = self

		# Add back context at the top, if set
		result = {}
		if top is self and id(self) not in done and self._factory.context_uri: 
			result['@context'] = self._factory.context_uri

		if self.id:
			# Could be a bnode
			result['id'] = self.id
		if self.type:
			# Could be an external reference
			result['type'] = self.type
		try:
			result['_label'] = self._label
		except:
			pass

		# Need only minimal representation of self
		if (self._factory.id_type_label and id(self) in done) or (top is not self and not self._embed):
			# limit to only id, type, label
			return result
		else:	
			# otherwise, we're about to serialize the resource completely
			done[id(self)] = 1			

		d = self.__dict__.copy()
		del d['_factory']
		del d['id']

		# Need to do in order now to get done correctly ordered
		kvs = list(d.items())
		if self._factory.order_json:
			KOH = self._factory.key_order_hash
			kodflt = self._factory.key_order_default
			# in place sort is 2% faster than sorted
			# rather than over an iterable
			# and sorting is 10% of srlz cost
			# and use setdefault to amortize cost of default
			# over multiple calls to the factory's hash
			kvs.sort(key=lambda x: KOH.setdefault(x[0], kodflt))
			
		# tbd vs done is to ensure that in a DAG rather than a tree
		# that children of the current node are given priority for
		# full serialization, rather than deeper descendent nodes
		# that would otherwise be processed first during the (recursive)
		# depth-first traversal.

		# This doesn't catch the pattern A-B-C-D / A-E-D,
		# (D will be under B-C, not under E) as B is processed completely
		# before E.

		# Note: Reimplemented this in a single depth-first loop
		# and it was less than 2% faster. The 2 loops are not that expensive

		tbd = []
		for (k, v) in kvs:
			if k[0] == "_" and not k in self._factory.underscore_properties:
				continue
			k = self._property_name_map.get(k, k)
			if isinstance(v, ExternalResource):
				if self._factory.linked_art_boundaries and \
					not self._linked_art_boundary_okay(top, k, v):
					# never follow, so just add to done
					done[id(v)] = 1
				else:
					tbd.append(id(v))
			elif type(v) is list:
				for ni in v:
					if isinstance(ni, ExternalResource):
						if self._factory.linked_art_boundaries and \
							not self._linked_art_boundary_okay(top, k, ni):
							# never follow, so just add to done
							done[id(ni)] = 1							
						else:
							tbd.append(id(ni))

		for t in tbd:
			if not t in done:
				done[t] = id(self)
			
		# This is already sorted if needed
		for (k,v) in kvs:
			if k[0] == "_" and not k in self._factory.underscore_properties:
				continue
			k = self._property_name_map.get(k, k)
			if not v:
				pass
			elif isinstance(v, ExternalResource):
				if done[id(v)] == id(self):
					del done[id(v)]
				result[k] = v._toJSON_fast(done=done, top=top)
			elif type(v) is list:
				newl = []
				uniq = set()
				for ni in v:
					if self._factory.multiple_instances_per_property == "drop":
						if id(ni) in uniq:
							continue
						else:
							uniq.add(id(ni))
					if isinstance(ni, ExternalResource):
						if done[id(ni)] == id(self):
							del done[id(ni)]
						newl.append(ni._toJSON_fast(done=done, top=top))
					else:
						# A number or string
						newl.append(ni)
				result[k] = newl
			elif isinstance(v, datetime.datetime):
				result[k] = v.strftime("%Y-%m-%dT%H:%M:%SZ")
			else:
				result[k] = v
		return result

	def _linked_art_boundary_okay(self, top, prop, value):
		# Return false to say do not cross this boundary
		# Without replacement, just return True always
		return True

	def list_all_props(self, filter=None, okay=None):
		props = []
		for c in self._classhier:		
			for k,v in c._all_properties.items():
				if not k in props and \
					(not okay or (okay and v.profile_okay)) and \
					(filter is None or isinstance(filter, v.range) or \
						filter is v.range):
					props.append(k)
		props.sort()
		return props

	def list_all_props_with_range(self, filter=None, okay=None):
		props = {}
		for c in self._classhier:		
			for k,v in c._all_properties.items():
				if not k in props and \
					(not okay or (okay and v.profile_okay)) and \
					(filter is None or isinstance(filter, v.range) or \
						filter is v.range):
					props[k] = v.range
		return props

	def list_my_props(self, filter=None):
		d = self.__dict__.copy()		
		props = []
		for (k,v) in d.items():
			if k[0] != "_" or k in self._factory.underscore_properties:
				# real property
				if filter:
					if isinstance(v, filter):
						props.append(k)
					elif isinstance(v, list):
						for i in v:
							if isinstance(i, filter):
								props.append(k)
								break
				else:
					props.append(k)		
		return props

	def allows_multiple(self, propName):
		""" Does propName allow multiple values on this class """
		for c in self._classhier:
			if propName in c._all_properties:
				v = c._all_properties[propName]
				return bool(v.multiple_okay)
		raise DataError("Cannot set '%s' on '%s'" % (propName, self.__class__.__name__))

	def clone(self, minimal=False):
		""" Make a shallow copy of self. If minimal flag is set, only return id, type and _label """
		new = self.__class__(ident=self.id)
		if minimal:
			if self._label:
				new._label = self._label
		else:
			for p in self.list_my_props():
				curr = getattr(self, p)
				if type(curr) == list:
					for v in curr:
						setattr(new, p, v)
				else:
					setattr(new, p, curr)
		return new

# Ontology / Profile manipulation

def override_okay(clss, propName):
	""" set particular property on the class to be okay to use """
	pinfo = clss._all_properties.get(propName, None)
	if pinfo:
		npinfo = PropInfo(pinfo.property, pinfo.predicate, pinfo.range,
			pinfo.inverse_property, pinfo.inverse_predicate, 
			pinfo.multiple_okay, 1)
		clss._all_properties[propName] = npinfo
	else:
		raise DataError("%s does not have a %s property to allow" % 
			(clss.__name__, propName))


# Ensure everything can have id, type, label and description
BaseResource._properties = {'id': {"rdf": "@id", "range": str, "okayToUse": 1}, 
	'type': {"rdf": "rdf:type", "range": str, "rangeStr": "rdfs:Class", "okayToUse": 1}, 
	'_label': {"rdf": "rdfs:label", "range": str, "rangeStr": "xsd:string", "okayToUse": 1}
}
BaseResource._all_properties = {
	'id': PropInfo('id', '@id', str, None, None, 0, 1),
	'type': PropInfo('type', 'rdf:type', str, None, None, 0, 1),
	'_label': PropInfo('_label', 'rdfs:label', str, None, None, 0, 1)
}
BaseResource._classhier = (BaseResource, ExternalResource)

def process_tsv(fn):
	fh = codecs.open(fn, 'r', 'utf-8')
	lines = fh.readlines()[1:] # chomp header line
	fh.close()
	vocabData = {"rdf:Resource": 
		{"props": [], "label": "Resource", "className": "Resource", 
		"subs":[], "desc": "", "class": BaseResource, "okay": 1}}

	for l in lines:
		l = l[:-1] # chomp
		info= l.split('\t')
		name = info[0]	
		if info[1] == "class":
			data = {"subOf": info[5], "label": info[3], 'className': info[2],
				"desc": info[4], "class": None, "props": [], "subs": [], "okay": info[6]}
			vocabData[name] = data
		else:
			# property
			data = {"name": name, "subOf": info[5], "label": info[3], "propName": info[2],
			"desc": info[4], "range": info[7], "inverse": info[8], "okay": info[10], "multiple": info[11]}
			try:
				what = vocabData[info[6]]
			except:
				print("Failed to find class for %s given %s" % (data, info[6]))
				raise
			what["props"].append(data)

			koh = int(info[9])
			if koh != KEY_ORDER_DEFAULT:
				factory.key_order_hash[data['propName']] = koh
				factory.full_key_order_hash[data['name']] = koh

	# invert subclass hierarchy
	for k, v in vocabData.items():
		if k != "rdf:Resource":		
			sub = v['subOf']
			# | separated list
			for s in sub.split('|'):
				if s:
					try:
						vocabData[s]['subs'].append(k)
					except:
						pass
	return vocabData

# Build class heirarchy recursively
def build_class(crmName, parent, vocabData):

	data = vocabData[crmName]
	name = str(data['className'])

	# check to see if we already exist
	# nb globals() here is only this module
	if name in globals():
		c = globals()[name]
		try:	
			c.__bases__ += (parent,)
		except:
			print("MRO FAILURE: %r --> %r + %r" % (c, c.__bases__, parent))
			raise
		return

	c = type(name, (parent,), {'__doc__': data['desc']})
	globals()[name] = c
	data['class'] = c
	if not ":" in crmName:
		c._type = "crm:%s" % crmName
	else:
		c._type = crmName
	c._uri_segment = name
	c._properties = {}
	c._all_properties = {}
	c._okayToUse = int(data['okay'])

	factory._all_classes[name] = c

	# Set up real properties
	for p in data['props']:
		ccname, pvalue = _make_property_def(p)
		c._properties[ccname] = pvalue

	# Build subclasses
	for s in data['subs']:
		build_class(s, c, vocabData)

def _make_property_def(p):
	name = p['name']
	if not ":" in name:
		name = "crm:%s" % name
	rng = p['range']
	ccname = p['propName']
	if p['inverse']:
		i = p['inverse']
		if i[0] == "P":
			invRdf = "crm:%s" % i
		else:
			invRdf = i
	else:
		invRdf = ""
	okay = p['okay']
	if not okay:
		okay = '1'
	okay = int(okay)
	mult = p['multiple']
	if not mult:
		mult = '0'
	mult = int(mult)
	# can't guarantee all classes have been built at this stage :(
	return (ccname, {"rdf": name, 
		"rangeStr": rng,
		"inverseRdf": invRdf,
		"okayToUse": okay,
		"multiple": mult})

def build_classes(fn=None, topClass=None):
	# Default to building our core dataset

	if not fn:
		dd = os.path.join(os.path.dirname(__file__), 'data')
		fn = os.path.join(dd, 'crm_vocab.tsv')
		topClass = 'E1_CRM_Entity'

	vocabData = process_tsv(fn)

	# Everything can have an id, a type, a label, a description
	build_class(topClass, BaseResource, vocabData)

	# And reset definitions now we have class objects
	for v in vocabData.values():
		c = v['class']
		if c is BaseResource:
			continue
		for (name, value) in c._properties.items():
			# recreate with namedtuple
			rngs = value.get('rangeStr', None)
			if not rngs:
				# Precomputed ones like rdf:type
				continue

			inverse = None
			rngd = vocabData.get(value['rangeStr'], None)
			if rngs in ['rdfs:Literal', 'xsd:dateTime', 'xsd:string', 'rdfs:Class']:
					rng = str 
			elif not rngd:
				raise ConfigurationError("Failed to get range for %s property %s - %s" % (c, name, rngs))
			else:
				rng = rngd['class']
				# and add inverse prop name from range
				if rng == None:
					# Uh-oh!
					raise ConfigurationError("Class %s did not get built, range of %s" % (value['rangeStr'], name))
				for (ik, iv) in rng._properties.items():
					if iv['inverseRdf'] == value['rdf']:
						inverse = ik
						break

			c._all_properties[name] = PropInfo(name,
				value['rdf'], rng, inverse,
				value.get('inverseRdf', None),
				value.get('multiple', 1),
				value.get('okayToUse', 0)
			)

	# set all of the classhiers
	for v in vocabData.values():
		c = v['class']
		c._classhier = inspect.getmro(c)[:-1]
		try:
			del c._properties
		except:
			# Never had it set?
			pass

# XXX This should be invoked rather than inline so the module can be loaded
# and a different context used. But for now ...
# Build the factory first, so properties can be added to key_order
factory = CromulentFactory("http://lod.example.org/museum/", context="https://linked.art/ns/v1/linked-art.json")
build_classes()
# Need to then configure the boundary classes after they're created
