###############################################################
# Name:			 Tokenizer.py
# Purpose:	 Abstract tokenizer
# Author:		 Luca Allulli (webmaster@roma21.it)
# Created:	 2009-01-31
# Copyright: Luca Allulli (https://www.skeed.it/songpress)
# License:	 GNU GPL v2
##############################################################

import re


class Token(object):
	def __init__(self, token, start, end, content):
		object.__init__(self)
		self.token = token
		self.start = start
		self.end = end
		self.content = content

	def __str__(self):
		return '%s, %d, %d, %s' % (str(self.token), self.start, self.end, self.content)


class TokenType(object):
	def __init__(self, r, name):
		object.__init__(self)
		self.r = re.compile(r)
		self.name = name

	def __str__(self):
		return self.name


class Tokenizer(object):
	def __init__(self, l):
		object.__init__(self)
		self.line = l
		self.pos = 0
		self.state = self.normal
		self.prevToken = None
		self.repeatToken = False

	def __iter__(self):
		return self

	def next(self):
		return self.__next__()

	def __next__(self):
		if self.repeatToken:
			self.repeatToken = False
			return self.prevToken
		if self.pos >= len(self.line):
			#End of line
			raise StopIteration
		for t in self.transition[self.state]:
			# print("Trying: %s" % (t[0],))
			m = t[0].r.match(self.line, self.pos)
			if m != None:
				# print("Matched!")
				break
		if m == None:
			return None
		tok = Token(t[0], self.pos, m.end(0), m.group(0))
		##print (type(m.group(0)), m.group(0))
		self.state = t[1]
		##print("Found %s" % (tok,))
		self.pos = m.end(0)
		self.prevToken = tok
		return tok

	def Repeat(self, repeat = True):
		self.repeatToken = repeat
