import sys
import re


def lex(characters, token_exps):
    pos = 0
    tokens = []
    lines = characters.splitlines()
    schar = ""
    for line in lines:
        schar += line.strip() + "\n"
    while pos < len(schar):
        match = None
        for token_expr in token_exps:
            pattern, tag = token_expr
            regex = re.compile(pattern)
            match = regex.match(schar, pos)
            if match:
                text = match.group(0).strip()
                if tag:
                    token = (text, tag)
                    tokens.append(token)
                break
        if not match:
            sys.stderr.write('Illegal Character: %s\n' % schar[pos])
            sys.exit(1)
        else:
            pos = match.end(0)
    return tokens
