
import re

class VroxLexer:
    
    def __init__(self):
        self.rules = {}

    def add_rule(self, name, regex):
        self.rules[name] = re.compile(regex)  # Precompile
        return self

       
    
    def tokenize(self, textPath, outputPath,callback):
        # Open input and output files
        with open(textPath, 'r', encoding='utf-8') as input_file, open(outputPath, 'w', encoding='utf-8') as output_file:
            i = 0
            length = 0
            
            while True:
                # Read the next chunk from the input file
                chunk = input_file.read(1024)  # Read in chunks of 1024 bytes or adjust as needed
                if not chunk:
                    break  # End of file reached
                
                length = len(chunk)
                i = 0

                # Process the chunk
                while i < length:
                    # Skip whitespace
                    if chunk[i].isspace():
                        i += 1
                        continue
                    
                    # Try matching each rule from the current position
                    matched = False
                    for rule_name, rule_info in self.rules.items():
                        match = rule_info.match(chunk, i)
                        if match:
                            matched = True
                            # Write the token directly to the output file
                            output_file.write(f"{rule_name}: {match.group(0)}\n")
                            i += len(match.group(0))
                            break
                    
                    # If no match found, increment by 1 (to handle invalid characters)
                    if not matched:
                        i += 1
                        
            callback(outputPath)
        


