"""
This algorithm trains a model of numerical tensors dynamically adjusted through indexing, creating clusters of tensors sorted by token type.
This enables training and inference in batch, resulting in an extraordinary increase in speed for both training and inference, even on modest hardware without a GPU.

Copying, editing, sharing, or reverse-engineering this or other algorithms developed by Sapiens Technology®️ is not allowed without prior authorization from the company.
Public posts and comments about the operation of our technologies are also strictly prohibited. Failure to comply with these guidelines will result in legal actions taken by our legal team.
"""
# --------------------------> A SAPIENS TECHNOLOGY®️ PRODUCTION) <--------------------------
class SAPITensors:
	def __init__(self, show_errors=True, display_error_point=False):
		try:
			self.__show_errors = bool(show_errors) if type(show_errors) in (bool, int, float) else True
			self.__display_error_point = bool(display_error_point) if type(display_error_point) in (bool, int, float) else False
			try:
				from warnings import simplefilter, filterwarnings
				from logging import getLogger, ERROR, disable, CRITICAL
				from os import environ
				from dotenv import load_dotenv
				simplefilter('ignore')
				filterwarnings('ignore')
				filterwarnings('ignore', category=UserWarning, module='torch.distributed')
				getLogger('torch.distributed.elastic.multiprocessing.redirects').setLevel(ERROR)
				environ['DISABLE_MODEL_SOURCE_CHECK'] = 'True'
				load_dotenv()
				disable(CRITICAL)
			except: pass
			from traceback import print_exc
			self.__print_exc = print_exc
			self.precision = None
			self.end_tag = ''
			self.delay = 0.02
			self.generalization = False
			self.__Tensors = {}
			self.__files_path = ''
			self.__extraction_directory = ''
			self.__model_path = ''
			self.__disregard = []
			self.__OUTPUT = {'answer': '', 'answer_for_files': '', 'files': [], 'sources': [], 'javascript_charts': [], 'artifacts': [], 'next_token': '', 'str_cost': '0.0000000000', 'tool_call': []}
		except Exception as error:
			try:
				if self.__show_errors:
					error_message = 'ERROR in SAPITensors.__init__: '+str(error)
					print(error_message)
					try: self.__print_exc() if self.__display_error_point else None
					except: pass
			except: pass
	def __find_file(self, data_path='', extension='', disregard=[]):
		try:
			data_path = str(data_path).strip()
			extension = str(extension).strip()
			disregard = list(disregard) if type(disregard) in (tuple, list) else [str(disregard).strip()]
			if not data_path: return data_path
			from pathlib import Path
			target = Path(data_path)
			if target.is_file(): return data_path
			if target.is_dir():
				matches = [str(file).strip() for file in sorted(target.rglob(f'*{extension}')) if file.is_file() and not any(str(file).strip().endswith(str(suffix).strip()) for suffix in disregard)]
				if matches: return str(matches[0]).strip()
			return data_path
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SAPITensors.__find_file: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return data_path
	def __string_to_dictionary_or_json(self, string=''):
		try:
			dictionary_or_json = {}
			original_string = string
			string = str(string).strip()
			try:
				from json import loads
				try: dictionary_or_json = loads(string)
				except: dictionary_or_json = loads(original_string)
			except:
				from ast import literal_eval
				try: dictionary_or_json = literal_eval(string)
				except: dictionary_or_json = literal_eval(original_string)
			return dictionary_or_json
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SAPITensors.__string_to_dictionary_or_json: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return string
	def __get_input_output(self, json_dictionary={}):
		try:
			return_dictionary = {'input': '', 'output': '', 'file_path': ''}
			if not json_dictionary or type(json_dictionary) != dict: return return_dictionary
			input_output, _input, _output, _file_path = json_dictionary, '', '', ''
			if input_output and type(input_output) == dict:
				json_keys = list(input_output.keys())
				if 'input' in json_keys: _input = str(input_output.get('input', '')).strip()
				elif 'Input' in json_keys: _input = str(input_output.get('Input', '')).strip()
				elif 'INPUT' in json_keys: _input = str(input_output.get('INPUT', '')).strip()
				elif 'question' in json_keys: _input = str(input_output.get('question', '')).strip()
				elif 'Question' in json_keys: _input = str(input_output.get('Question', '')).strip()
				elif 'QUESTION' in json_keys: _input = str(input_output.get('QUESTION', '')).strip()
				elif 'prompt' in json_keys: _input = str(input_output.get('prompt', '')).strip()
				elif 'Prompt' in json_keys: _input = str(input_output.get('Prompt', '')).strip()
				elif 'PROMPT' in json_keys: _input = str(input_output.get('PROMPT', '')).strip()
				if 'output' in json_keys: _output = str(input_output.get('output', '')).strip()
				elif 'Output' in json_keys: _output = str(input_output.get('Output', '')).strip()
				elif 'OUTPUT' in json_keys: _output = str(input_output.get('OUTPUT', '')).strip()
				elif 'answer' in json_keys: _output = str(input_output.get('answer', '')).strip()
				elif 'Answer' in json_keys: _output = str(input_output.get('Answer', '')).strip()
				elif 'ANSWER' in json_keys: _output = str(input_output.get('ANSWER', '')).strip()
				elif 'response' in json_keys: _output = str(input_output.get('response', '')).strip()
				elif 'Response' in json_keys: _output = str(input_output.get('Response', '')).strip()
				elif 'RESPONSE' in json_keys: _output = str(input_output.get('RESPONSE', '')).strip()
				if 'file_path' in json_keys: _file_path = str(input_output.get('file_path', '')).strip()
				elif 'File_path' in json_keys: _file_path = str(input_output.get('File_path', '')).strip()
				elif 'FILE_PATH' in json_keys: _file_path = str(input_output.get('FILE_PATH', '')).strip()
				if not _input: _input = str(input_output[json_keys[0]]).strip()
				if not _output: _output = str(input_output[json_keys[1]]).strip()
				return_dictionary['input'], return_dictionary['output'] = _input, _output
				if _file_path: return_dictionary['file_path'] = _file_path
			return return_dictionary
		except Exception as error:
			try:
				if self.__show_errors:
					error_message = 'ERROR in SAPITensors.__get_input_output: '+str(error)
					print(error_message)
					try: self.__print_exc() if self.__display_error_point else None
					except: pass
			except: pass
			return {'input': '', 'output': '', 'file_path': ''}
	def __normalize_keywords(self, input_text=''):
		try:
			input_text = str(input_text).strip().lower()
			result = input_text
			from unicodedata import normalize, combining
			from re import sub
			normalized = normalize('NFD', input_text)
			without_accents = ''.join(character for character in normalized if not combining(character))
			only_letters_numbers_spaces = sub(r'[^a-z\s]', '', without_accents)
			filtered_words = [word for word in only_letters_numbers_spaces.split() if len(word)>1]
			result = ' '.join(filtered_words)
			return result
		except Exception as error:
			try:
				if self.__show_errors:
					error_message = 'ERROR in SAPITensors.__normalize_keywords: '+str(error)
					print(error_message)
					try: self.__print_exc() if self.__display_error_point else None
					except: pass
			except: pass
			return input_text
	def __normalize_string(self, input_text=''):
		try:
			input_text = str(input_text).lower().strip()
			from unicodedata import normalize, category
			from re import sub, search
			input_text = ''.join(character for character in normalize('NFD', input_text) if category(character)!='Mn')
			def _is_neighbor_number(index=0, text='', allow_space=True, check_before=True):
				if check_before:
					if index == 0: return False
					before_part = text[:index]
					if allow_space:
						match_before = search(r'\S', before_part[::-1])
						if not match_before or not match_before.group(0).isdigit(): return False
					else:
						if not before_part[-1].isdigit(): return False
				if index == len(text)-1: return False
				after_part = text[index+1:]
				if allow_space:
					match_after = search(r'\S', after_part)
					if not match_after or not match_after.group(0).isdigit(): return False
				else:
					if not after_part[0].isdigit(): return False
				return True
			chars_to_check = list(input_text)
			for index, char in enumerate(chars_to_check):
				if char in '.,':
					if not _is_neighbor_number(index, input_text, allow_space=False): chars_to_check[index] = ' '
				elif char in '+-*/%^ˆ':
					if not _is_neighbor_number(index, input_text, allow_space=True): chars_to_check[index] = ' '
				elif char=='$':
					if not _is_neighbor_number(index, input_text, allow_space=True, check_before=False): chars_to_check[index] = ' '
			input_text = ''.join(chars_to_check)
			input_text = sub(r'[^a-z0-9\s.,+\-*/%^ˆ$]', ' ', input_text)
			input_text = sub(r'\s{2,}', ' ', input_text)
			return str(input_text).strip()
		except Exception as error:
			try:
				if self.__show_errors:
					error_message = 'ERROR in SAPITensors.__normalize_string: '+str(error)
					print(error_message)
					try: self.__print_exc() if self.__display_error_point else None
					except: pass
			except: pass
			return input_text
	def __embedding_to_key(self, embedding=[]):
		try:
			embedding_key = ''
			embedding = list(embedding) if type(embedding) in (tuple, list) else []
			embedding_key = str(embedding).replace('[', '').replace(']', '').replace(' ', '').replace(',', '_').strip()
			return embedding_key
		except Exception as error:
			try:
				if self.__show_errors:
					error_message = 'ERROR in SAPITensors.__embedding_to_key: '+str(error)
					print(error_message)
					try: self.__print_exc() if self.__display_error_point else None
					except: pass
			except: pass
			return ''
	def __key_to_embedding(self, key=''):
		try:
			embedding = []
			key = str(key).strip()
			from ast import literal_eval
			embedding = literal_eval(f"[{key.replace('_', ', ')}]".strip())
			return embedding
		except Exception as error:
			try:
				if self.__show_errors:
					error_message = 'ERROR in SAPITensors.__key_to_embedding: '+str(error)
					print(error_message)
					try: self.__print_exc() if self.__display_error_point else None
					except: pass
			except: pass
			return []
	def __get_system_temp_directory(self):
		try:
			system_temp_directory = ''
			try:
				from tempfile import gettempdir
				system_temp_directory = gettempdir()
			except:
				from os import getenv
				system_temp_directory = getenv('TMPDIR') or getenv('TEMP') or getenv('TMP') or getenv('TMPDIR', '/tmp') or ''
			from os.path import join
			def ___generate_hash():
				from time import time_ns
				from hashlib import sha256
				timestamp_nanoseconds = str(time_ns())
				return sha256(timestamp_nanoseconds.encode('utf-8')).hexdigest()
			system_temp_directory = join(system_temp_directory, ___generate_hash())
			return system_temp_directory
		except Exception as error:
			try:
				if self.__show_errors:
					error_message = 'ERROR in SAPITensors.__get_system_temp_directory: '+str(error)
					print(error_message)
					try: self.__print_exc() if self.__display_error_point else None
					except: pass
			except: pass
			return ''
	def __find_matching_file_optimized(self, files_path='', attention_embeddings_string=''):
		try:
			files_path = str(files_path).strip()
			attention_embeddings_string = str(attention_embeddings_string).strip()
			if not files_path or not attention_embeddings_string: return attention_embeddings_string
			from os import path, scandir
			if not path.exists(files_path): return attention_embeddings_string
			target_numbers = set(attention_embeddings_string.split('_'))
			found_step_2 = None; found_step_3 = None; found_step_4 = None
			found_step_5 = None; found_step_6 = None; found_step_7 = None
			max_tokens_count = 0
			with scandir(files_path) as iterator:
				for entry in iterator:
					if not entry.name.endswith('.json'): continue
					file_name = path.splitext(entry.name)[0]
					if file_name == attention_embeddings_string: return file_name
					if not found_step_2 and file_name.startswith(attention_embeddings_string): found_step_2 = file_name
					if not found_step_3 and attention_embeddings_string.startswith(file_name): found_step_3 = file_name
					if attention_embeddings_string in file_name: found_step_4 = file_name
					if file_name in attention_embeddings_string: found_step_5 = file_name
					if not found_step_6 and target_numbers.issubset(set(file_name.split('_'))): found_step_6 = file_name
					if not found_step_2 and not found_step_3 and not found_step_4 and not found_step_5 and not found_step_6:
						tokens_count = 0
						for target_number in target_numbers:
							if target_number in file_name: tokens_count += 1
						if tokens_count > max_tokens_count: max_tokens_count, found_step_7 = tokens_count, file_name
			return found_step_2 or found_step_3 or found_step_4 or found_step_5 or found_step_6 or found_step_7 or attention_embeddings_string
		except Exception as error:
			try:
				if self.__show_errors:
					error_message = 'ERROR in SAPITensors.__find_matching_file_optimized: '+str(error)
					print(error_message)
					try: self.__print_exc() if self.__display_error_point else None
					except: pass
			except: pass
			return attention_embeddings_string
	def __load_tensors_dictionary(self, tensors_path=''):
		try:
			tensors = {}
			if not tensors_path: return tensors
			from os import path
			from json import load
			if not path.exists(tensors_path): return {}
			with open(tensors_path, 'r', encoding='utf-8') as file_object: tensors = load(file_object)
			return tensors
		except Exception as error:
			try:
				if self.__show_errors:
					error_message = 'ERROR in SAPITensors.__load_tensors_dictionary: '+str(error)
					print(error_message)
					try: self.__print_exc() if self.__display_error_point else None
					except: pass
			except: pass
			return {}
	def __save_tensors_dictionary(self, tensors={}, tensors_path=''):
		try:
			tensors_path = str(tensors_path).strip()
			if not tensors or not tensors_path: return False
			from pathlib import Path
			from json import dump
			path_object = Path(tensors_path)
			path_object.parent.mkdir(parents=True, exist_ok=True)
			with path_object.open('w', encoding='utf-8') as file_object: dump(tensors, file_object, ensure_ascii=False, indent='\t')
			return path_object.exists()
		except Exception as error:
			try:
				if self.__show_errors:
					error_message = 'ERROR in SAPITensors.__save_tensors_dictionary: '+str(error)
					print(error_message)
					try: self.__print_exc() if self.__display_error_point else None
					except: pass
			except: pass
			return False
	def __add_fit(self, prompt='', answer='', file_path=''):
		try:
			added = False
			prompt = str(prompt).strip()
			answer = str(answer).strip()
			file_path = str(file_path).strip()
			if not prompt or not answer: return added
			if file_path:
				from INFINITE_CONTEXT_WINDOW import InfiniteContextWindow as SAPIENS_INFINITE_CONTEXT_WINDOW
				interpreted_file = SAPIENS_INFINITE_CONTEXT_WINDOW(show_errors=self.__show_errors, display_error_point=self.__display_error_point).interpreter(file_path=file_path)
				if interpreted_file: prompt = f'{interpreted_file}\n\n{prompt}'.strip()
			keywords_text = self.__normalize_keywords(input_text=prompt)
			from sapiens_attention import SapiensAttention
			sapiens_attention, maximum_length = SapiensAttention(), 5
			attention_words = sapiens_attention.get_attention_words(text=keywords_text, maximum_length=maximum_length)
			attention_words = sorted(list(set(attention_words))) if attention_words else []
			if not attention_words or len(attention_words) < maximum_length:
				attention_words += sapiens_attention.get_textual_elements(text=keywords_text, maximum_length=maximum_length)
				attention_words = sorted(list(set(attention_words)))
			if not attention_words: return added
			from sapiens_tokenizer import SapiensTokenizer
			sapiens_tokenizer = SapiensTokenizer()
			attention_embeddings_string = self.__embedding_to_key(sapiens_tokenizer.to_encode(str(chr(32).join(attention_words)).strip()))
			if not attention_embeddings_string: return added
			from os.path import join
			files_path = self.__files_path
			last_number = attention_embeddings_string.split('_')[-1]
			files_path = join(files_path, last_number)
			attention_embeddings_string = self.__find_matching_file_optimized(files_path=files_path, attention_embeddings_string=attention_embeddings_string)
			if not attention_embeddings_string: return added
			file_name = f'{attention_embeddings_string}.json'
			tensors_path = join(files_path, file_name)
			self.__Tensors = self.__load_tensors_dictionary(tensors_path=tensors_path)
			prompt = self.__normalize_string(input_text=prompt)
			if not prompt: return added
			input_embedding = sapiens_tokenizer.to_encode(prompt)
			output_embedding = sapiens_tokenizer.to_encode(answer)
			input_embedding_key = self.__embedding_to_key(embedding=input_embedding)
			input_embedding_list = self.__Tensors.get(input_embedding_key, [])
			if not input_embedding_list: self.__Tensors[input_embedding_key] = [output_embedding]
			elif output_embedding not in input_embedding_list: self.__Tensors[input_embedding_key].append(output_embedding)
			added = self.__save_tensors_dictionary(tensors=self.__Tensors, tensors_path=tensors_path)
			return added
		except Exception as error:
			try:
				if self.__show_errors:
					error_message = 'ERROR in SAPITensors.__add_fit: '+str(error)
					print(error_message)
					try: self.__print_exc() if self.__display_error_point else None
					except: pass
			except: pass
			return False
	def fit(self, dataset_path='', progress=True):
		try:
			adjusted_model = False
			dataset_path = str(dataset_path).strip()
			progress = bool(progress) if type(progress) in (bool, int, float) else True
			if not dataset_path: return adjusted_model
			notice0 = f'The path to the dataset "{dataset_path}" does not exist.'
			notice1 = 'The file must be a JSON with a "data" key containing an array of objects with the keys "input" and "output".'
			notice2 = 'The provided dataset is empty.'
			from os import path
			if not path.exists(dataset_path):
				if self.__show_errors: print(notice0)
				return adjusted_model
			dataset_path = self.__find_file(data_path=dataset_path, extension='.json', disregard=[])
			def _read_remote_file(remote_path=''):
				remote_path = str(remote_path).strip()
				from urllib.request import urlopen
				try:
					from os import environ
					from certifi import where
					environ['SSL_CERT_FILE'] = where()
					from logging import getLogger, ERROR
					getLogger('urlopen').setLevel(ERROR)
				except: pass
				remote_stream = urlopen(remote_path)
				content = remote_stream.read().decode('utf-8')
				return str(content).strip()
			is_json, string_content = dataset_path.lower().strip().endswith('.json'), ''
			if dataset_path.startswith(('https://', 'http://')): string_content = _read_remote_file(remote_path=dataset_path)
			else:
				with open(dataset_path, 'r', encoding='utf-8') as file_object: string_content = str(file_object.read()).strip()
			if is_json:
				fit_structure, data = {}, []
				if string_content: fit_structure = self.__string_to_dictionary_or_json(string=string_content)
				if fit_structure:
					if type(fit_structure) == dict:
						json_keys = list(fit_structure.keys())
						if 'data' in json_keys: data = list(fit_structure.get('data', []))
						else:
							data_key = json_keys[0]
							data = list(fit_structure.get(data_key, []))
					elif type(fit_structure) in (tuple, list): data = fit_structure
					if data:
						self.__files_path = self.__get_system_temp_directory()
						from tqdm import tqdm
						total_length = len(data)
						with tqdm(total=total_length, unit='item', disable=not progress) as progress_bar:
							for input_output in data:
								_input, _output, _file_path = '', '', ''
								input_output = self.__get_input_output(json_dictionary=input_output)
								_input, _output, _file_path = str(input_output.get('input', '')).strip(), str(input_output.get('output', '')).strip(), str(input_output.get('file_path', '')).strip()
								if _input and _output: adjusted_model = self.__add_fit(prompt=_input, answer=_output, file_path=_file_path)
								if progress:
									progress_bar.set_description('tensors')
									progress_bar.update(1)
						adjusted_model = total_length > 0 and adjusted_model
					elif self.__show_errors: print(notice2)
				elif self.__show_errors: print(notice1)
			elif self.__show_errors: print(notice1)
			configuration = {'precision': self.precision, 'end_tag': self.end_tag, 'delay': self.delay, 'generalization': self.generalization}
			from os.path import join
			adjusted_model = self.__save_tensors_dictionary(tensors=configuration, tensors_path=join(self.__files_path, 'configuration.json'))
			return adjusted_model
		except Exception as error:
			try:
				if self.__show_errors:
					error_message = 'ERROR in SAPITensors.fit: '+str(error)
					print(error_message)
					try: self.__print_exc() if self.__display_error_point else None
					except: pass
			except: pass
			return False
	def saveModel(self, model_path='', progress=True):
		try:
			saved_model = False
			model_path = str(model_path).strip()
			progress = bool(progress) if type(progress) in (bool, int, float) else True
			if not model_path: model_path = 'model.sapitensors'
			if not model_path.endswith('.sapitensors'): model_path += '.sapitensors'
			from pathlib import Path
			path_object = Path(model_path)
			path_object.parent.mkdir(parents=True, exist_ok=True)
			def _resolve_sapitensors_path(model_path=''):
				from re import compile
				path_object = Path(model_path)
				if path_object.suffix != '.sapitensors': return model_path
				parent_directory = path_object.parent
				stem_name = path_object.stem
				pattern = compile(r'^'+stem_name+r'(?:-(\d+))?\.sapitensors$')
				files_list = [file_object for file_object in parent_directory.glob(stem_name+'*.sapitensors') if pattern.match(file_object.name)]
				n_sapitensors = len(files_list)
				if n_sapitensors == 0: return model_path
				if n_sapitensors == 1:
					existing_file = files_list[0]
					new_existing_name = existing_file.with_name(stem_name+'-1.sapitensors')
					existing_file.rename(new_existing_name)
					return str(path_object.with_name(stem_name+'-2.sapitensors'))
				next_index = n_sapitensors+1
				width = len(str(next_index))
				if width > 1:
					for file_object in files_list:
						match_object = pattern.match(file_object.name)
						number_part = match_object.group(1)
						if number_part is None: new_name = stem_name+'-'+str(1).zfill(width)+'.sapitensors'
						else: new_name = stem_name+'-'+str(int(number_part)).zfill(width)+'.sapitensors'
						new_path = file_object.with_name(new_name)
						if file_object.name != new_name: file_object.rename(new_path)
				return str(path_object.with_name(stem_name+'-'+str(next_index).zfill(width)+'.sapitensors'))
			model_path = _resolve_sapitensors_path(model_path=model_path)
			from utilities_nlp import UtilitiesNLP as SapiensUtilities
			sapiens_utilities = SapiensUtilities(show_errors=self.__show_errors, display_error_point=self.__display_error_point)
			saved_model = sapiens_utilities.compactModel(self.__files_path, model_path, progress)
			if saved_model: saved_model = sapiens_utilities.deleteDirectory(self.__files_path)
			return saved_model
		except Exception as error:
			try:
				if self.__show_errors:
					error_message = 'ERROR in SAPITensors.saveModel: '+str(error)
					print(error_message)
					try: self.__print_exc() if self.__display_error_point else None
					except: pass
			except: pass
			return False
	def loadModel(self, model_path='', progress=True):
		try:
			loaded_model = False
			model_path = str(model_path).strip()
			progress = bool(progress) if type(progress) in (bool, int, float) else True
			model_path = self.__find_file(data_path=model_path, extension='.sapitensors', disregard=[])
			if not model_path: model_path = 'model.sapitensors'
			if not model_path.endswith('.sapitensors'): model_path += '.sapitensors'
			from os import path
			if not path.exists(model_path):
				if self.__show_errors: print(f'The path to the model "{model_path}" does not exist.')
				return loaded_model
			self.__model_path = model_path
			from utilities_nlp import UtilitiesNLP as SapiensUtilities
			sapiens_utilities = SapiensUtilities(show_errors=self.__show_errors, display_error_point=self.__display_error_point)
			def _convert_file_name_to_base64(file_path=''):
				from os.path import basename
				from base64 import b64encode
				file_name = basename(file_path)
				base64_string = b64encode(file_name.encode('utf-8')).decode('utf-8')
				return base64_string
			from tempfile import gettempdir
			from os.path import join
			system_temp_directory = gettempdir()
			self.__extraction_directory = join(system_temp_directory, _convert_file_name_to_base64(file_path=model_path))
			if not path.exists(self.__extraction_directory): loaded_model = sapiens_utilities.decompactModel(file_path=model_path, model_path=self.__extraction_directory, progress=progress)
			loaded_model = path.exists(self.__extraction_directory)
			configuration = self.__load_tensors_dictionary(tensors_path=join(self.__extraction_directory, 'configuration.json'))
			if configuration:
				self.precision = configuration.get('precision', None)
				if self.precision: self.precision = min(1.0, max(0.0, float(self.precision)))
				self.end_tag = str(configuration.get('end_tag', ''))
				self.delay = max(0.0, float(configuration.get('delay', 0.02)))
				self.generalization = bool(configuration.get('generalization', False))
			return loaded_model
		except Exception as error:
			try:
				if self.__show_errors:
					error_message = 'ERROR in SAPITensors.loadModel: '+str(error)
					print(error_message)
					try: self.__print_exc() if self.__display_error_point else None
					except: pass
			except: pass
			return False
	def inference(self, messages=[], stream=True, precision=None):
		try:
			inference_result = self.__OUTPUT
			messages = list(messages) if type(messages) in (tuple, list) else []
			stream = bool(stream) if type(stream) in (bool, int, float) else True
			if precision is not None: precision = min(1.0, max(0.0, float(precision))) if type(precision) in (int, float) else None
			def _get_default_return(inference_result={}):
				if stream: yield inference_result
				else: return inference_result
			if not messages: return _get_default_return(inference_result)
			if precision is None: precision = min(1.0, max(0.0, float(self.precision))) if type(self.precision) in (int, float) else 1.0
			prompt, file_path = '', ''
			last_message = messages[-1]
			if not last_message or type(last_message) != dict: return _get_default_return(inference_result)
			role = str(last_message.get('role', '')).lower().strip()
			if role != 'user': return _get_default_return(inference_result)
			prompt = original_prompt = str(last_message.get('content', '')).strip()
			if not prompt: return _get_default_return(inference_result)
			if 'file_path' in last_message: file_path = last_message.get('file_path', '')
			elif 'file_paths' in last_message: file_path = last_message.get('file_paths', '')
			if type(file_path) in (tuple, list): file_path = file_path[-1] if file_path else ''
			file_path = str(file_path).strip()
			if file_path:
				from INFINITE_CONTEXT_WINDOW import InfiniteContextWindow as SAPIENS_INFINITE_CONTEXT_WINDOW
				interpreted_file = SAPIENS_INFINITE_CONTEXT_WINDOW(show_errors=self.__show_errors, display_error_point=self.__display_error_point).interpreter(file_path=file_path)
				if interpreted_file: prompt = f'{interpreted_file}\n\n{prompt}'.strip()
			keywords_text = self.__normalize_keywords(input_text=prompt)
			from sapiens_attention import SapiensAttention
			sapiens_attention, maximum_length = SapiensAttention(), 5
			attention_words = sapiens_attention.get_attention_words(text=keywords_text, maximum_length=maximum_length)
			attention_words = sorted(list(set(attention_words))) if attention_words else []
			if not attention_words or len(attention_words) < maximum_length:
				attention_words += sapiens_attention.get_textual_elements(text=keywords_text, maximum_length=maximum_length)
				attention_words = sorted(list(set(attention_words)))
			if not attention_words: return _get_default_return(inference_result)
			from sapiens_tokenizer import SapiensTokenizer
			sapiens_tokenizer = SapiensTokenizer()
			attention_embeddings_string = self.__embedding_to_key(sapiens_tokenizer.to_encode(str(chr(32).join(attention_words)).strip()))
			if not attention_embeddings_string: return _get_default_return(inference_result)
			from os.path import join
			files_path = self.__extraction_directory
			last_number = attention_embeddings_string.split('_')[-1]
			files_path = join(files_path, last_number)
			attention_embeddings_string = self.__find_matching_file_optimized(files_path=files_path, attention_embeddings_string=attention_embeddings_string)
			if not attention_embeddings_string: return _get_default_return(inference_result)
			file_name = f'{attention_embeddings_string}.json'
			tensors_path = join(files_path, file_name)
			self.__Tensors = self.__load_tensors_dictionary(tensors_path=tensors_path)
			def _get_random_file_name_without_extension(directory_path=''):
				if not directory_path: return ''
				from os import path, listdir
				if not path.isdir(directory_path): return ''
				file_list = [file_name for file_name in listdir(directory_path) if path.isfile(path.join(directory_path, file_name))]
				if not file_list: return ''
				from random import choice
				from os.path import splitext
				return splitext(choice(file_list))[0]
			def _get_alternative_inference():
				from pathlib import Path
				path_object = Path(self.__model_path)
				data_path = path_object.parent
				file_name = path_object.name
				self.__disregard.append(file_name)
				model_path = self.__find_file(data_path=data_path, extension='.sapitensors', disregard=self.__disregard)
				if model_path != self.__model_path:
					loaded_model = self.loadModel(model_path=model_path, progress=False)
					if loaded_model: return self.inference(messages=messages, stream=stream, precision=precision)
				return				
			if not self.__Tensors:
				alternative_inference = _get_alternative_inference()
				if alternative_inference: return alternative_inference
			if not self.__Tensors:
				_attention_embeddings_string = self.__embedding_to_key(sapiens_tokenizer.to_encode(keywords_text))
				files_path = self.__extraction_directory
				attention_list = _attention_embeddings_string.split('_')
				_last_number = attention_list[-1]
				if _last_number == last_number:
					try:
						from random import randint
						_last_number = attention_list[randint(0, len(attention_list)-1)]
					except: _last_number = attention_list[0]
				last_number = _last_number
				files_path = join(files_path, last_number)
				_attention_embeddings_string = _get_random_file_name_without_extension(directory_path=files_path)
				file_name = f'{_attention_embeddings_string}.json'
				tensors_path = join(files_path, file_name)
				self.__Tensors = self.__load_tensors_dictionary(tensors_path=tensors_path)
			if precision <= 0.0 and not self.__Tensors:
				files_path = self.__extraction_directory
				last_numbers = attention_embeddings_string.split('_')
				for last_number in last_numbers:
					files_path = join(files_path, last_number)
					attention_embeddings_string = _get_random_file_name_without_extension(directory_path=files_path)
					tensors_path = join(files_path, f'{attention_embeddings_string}.json')
					self.__Tensors = self.__load_tensors_dictionary(tensors_path=tensors_path)
					if self.__Tensors: break
			prompt = self.__normalize_string(input_text=prompt)
			if not prompt: return _get_default_return(inference_result)
			input_embedding = sapiens_tokenizer.to_encode(prompt)
			embeddings_vector = list(self.__Tensors.keys())
			input_embedding_length = max(1, len(input_embedding))
			def _is_subsequence(smaller_list=[], larger_list=[]):
				if not smaller_list or not larger_list: return False
				smaller_length, larger_length = len(smaller_list), len(larger_list)
				if smaller_length > larger_length: return False
				limit = larger_length-smaller_length+1
				for index in range(limit):
					match = True
					for offset in range(smaller_length):
						if larger_list[index+offset] != smaller_list[offset]:
							match = False
							break
					if match: return True
				return False
			def _get_minimum_absolute_difference(number=0, numeric_list=[]):
				if not number or not numeric_list: return 10**10
				minimum_difference = float('inf')
				for element in numeric_list:
					absolute_difference = abs(number - element)
					if absolute_difference < minimum_difference: minimum_difference = absolute_difference
				return minimum_difference
			def _choose_random_list(list_of_lists=[]):
				if not list_of_lists: return []
				from random import choice
				return list_of_lists[0] if len(list_of_lists) == 1 else choice(list_of_lists)
			def _get_generalization(prompt='', original_input='', original_output=''):
				if (not prompt or not original_input or not original_output) or prompt == original_input: return original_output
				embedding = self.__key_to_embedding(key=original_input)
				original_input = sapiens_tokenizer.to_decode(embedding)
				from sapiens_generalization import SapiensGeneralization
				sapiens_generalization = SapiensGeneralization()
				generalized_output = sapiens_generalization.generalization(prompt=prompt, original_input=original_input, original_output=original_output, direction=0)
				generalized_output = sapiens_generalization.generalization(prompt=prompt, original_input=original_input, original_output=generalized_output, direction=1)
				generalized_output = sapiens_generalization.generalization(prompt=prompt, original_input=original_input, original_output=generalized_output, direction=2)
				return generalized_output if generalized_output else original_output
			inference_string, original_input = '', ''
			if precision <= 0.0 and not self.__Tensors: inference_string = original_prompt
			from statistics import mean
			best_difference, best_key = float('inf'), ''
			maximum_probability = float('-inf')
			for embedding_string in embeddings_vector:
				embedding = self.__key_to_embedding(key=embedding_string)
				embedding_length = len(embedding)
				length_half = embedding_length//2
				if embedding == input_embedding or (input_embedding_length > length_half and _is_subsequence(smaller_list=input_embedding, larger_list=embedding)):
					inference_string = sapiens_tokenizer.to_decode(_choose_random_list(list_of_lists=self.__Tensors.get(embedding_string, [])))
					if self.generalization: original_input = embedding_string
					break
				else:
					if precision <= 0.0:
						difference_list = []
						for element_x in input_embedding:
							minimum_difference = _get_minimum_absolute_difference(number=element_x, numeric_list=embedding)
							difference_list.append(minimum_difference)
						total_difference = mean(difference_list)
						if total_difference < best_difference: best_difference, best_key = total_difference, embedding_string
					else:
						counting_hits = 0
						for element_x in input_embedding:
							if element_x in embedding: counting_hits += 1
						if input_embedding_length < length_half: probability = ((counting_hits/input_embedding_length)+(min(input_embedding_length, embedding_length)/max(input_embedding_length, embedding_length)))/2
						else: probability = counting_hits/input_embedding_length
						if precision <= 0.1 and probability > maximum_probability: maximum_probability, best_key = probability, embedding_string
						elif probability >= precision:
							inference_string = sapiens_tokenizer.to_decode(_choose_random_list(list_of_lists=self.__Tensors.get(embedding_string, [])))
							if self.generalization: original_input = embedding_string
							break
			if (best_key and best_difference < float('inf')) or (best_key and maximum_probability > float('-inf')):
				inference_string = sapiens_tokenizer.to_decode(_choose_random_list(list_of_lists=self.__Tensors.get(best_key, [])))
				if self.generalization: original_input = best_key
			if not inference_string:
				alternative_inference = _get_alternative_inference()
				if alternative_inference: return alternative_inference
			if inference_string:
				self.__disregard = []
				if self.generalization: inference_string = _get_generalization(prompt=prompt, original_input=original_input, original_output=inference_string)
				if stream:
					from time import sleep
					self.delay = max(0.0, float(self.delay)) if type(self.delay) in (int, float) else 0.02
					def _get_tokens(inference_string=''):
						tokens = inference_string.split(chr(32))
						tokens_length = len(tokens)
						for count, token in enumerate(tokens, start=1):
							if token:
								if count <= 1: token = token.lstrip()+chr(32)
								elif count < tokens_length: token = token+chr(32)
								inference_result['next_token'] = token
								inference_result['answer'] += token
								yield inference_result
							sleep(self.delay)
					return _get_tokens(inference_string=inference_string)
				else:
					inference_result['answer'] = inference_string
					return inference_result
			if self.end_tag:
				inference_result['next_token'] = self.end_tag
				inference_result['answer'] = self.end_tag
			if stream:
				def _get_inference(inference_result={}): yield inference_result
				return _get_inference(inference_result=inference_result)
			else: return inference_result
		except Exception as error:
			try:
				if self.__show_errors:
					error_message = 'ERROR in SAPITensors.inference: '+str(error)
					print(error_message)
					try: self.__print_exc() if self.__display_error_point else None
					except: pass
			except: pass
			return self.__OUTPUT
	def close(self):
		try:
			from os.path import exists
			from shutil import rmtree
			if len(self.__files_path.strip()) > 0 and exists(self.__files_path) and self.__files_path != self.__model_path: rmtree(self.__files_path)
			if len(self.__extraction_directory.strip()) > 0 and exists(self.__extraction_directory) and self.__extraction_directory != self.__model_path: rmtree(self.__extraction_directory)
			del self.__Tensors
			self.__Tensors = {}
			return not exists(self.__files_path) and not exists(self.__extraction_directory)
		except Exception as error:
			try:
				if self.__show_errors:
					error_message = 'ERROR in SAPITensors.close: '+str(error)
					print(error_message)
					try: self.__print_exc() if self.__display_error_point else None
					except: pass
			except: pass
			return False
"""
This algorithm trains a model of numerical tensors dynamically adjusted through indexing, creating clusters of tensors sorted by token type.
This enables training and inference in batch, resulting in an extraordinary increase in speed for both training and inference, even on modest hardware without a GPU.

Copying, editing, sharing, or reverse-engineering this or other algorithms developed by Sapiens Technology®️ is not allowed without prior authorization from the company.
Public posts and comments about the operation of our technologies are also strictly prohibited. Failure to comply with these guidelines will result in legal actions taken by our legal team.
"""
# --------------------------> A SAPIENS TECHNOLOGY®️ PRODUCTION) <--------------------------
