# This is a Transformer algorithm of the BERT type (Bidirectional Encoder Representations from Transformers) developed by Sapiens Technology®️
# with the purpose of training, saving, fine-tuning, loading, and inferring bidirectional language models.
# The code includes a wide range of pre-established functions that make it easier to develop conversational Artificial Intelligence models of any scale.
# --------------------------> A SAPIENS TECHNOLOGY®️ PRODUCTION) <--------------------------
class SapiensBERT:
    def __init__(self, show_errors=True, display_error_point=False):
        try:
            self.__show_errors = bool(show_errors) if type(show_errors) in (bool, int, float) else True
            self.__display_error_point = bool(display_error_point) if type(display_error_point) in (bool, int, float) else False
            try:
                from warnings import filterwarnings
                from logging import getLogger, ERROR, disable, CRITICAL
                from os import environ
                from dotenv import load_dotenv
                filterwarnings('ignore')
                filterwarnings('ignore', category=UserWarning, module='torch.distributed')
                getLogger('torch.distributed.elastic.multiprocessing.redirects').setLevel(ERROR)
                environ['DISABLE_MODEL_SOURCE_CHECK'] = 'True'
                load_dotenv()
                disable(CRITICAL)
            except: pass
            from traceback import print_exc
            self.__print_exc = print_exc
            self.embedding_dim = 48
            self.block_size = 64
            self.batch_size = 4
            self.number_heads = 4
            self.number_layers = 2
            self.dropout = 0.0
            self.learning_rate = 1e-3
            self.eval_interval = 50
            self.epochs = 2000
            self.string = ''
            self.precision = 0.99
            self.tokenizer = 'bert'
            self.context_window = 48
            self.end_tag = '<|end|>'
            self.mask_token_tag = '<|mask|>'
            self.mask_token_id = None
            self.validate = 0.0
            self.max_tokens = 500
            self.temperature = 0.5
            self.top_k = 0
            self.top_p = 1.0
            self.delay = 0.01
            self.device = None
            from torch import cuda, device as sapiens_device, backends
            from torch.utils.data import Dataset, DataLoader
            from torch.nn import Module, functional as Function, utils
            from torch import nn as artificial_neural_network
            from torch import tensor, no_grad, int64, multinomial, cat, topk, sort, cumsum, zeros_like, save, load
            from tiktoken import get_encoding
            from json import load as json_load
            from torch import optim, rand, randint, arange
            from tqdm import tqdm
            from os import path as os_path, makedirs as os_makedirs
            def _tpu_is_available():
                from os import environ, path
                if 'COLAB_TPU_ADDR' in environ:
                    if 'TPU_NAME' in environ or 'TPU_WORKER_ID' in environ or 'PJRT_DEVICE' in environ: return True
                if path.exists('/sys/bus/pci/devices') and any('tpu' in name.lower() for name in environ.keys()): return True
                if path.exists('/dev/vfio'): return True
                return False
            self.__xm = None
            if not self.device:
                if cuda.is_available(): self.device, self.__xm = sapiens_device('cuda'), None
                elif backends.mps.is_available(): self.device, self.__xm = sapiens_device('mps'), None
                elif _tpu_is_available():
                    try:
                        import torch_xla.core.xla_model as xm
                        self.__xm = xm
                    except: self.__xm = None
                    if self.__xm is not None: self.device = self.__xm.xla_device()
                else: self.device, self.__xm = sapiens_device('cpu'), None
            from sys import setrecursionlimit
            setrecursionlimit(self.epochs+10)
            self.__Dataset = Dataset
            self.__Module = Module
            self.__neural_network = artificial_neural_network
            self.__cuda = cuda
            self.__tensor = tensor
            self.__no_grad = no_grad
            self.__Function = Function
            self.__int64 = int64
            self.__multinomial = multinomial
            self.__cat = cat
            self.__topk = topk
            self.__sort = sort
            self.__cumsum = cumsum
            self.__zeros_like = zeros_like
            self.__rand = rand
            self.__randint = randint
            self.__arange = arange
            self.__get_encoding = get_encoding
            self.__json_load = json_load
            self.__DataLoader = DataLoader
            self.__optim = optim
            self.__utils = utils
            self.__tqdm = tqdm
            self.__os_path = os_path
            self.__os_makedirs = os_makedirs
            self.__save = save
            self.__load = load
            self.__model = None
            self.__encode = None
            self.__decode = None
            self.__end_tag = None
            self.__string = ''
            self.__vocab_size = 0
            self.__char_to_idx = {}
            self.__idx_to_char = {}
            self.__optimizer = None
            self.__train = False
            self.parameters_number = 0
            class TextDataset(self.__Dataset):
                def __init__(self, data=[], block_size=128):
                    self.data = data
                    self.block_size = block_size
                    self.stride = 1 
                    self.indices = range(0, len(data) - block_size, self.stride)
                    if len(self.indices) == 0:  self.indices = [0]
                def __len__(self): return len(self.indices)
                def __getitem__(self, idx=0):
                    if idx >= len(self.indices): idx = 0
                    start_idx = self.indices[idx]
                    if len(self.data) < self.block_size:                        
                        if isinstance(self.data, list):
                            sequence_list = self.data + [0] * (self.block_size - len(self.data))
                            sequence_tensor = self.outer._SapiensBERT__tensor(sequence_list, dtype=self.outer._SapiensBERT__int64)
                        else:
                            padding_len = self.block_size - len(self.data)
                            padding = self.outer._SapiensBERT__zeros_like(self.data)[:padding_len]
                            padding = self.outer._SapiensBERT__tensor([0]*padding_len, dtype=self.outer._SapiensBERT__int64, device=self.data.device)
                            sequence_tensor = self.outer._SapiensBERT__cat((self.data, padding))
                        sequence_tensor = sequence_tensor[:self.block_size]
                        return sequence_tensor, sequence_tensor.clone()
                    else:
                        sequence = self.data[start_idx : start_idx + self.block_size]
                        return sequence.clone(), sequence.clone()
            class Transformer(self.__Module):
                def __init__(self, outer=None, vocab_size=0, embedding_dim=128, number_heads=8, number_layers=8, dropout=0.1, block_size=128):
                    super().__init__()
                    self.outer = outer
                    self.positional_encoding = outer._SapiensBERT__neural_network.Embedding(block_size, embedding_dim)
                    self.dropout = outer._SapiensBERT__neural_network.Dropout(dropout)
                    self.input_embedding = outer._SapiensBERT__neural_network.Embedding(vocab_size, embedding_dim)
                    encoder_layers = outer._SapiensBERT__neural_network.TransformerEncoderLayer(d_model=embedding_dim, nhead=number_heads, dropout=dropout, batch_first=True)
                    self.transformer_encoder = outer._SapiensBERT__neural_network.TransformerEncoder(encoder_layers, num_layers=number_layers)
                    self.output_function = outer._SapiensBERT__neural_network.Linear(embedding_dim, vocab_size)
                    self.block_size = block_size
                def forward(self, input_tensor=[]):
                    batch_size, sequence_length = input_tensor.size()
                    positions = self.outer._SapiensBERT__arange(sequence_length, device=input_tensor.device).unsqueeze(0)
                    embedding = self.input_embedding(input_tensor) + self.positional_encoding(positions)
                    embedding = self.dropout(embedding)
                    encoded_output = self.transformer_encoder(embedding)
                    return self.output_function(encoded_output)
            self.__TextDatasets = TextDataset
            self.__Transformers = Transformer
        except Exception as error:
            try:
                if self.__show_errors:
                    error_message = 'ERROR in SapiensBERT.__init__: '+str(error)
                    print(error_message)
                    try: self.__print_exc() if self.__display_error_point else None
                    except: pass
            except: pass
    def __apply_mlm_mask(self, inputs=[], probability=0.15):
        try:
            device = inputs.device
            batch_size, seq_len = inputs.shape
            final_labels = inputs.clone()
            final_labels.fill_(-100)
            is_gen_mask = self.__rand(batch_size, device=device) < 0.3
            is_gen_mask_expanded = is_gen_mask.unsqueeze(1).expand_as(inputs)
            probs = self.__rand(inputs.shape, device=device)
            mlm_mask = (probs < probability) & (inputs != 0) & (~is_gen_mask_expanded)
            final_labels[mlm_mask] = inputs[mlm_mask]
            replace_probs = self.__rand(inputs.shape, device=device)
            mask_indices = mlm_mask & (replace_probs < 0.8)
            inputs[mask_indices] = self.mask_token_id
            random_indices = mlm_mask & (replace_probs >= 0.8) & (replace_probs < 0.9)
            if random_indices.any():
                random_words = self.__randint(0, self.__vocab_size, inputs.shape, device=device)
                inputs[random_indices] = random_words[random_indices]        
            if is_gen_mask.any():
                split_indices = self.__randint(1, seq_len, (batch_size, 1), device=device)
                seq_indices = self.__arange(seq_len, device=device).unsqueeze(0)
                target_mask = (seq_indices == split_indices) & is_gen_mask_expanded
                future_mask = (seq_indices > split_indices) & is_gen_mask_expanded
                final_labels[target_mask] = inputs[target_mask]
                inputs[target_mask] = self.mask_token_id
                inputs[future_mask] = 0
            return inputs, final_labels
        except Exception as error:
            try:
                if self.__show_errors:
                    error_message = 'ERROR in SapiensBERT.__apply_mlm_mask: '+str(error)
                    print(error_message)
                    try: self.__print_exc() if self.__display_error_point else None
                    except: pass
            except: pass
            return inputs, inputs
    def __is_xla(self):
        try:
            device = str(self.device).lower().strip()
            return self.__xm is not None and ('xla' in device or 'tpu' in device)
        except Exception as error:
            try:
                if self.__show_errors:
                    error_message = 'ERROR in SapiensBERT.__is_xla: '+str(error)
                    print(error_message)
                    try: self.__print_exc() if self.__display_error_point else None
                    except: pass
            except: pass
            return False 
    def __compute_loss(self, loader=[]):
        try:
            self.__model.eval()
            total_loss = 0
            with self.__no_grad():
                for input_batch, _ in loader:
                    input_batch = input_batch.to(self.device)
                    masked_input, masked_labels = self.__apply_mlm_mask(input_batch)
                    logits = self.__model(masked_input)
                    loss = self.__Function.cross_entropy(logits.view(-1, self.__vocab_size), masked_labels.view(-1))
                    total_loss += loss.item()
            return total_loss / len(loader)
        except Exception as error:
            try:
                if self.__show_errors:
                    error_message = 'ERROR in SapiensBERT.__compute_loss: '+str(error)
                    print(error_message)
                    try: self.__print_exc() if self.__display_error_point else None
                    except: pass
            except: pass
            return 0
    def __format_params(self, number_params=0):
        try:
            if number_params < 1_000: return f'{number_params}U'
            elif number_params < 1_000_000: return f'{number_params // 1_000}K'
            elif number_params < 1_000_000_000: return f'{number_params // 1_000_000}M'
            else: return f'{number_params // 1_000_000_000}B'
        except Exception as error:
            try:
                if self.__show_errors:
                    error_message = 'ERROR in SapiensBERT.__format_params: '+str(error)
                    print(error_message)
                    try: self.__print_exc() if self.__display_error_point else None
                    except: pass
            except: pass
            return f'{number_params}U'
    def __get_found_end_tag(self, decoded_token='', decoded_tokens='', limits=[]):
        try:
            if self.__end_tag is None: return False
            for limit in [''] + list(limits) + [' ']:
                if decoded_token.endswith(limit + self.__end_tag) or decoded_tokens.endswith(limit + self.__end_tag): return True
            return False
        except Exception as error:
            try:
                if self.__show_errors:
                    error_message = 'ERROR in SapiensBERT.__get_found_end_tag: '+str(error)
                    print(error_message)
                    try: self.__print_exc() if self.__display_error_point else None
                    except: pass
            except: pass
            return False
    def __generate_tokens_bert_legacy(self, prompt='', max_tokens=500, temperature=1.0, top_k=0, top_p=1.0):
        try:
            self.__model.eval()
            encoded_prompt = list(self.__encode(prompt))
            current_ids = self.__tensor(encoded_prompt, dtype=self.__int64).unsqueeze(0).to(self.device)
            all_tokens = list(encoded_prompt)
            printed_length = len(self.__decode(all_tokens))
            decoded_tokens_accum = ''
            buffer_tokens = ''
            def _get_token(token='', position=0):
                def __remove_first_punctuation(text=''):
                    if not text: return text
                    punctuation_symbols = '.;!¡?¿,'
                    if text[0] in punctuation_symbols: return text[1:]
                    return text
                if position <= 1:
                    token = token.lstrip()
                    return __remove_first_punctuation(text=token)
                elif position >= max_tokens: return token.rstrip()
                return token
            with self.__no_grad():
                for index in range(max_tokens):
                    position = index+1
                    mask_tensor = self.__tensor([[self.mask_token_id]], dtype=self.__int64).to(self.device)
                    input_with_mask = self.__cat((current_ids, mask_tensor), dim=1)
                    if input_with_mask.size(1) > self.block_size: input_with_mask = input_with_mask[:, -self.block_size:]
                    logits = self.__model(input_with_mask)
                    mask_logits = logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
                    if top_k > 0:
                        v, _ = self.__topk(mask_logits, min(top_k, mask_logits.size(-1)))
                        mask_logits[mask_logits < v[:, [-1]]] = float('-inf')
                    if top_p < 1.0:
                        sorted_logits, sorted_indices = self.__sort(mask_logits, descending=True)
                        cumulative_probs = self.__cumsum(self.__Function.softmax(sorted_logits, dim=-1), dim=-1)
                        sorted_indices_to_remove = cumulative_probs > top_p
                        sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
                        sorted_indices_to_remove[..., 0] = 0
                        indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
                        mask_logits[indices_to_remove] = float('-inf')
                    probs = self.__Function.softmax(mask_logits, dim=-1)
                    if temperature == 0: predicted_token_id = self.__topk(probs, 1)[1]
                    else: predicted_token_id = self.__multinomial(probs, 1)
                    current_ids = self.__cat((current_ids, predicted_token_id), dim=1)
                    token_val = predicted_token_id.item()
                    if token_val == self.mask_token_id: break
                    all_tokens.append(token_val)
                    full_text = self.__decode(all_tokens)
                    if full_text.endswith('\ufffd'): continue
                    decoded_token = full_text[printed_length:]
                    printed_length = len(full_text)
                    decoded_tokens_accum += decoded_token
                    if self.end_tag:
                        buffer_tokens += decoded_token
                        if self.end_tag in buffer_tokens: break
                        match = False
                        for i in range(len(buffer_tokens)):
                            suffix = buffer_tokens[i:]
                            if self.end_tag.startswith(suffix):
                                match = True
                                if i > 0:
                                    to_yield = buffer_tokens[:i]
                                    buffer_tokens = suffix
                                    for char in to_yield: yield _get_token(char, position)
                                break
                        if not match:
                            for char in buffer_tokens: yield _get_token(char, position)
                            buffer_tokens = ''
                    else: yield _get_token(decoded_token, position)
        except Exception as error:
            try:
                if self.__show_errors:
                    error_message = 'ERROR in SapiensBERT.__generate_tokens_bert_legacy: '+str(error)
                    print(error_message)
                    try: self.__print_exc() if self.__display_error_point else None
                    except: pass
            except: pass
            return ''
    def __generate_tokens_bert(self, prompt='', max_tokens=500, temperature=1.0, top_k=0, top_p=1.0):
        try:
            self.__model.eval()
            encoded_prompt = list(self.__encode(prompt))
            current_ids = self.__tensor(encoded_prompt, dtype=self.__int64).unsqueeze(0).to(self.device)
            all_tokens = list(encoded_prompt)
            printed_length = len(self.__decode(all_tokens))
            buffer_tokens = ''
            pending_whitespace = [] 
            is_first_token_output = True
            def _get_token(token='', position=0):
                nonlocal is_first_token_output 
                def __remove_first_punctuation(text=''):
                    if not text: return text
                    punctuation_symbols = '.;!¡?¿,'
                    if text[0] in punctuation_symbols: return text[1:]
                    return text
                if position <= 1: token = __remove_first_punctuation(text=token.lstrip())
                elif position >= max_tokens: token = token.rstrip()
                if is_first_token_output and token:
                    token = token[0].upper() + token[1:]
                    is_first_token_output = False
                return token
            with self.__no_grad():
                for index in range(max_tokens):
                    position = index+1
                    mask_tensor = self.__tensor([[self.mask_token_id]], dtype=self.__int64).to(self.device)
                    input_with_mask = self.__cat((current_ids, mask_tensor), dim=1)
                    if input_with_mask.size(1) > self.block_size: input_with_mask = input_with_mask[:, -self.block_size:]
                    logits = self.__model(input_with_mask)
                    mask_logits = logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
                    if top_k > 0:
                        v, _ = self.__topk(mask_logits, min(top_k, mask_logits.size(-1)))
                        mask_logits[mask_logits < v[:, [-1]]] = float('-inf')
                    if top_p < 1.0:
                        sorted_logits, sorted_indices = self.__sort(mask_logits, descending=True)
                        cumulative_probs = self.__cumsum(self.__Function.softmax(sorted_logits, dim=-1), dim=-1)
                        sorted_indices_to_remove = cumulative_probs > top_p
                        sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
                        sorted_indices_to_remove[..., 0] = 0
                        indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
                        mask_logits[indices_to_remove] = float('-inf')
                    probs = self.__Function.softmax(mask_logits, dim=-1)
                    if temperature == 0: predicted_token_id = self.__topk(probs, 1)[1]
                    else: predicted_token_id = self.__multinomial(probs, 1)
                    current_ids = self.__cat((current_ids, predicted_token_id), dim=1)
                    token_val = predicted_token_id.item()
                    if token_val == self.mask_token_id: break
                    all_tokens.append(token_val)
                    full_text = self.__decode(all_tokens)
                    if full_text.endswith('\ufffd'): continue
                    decoded_token = full_text[printed_length:]
                    printed_length = len(full_text)
                    if self.end_tag:
                        buffer_tokens += decoded_token
                        if self.end_tag in buffer_tokens:
                            pending_whitespace = [] 
                            final_content = buffer_tokens.split(self.end_tag)[0]
                            final_content = final_content.rstrip()
                            for char in final_content: yield _get_token(char, position)
                            break
                        match = False
                        for i in range(len(buffer_tokens)):
                            suffix = buffer_tokens[i:]
                            if self.end_tag.startswith(suffix):
                                match = True
                                if i > 0:
                                    to_yield = buffer_tokens[:i]
                                    buffer_tokens = suffix
                                    for char in to_yield:
                                        if char.isspace(): pending_whitespace.append((char, position))
                                        else:
                                            for (ws_char, ws_pos) in pending_whitespace: yield _get_token(ws_char, ws_pos)
                                            pending_whitespace = []
                                            yield _get_token(char, position)
                                break
                        if not match:
                            for char in buffer_tokens:
                                if char.isspace(): pending_whitespace.append((char, position))
                                else:
                                    for (ws_char, ws_pos) in pending_whitespace: yield _get_token(ws_char, ws_pos)
                                    pending_whitespace = []
                                    yield _get_token(char, position)
                            buffer_tokens = ''
                    else: yield _get_token(decoded_token, position)
        except Exception as error:
            try:
                if self.__show_errors:
                    error_message = 'ERROR in SapiensBERT.__generate_tokens_bert: '+str(error)
                    print(error_message)
                    try: self.__print_exc() if self.__display_error_point else None
                    except: pass
            except: pass
            return self.__generate_tokens_bert_legacy(prompt, max_tokens, temperature, top_k, top_p)
    def __generate_tokens(self, prompt='', max_tokens=500, temperature=1.0, top_k=0, top_p=1.0): return self.__generate_tokens_bert(prompt, max_tokens, temperature, top_k, top_p)
    def train(self, dataset_path='', progress=True):
        try:
            training_metrics = {'val_loss': 0.0, 'loss': 0.0, 'generalization_rate': 0.0, 'precision': 0.0}
            if self.__train: return training_metrics
            dataset_path = str(dataset_path).strip()
            string = str(self.string).strip()
            precision = min((1.0, max((0.0, float(self.precision))))) if type(self.precision) in (bool, int, float) else 0.99
            tokenizer = str(self.tokenizer).lower().strip() if self.tokenizer else 'bert'
            self.block_size = max((1, int(self.context_window))) if type(self.context_window) in (bool, int, float) else 48
            if self.end_tag is not None and self.__end_tag is None: self.__end_tag = str(self.end_tag)
            validate = min((1.0, max((0.0, float(self.validate))))) if type(self.validate) in (bool, int, float) else 0.0
            progress = bool(progress) if type(progress) in (bool, int, float) else True
            self.embedding_dim = max((1, int(self.embedding_dim))) if type(self.embedding_dim) in (bool, int, float) else 48
            self.block_size = max((1, int(self.block_size))) if type(self.block_size) in (bool, int, float) else 64
            self.batch_size = max((1, int(self.batch_size))) if type(self.batch_size) in (bool, int, float) else 4
            self.number_heads = max((1, int(self.number_heads))) if type(self.number_heads) in (bool, int, float) else 4
            self.number_layers = max((1, int(self.number_layers))) if type(self.number_layers) in (bool, int, float) else 2
            self.dropout = max((0, float(self.dropout))) if type(self.dropout) in (bool, int, float) else 0.0
            self.learning_rate = max((0, float(self.learning_rate))) if type(self.learning_rate) in (bool, int, float) else 1e-3
            self.eval_interval = max((1, int(self.eval_interval))) if type(self.eval_interval) in (bool, int, float) else 50
            self.epochs = max((1, int(self.epochs))) if type(self.epochs) in (bool, int, float) else 2000
            if type(self.device) == str:
                from torch import device as sapiens_device
                if self.device in ('cuda', 'gpu'): self.device, self.__xm = sapiens_device('cuda'), None
                elif self.device in ('apple', 'mps'): self.device, self.__xm = sapiens_device('mps'), None
                elif self.__is_xla(): self.device = self.__xm.xla_device()
                else: self.device, self.__xm = sapiens_device('cpu'), None
            if tokenizer not in ('sapi', 'bert'): tokenizer = 'bert'
            self.__string = str(self.__string+'\n\n'+string).strip()
            loss_limit = min(1.0, max(0.0, 1.0 - precision))
            is_txt, is_json, text_data = dataset_path.endswith('.txt'), dataset_path.endswith('.json'), ''
            def prepare_json(json_data={}):
                if type(json_data) == dict: pairs = json_data[list(json_data.keys())[0]]
                else: pairs = json_data
                if self.__end_tag is None: self.__end_tag = '<|end|>'
                return '\n\n'.join([str(pair[list(pair.keys())[0]]+'\n'+pair[list(pair.keys())[1]]).replace(self.__end_tag, '').strip()+self.__end_tag for pair in pairs])
            def is_web_address(url_path=''):
                url_path = str(url_path).lower().strip()
                return url_path.startswith('https://') or url_path.startswith('http://') or url_path.startswith('www.')
            _is_web_address = is_web_address(url_path=dataset_path)
            if _is_web_address:
                is_json = True if '.json' in dataset_path.lower() else False
                def read_remote_file(url_path=''):
                    from urllib.request import urlopen
                    with urlopen(url_path) as response: return str(response.read().decode('utf-8', errors='replace').replace('\r\n', '\n').replace('\r', '\n')).strip()
                text_data = read_remote_file(url_path=dataset_path)
                if is_json:
                    def load_json(string_content=''):
                        json_content = {}
                        string_content = str(string_content)
                        try:
                            from json import loads
                            json_content = loads(string_content)
                        except:
                            from ast import literal_eval
                            json_content = literal_eval(string_content)
                        return json_content
                    json_data = load_json(string_content=text_data)
                    text_data = prepare_json(json_data=json_data)
            else:
                if not is_txt and not is_json and len(self.__string) < 1: raise ValueError('Unsupported file format. Use .txt or .json.')
                if is_txt:
                    with open(dataset_path, 'r', encoding='utf-8') as file: text_data = str(file.read()).strip()
                elif is_json:
                    with open(dataset_path, 'r', encoding='utf-8') as file: json_data = self.__json_load(file)
                    text_data = prepare_json(json_data=json_data)
            if len(self.__string) > 0: text_data += '\n\n' + self.__string
            text_data = text_data.strip()
            if self.tokenizer == 'sapi':
                chars = sorted(list(set(text_data)))
                self.mask_token_id = len(chars)
                self.__vocab_size = len(chars) + 1
                self.__char_to_idx = {ch: i for i, ch in enumerate(chars)}
                self.__idx_to_char = {i: ch for i, ch in enumerate(chars)}
                self.__char_to_idx[self.mask_token_tag] = self.mask_token_id
                self.__idx_to_char[self.mask_token_id] = self.mask_token_tag
                self.__encode = lambda s: [self.__char_to_idx.get(c, 0) for c in s]
                self.__decode = lambda idxs: ''.join([self.__idx_to_char.get(i, '') for i in idxs])
            else:
                encode = self.__get_encoding('cl100k_base')
                self.__vocab_size = encode.n_vocab + 1 
                self.mask_token_id = encode.n_vocab
                def encode_plus_mask(text):
                    ids = encode.encode(text)
                    return ids
                self.__encode = encode_plus_mask
                self.__decode = lambda idxs: encode.decode([i for i in idxs if i < encode.n_vocab])
            raw_ids = self.__encode(text_data)
            data_tensor = self.__tensor(raw_ids, dtype=self.__int64)
            n = int(0.9 * len(data_tensor)) if self.validate > 0 else len(data_tensor)
            train_data = data_tensor[:n]
            val_data = data_tensor[n:] if self.validate > 0 else None
            train_ds = self.__TextDatasets(train_data, self.block_size)
            train_loader = self.__DataLoader(train_ds, batch_size=self.batch_size, shuffle=True)
            val_loader = self.__DataLoader(self.__TextDatasets(val_data, self.block_size), batch_size=self.batch_size) if val_data is not None else None
            if self.__is_xla(): self.device = self.__xm.xla_device()
            self.__model = self.__Transformers(self, self.__vocab_size, self.embedding_dim, self.number_heads, self.number_layers, self.dropout, self.block_size).to(self.device)
            self.__optimizer = self.__optim.AdamW(self.__model.parameters(), lr=self.learning_rate)
            self.__model.train()
            formatted_bar = '{desc}{percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}]'
            current_precision = 0.0
            current_precisions = []
            from statistics import mean
            for epoch in range(self.epochs):
                progress_bar = self.__tqdm(train_loader, disable=not progress, bar_format=formatted_bar)
                epoch_loss = 0
                for input_batch, _ in progress_bar:
                    input_batch = input_batch.to(self.device)
                    masked_input, masked_labels = self.__apply_mlm_mask(input_batch)
                    logits = self.__model(masked_input)
                    loss = self.__Function.cross_entropy(logits.view(-1, self.__vocab_size), masked_labels.view(-1))
                    self.__optimizer.zero_grad()
                    loss.backward()
                    self.__utils.clip_grad_norm_(self.__model.parameters(), 1.0)
                    if self.__is_xla(): self.__xm.optimizer_step(self.__optimizer)
                    else: self.__optimizer.step()
                    epoch_loss += loss.item()
                    loss_value = loss.item()
                    current_precision = min(mean(current_precisions), abs(self.precision-0.0001)) if current_precisions else min(current_precision, abs(self.precision-0.0001))
                    str_epoch, str_loss, str_precision = f'{epoch+1}', f'{loss_value:.4f}', f'{current_precision:.4f}'
                    str_epoch = str_epoch.rjust(len(str(self.epochs)), '0')
                    str_loss = str_loss.rjust(8, '0')
                    str_precision = str_precision.rjust(6, '0')
                    progress_bar.set_description(f'Epoch {str_epoch} | Loss: {str_loss} | Precision: {str_precision}')
                    current_precision = max(0.0, 1-loss_value)
                    current_precisions.append(current_precision)
                    training_metrics['loss'] = loss_value
                    training_metrics['generalization_rate'] = 1-current_precision
                    training_metrics['precision'] = current_precision
                current_precision, exit = mean(current_precisions) if current_precisions else current_precision, False
                if current_precision >= self.precision: exit = True
                if not exit: current_precision = min(current_precision, self.precision)
                str_epoch, str_loss, str_precision = f'{epoch+1}', f'{loss_value:.4f}', f'{current_precision:.4f}'
                str_epoch = str_epoch.rjust(len(str(self.epochs)), '0')
                str_loss = str_loss.rjust(8, '0')
                str_precision = str_precision.rjust(6, '0')
                progress_bar.set_description(f'Epoch {str_epoch} | Loss: {str_loss} | Precision: {str_precision}')
                training_metrics['loss'] = loss_value
                training_metrics['generalization_rate'] = 1-current_precision
                training_metrics['precision'] = current_precision
                if val_loader:
                    val_loss = self.__compute_loss(val_loader)
                    training_metrics['val_loss'] = val_loss
                if exit:
                    if progress:
                        total_iterations = len(train_loader)
                        progress_bar = self.__tqdm(total=total_iterations, bar_format=formatted_bar)
                        progress_bar.n = total_iterations
                        current_precision = max(current_precision, self.precision)
                        str_precision = f'{current_precision:.4f}'
                        progress_bar.set_description(f'Final {str_epoch} | Loss: {str_loss} | Precision: {str_precision}')
                        progress_bar.refresh()
                        progress_bar.close()
                    break
            if training_metrics: self.__train = True
            return training_metrics
        except Exception as error:
            try:
                if self.__show_errors:
                    error_message = 'ERROR in SapiensBERT.train: '+str(error)
                    print(error_message)
                    try: self.__print_exc() if self.__display_error_point else None
                    except: pass
            except: pass
            return {'val_loss': 0.0, 'loss': 0.0, 'generalization_rate': 0.0, 'precision': 0.0}
    def training(self, dataset_path='', progress=True): return self.train(dataset_path, progress)
    def saveModel(self, model_path='', progress=True):
        try:
            model_path = str(model_path).strip()
            progress = bool(progress) if type(progress) in (bool, int, float) else True
            if self.__model is None: raise ValueError('Model is not initialized. Call train or loadModel first.')
            self.parameters_number = sum(parameters.numel() for parameters in self.__model.parameters())
            formatted_params = self.__format_params(self.parameters_number)
            if len(model_path) > 0:
                directory, file_name = self.__os_path.split(model_path)
                if not file_name: file_name = 'model.bert'
                elif not file_name.endswith('.bert'): file_name += '.bert'
            else: directory, file_name = str(model_path), 'model.bert'
            if directory and not self.__os_path.exists(directory): self.__os_makedirs(directory)
            save_path = self.__os_path.join(directory, file_name)            
            save_dict = {
                'embedding_dim': max((1, int(self.embedding_dim))) if type(self.embedding_dim) in (bool, int, float) else 48,
                'block_size': max((1, int(self.block_size))) if type(self.block_size) in (bool, int, float) else 64,
                'batch_size': max((1, int(self.batch_size))) if type(self.batch_size) in (bool, int, float) else 4,
                'number_heads': max((1, int(self.number_heads))) if type(self.number_heads) in (bool, int, float) else 4,
                'number_layers': max((1, int(self.number_layers))) if type(self.number_layers) in (bool, int, float) else 2,
                'dropout': max((0.0, float(self.dropout))) if type(self.dropout) in (bool, int, float) else 0.0,
                'learning_rate': max((0.0, float(self.learning_rate))) if type(self.learning_rate) in (bool, int, float) else 1e-3,
                'eval_interval': max((0, int(self.eval_interval))) if type(self.eval_interval) in (bool, int, float) else 50,
                'epochs': max((0, int(self.epochs))) if type(self.epochs) in (bool, int, float) else 2000,
                'precision': max((0.0, float(self.precision))) if type(self.precision) in (bool, int, float) else 0.99,
                'tokenizer': str(self.tokenizer).lower().strip() if self.tokenizer else 'bert',
                'context_window': max((1, int(self.context_window))) if type(self.context_window) in (bool, int, float) else 48,
                'end_tag': str(self.__end_tag) if self.__end_tag is not None else '<|end|>',
                'mask_token_tag': str(self.mask_token_tag) if self.mask_token_tag is not None else '<|mask|>',
                'mask_token_id': int(self.mask_token_id) if self.mask_token_id is not None else int(self.__vocab_size)-1,
                'validate': max((0.0, float(self.validate))) if type(self.validate) in (bool, int, float) else 0.0,
                'max_tokens': max((0, int(self.max_tokens))) if type(self.max_tokens) in (bool, int, float) else 500,
                'temperature': max((0.0, float(self.temperature))) if type(self.temperature) in (bool, int, float) else 1.0,
                'top_k': max((0, int(self.top_k))) if type(self.top_k) in (bool, int, float) else 0,
                'top_p': max((0.0, float(self.top_p))) if type(self.top_p) in (bool, int, float) else 1.0,
                'delay': max((0.0, float(self.delay))) if type(self.delay) in (bool, int, float) else 0.01,
                'vocab_size': max((0, int(self.__vocab_size))) if type(self.__vocab_size) in (bool, int, float) else 0,
                'parameters_number': max((0, int(self.parameters_number))) if type(self.parameters_number) in (bool, int, float) else 0,
                'architecture_type': 'bert_model',
                'model_state_dict': self.__model.state_dict()
            }
            if self.tokenizer == 'sapi':
                save_dict['char_to_idx'] = self.__char_to_idx if type(self.__char_to_idx) == dict else {}
                save_dict['idx_to_char'] = self.__idx_to_char if type(self.__idx_to_char) == dict else {}
            if progress:
                for _ in self.__tqdm(range(10), desc=f'Saving model with {formatted_params} parameters', leave=False): self.__save(save_dict, save_path)
            else: self.__save(save_dict, save_path)
            return True
        except Exception as error:
            try:
                if self.__show_errors:
                    error_message = 'ERROR in SapiensBERT.saveModel: '+str(error)
                    print(error_message)
                    try: self.__print_exc() if self.__display_error_point else None
                    except: pass
            except: pass
            return False
    def loadModel(self, model_path='', progress=True):
        try:
            if type(self.device) == str:
                from torch import device as sapiens_device
                if self.device in ('cuda', 'gpu'): self.device, self.__xm = sapiens_device('cuda'), None
                elif self.device in ('apple', 'mps'): self.device, self.__xm = sapiens_device('mps'), None
                elif self.__is_xla(): self.device = self.__xm.xla_device()
                else: self.device, self.__xm = sapiens_device('cpu'), None
            if progress:
                for _ in self.__tqdm(range(10), desc='Loading model', leave=False):
                    try: ckpt = self.__load(model_path, map_location=self.device)
                    except: ckpt = self.__load(model_path)
            else:
                try: ckpt = self.__load(model_path, map_location=self.device)
                except: ckpt = self.__load(model_path)
            self.embedding_dim = max((1, int(ckpt.get('embedding_dim', 48))))
            self.block_size = max((1, int(ckpt.get('block_size', 64))))
            self.batch_size = max((1, int(ckpt.get('batch_size', 4))))
            self.number_heads = max((1, int(ckpt.get('number_heads', 4))))
            self.number_layers = max((1, int(ckpt.get('number_layers', 2))))
            self.dropout = max((0.0, float(ckpt.get('dropout', 0.0))))
            self.learning_rate = max((0.0, float(ckpt.get('learning_rate', 1e-3))))
            self.eval_interval = max((0, int(ckpt.get('eval_interval', 50))))
            self.epochs = max((0, int(ckpt.get('epochs', 2000))))
            self.precision = max((0.0, float(ckpt.get('precision', 0.99))))
            self.tokenizer = str(ckpt.get('tokenizer', 'bert')).lower().strip()
            self.context_window = max((1, int(ckpt.get('context_window', 48))))
            self.__end_tag = str(ckpt.get('end_tag', '<|end|>'))
            self.mask_token_tag = str(ckpt.get('mask_token_tag', '<|mask|>'))
            self.validate = max((0.0, float(ckpt.get('validate', 0.0))))
            self.max_tokens = max((0, int(ckpt.get('max_tokens', 500))))
            self.temperature = max((0.0, float(ckpt.get('temperature', 1.0))))
            self.top_k = max((0, int(ckpt.get('top_k', 0))))
            self.top_p = max((0.0, float(ckpt.get('top_p', 1.0))))
            self.delay = max((0.0, float(ckpt.get('delay', 0.01))))
            self.__vocab_size = max((0, int(ckpt.get('vocab_size', 0))))
            self.mask_token_id = int(ckpt.get('mask_token_id', int(self.__vocab_size)-1))
            self.parameters_number = max((0, int(ckpt.get('parameters_number', 0))))
            if self.tokenizer == 'sapi':
                self.__char_to_idx = ckpt.get('char_to_idx', {})
                self.__idx_to_char = ckpt.get('idx_to_char', {})
                self.__idx_to_char = {int(k): v for k, v in self.__idx_to_char.items()}
                self.__vocab_size = len(self.__char_to_idx)
                self.mask_token_id = ckpt.get('mask_token_id')
                if self.mask_token_id is None:
                    if self.mask_token_tag in self.__char_to_idx: self.mask_token_id = self.__char_to_idx[self.mask_token_tag]
                    else: self.mask_token_id = len(self.__char_to_idx) - 1 
                self.__encode = lambda s: [self.__char_to_idx.get(c, 0) for c in s]
                self.__decode = lambda idxs: ''.join([self.__idx_to_char.get(int(i), '') for i in idxs])
            else:
                enc = self.__get_encoding('cl100k_base')
                self.__vocab_size = enc.n_vocab + 1 
                self.mask_token_id = enc.n_vocab
                self.__encode = lambda text: enc.encode(text)
                self.__decode = lambda idxs: enc.decode([i for i in idxs if i < enc.n_vocab])
            self.__model = self.__Transformers(self, self.__vocab_size, self.embedding_dim, self.number_heads, self.number_layers, self.dropout, self.block_size).to(self.device)
            self.__model.load_state_dict(ckpt['model_state_dict'])
            self.__model.eval()
            self.__train = True
            return True
        except Exception as error:
            try:
                if self.__show_errors:
                    error_message = 'ERROR in SapiensBERT.loadModel: '+str(error)
                    print(error_message)
                    try: self.__print_exc() if self.__display_error_point else None
                    except: pass
            except: pass
            return False
    def fineTuning(self, dataset_path='', progress=True):
        try:
            training_metrics = {'val_loss': 0.0, 'loss': 0.0, 'generalization_rate': 0.0, 'precision': 0.0}
            dataset_path = str(dataset_path).strip()
            progress = bool(progress) if type(progress) in (bool, int, float) else True
            def _read_remote_file(remote_path=''):
                remote_path = str(remote_path).strip()
                from urllib.request import urlopen
                try:
                    from os import environ
                    from certifi import where
                    environ['SSL_CERT_FILE'] = where()
                    from logging import getLogger, ERROR
                    getLogger('urlopen').setLevel(ERROR)
                except: pass
                remote_stream = urlopen(remote_path)
                content = remote_stream.read().decode('utf-8')
                return str(content).strip()
            is_json, string_content, data_list = dataset_path.lower().endswith('.json'), '', []
            if dataset_path.startswith(('https://', 'http://')): string_content = _read_remote_file(remote_path=dataset_path)
            else:
                with open(dataset_path, 'r', encoding='utf-8') as file_object: string_content = str(file_object.read()).strip()
            if is_json:
                fit_structure, data = [], []
                if string_content:
                    def _string_to_dictionary_or_json(string=''):
                        dictionary_or_json = {}
                        original_string = string
                        string = str(string).strip()
                        try:
                            from json import loads
                            try: dictionary_or_json = loads(string)
                            except: dictionary_or_json = loads(original_string)
                        except:
                            from ast import literal_eval
                            try: dictionary_or_json = literal_eval(string)
                            except: dictionary_or_json = literal_eval(original_string)
                        return dictionary_or_json
                    fit_structure = _string_to_dictionary_or_json(string=string_content)
                if fit_structure:
                    if type(fit_structure) == dict:
                        json_keys = list(fit_structure.keys())
                        if 'data' in json_keys: data = list(fit_structure.get('data', []))
                        else:
                            data_key = str(json_keys[0]).strip()
                            data = list(fit_structure.get(data_key, []))
                    elif type(fit_structure) in (tuple, list): data = fit_structure
                if data: data_list = data
            elif self.__show_errors:
                print('The file must be a JSON with a "data" key containing an array of objects with the keys "input" and "output".')
                return training_metrics
            epochs, _epoch = self.epochs, 0
            learning_rate = self.learning_rate if self.learning_rate else 3e-4
            self.__model.train()
            optimizer = self.__optim.AdamW(self.__model.parameters(), lr=learning_rate)
            end_tag = self.__end_tag if self.__end_tag else '<|end|>'
            from random import shuffle
            from statistics import mean
            total_iterations = epochs*len(data_list)
            progress_bar = self.__tqdm(total=total_iterations, disable=not progress)
            best_loss, val_loss, avg_loss, exit = float('inf'), 0, 0, False
            current_precision = 0.0
            current_precisions = []
            for epoch in range(epochs):
                _epoch = epoch+1
                total_loss, count = 0, 0
                shuffle(data_list)
                for input_output in data_list:
                    _input, _output = '', ''
                    if input_output and type(input_output) == dict:
                        json_keys = list(input_output.keys())
                        if 'input' in json_keys: _input = str(input_output.get('input', '')).strip()
                        elif 'Input' in json_keys: _input = str(input_output.get('Input', '')).strip()
                        elif 'INPUT' in json_keys: _input = str(input_output.get('INPUT', '')).strip()
                        elif 'question' in json_keys: _input = str(input_output.get('question', '')).strip()
                        elif 'Question' in json_keys: _input = str(input_output.get('Question', '')).strip()
                        elif 'QUESTION' in json_keys: _input = str(input_output.get('QUESTION', '')).strip()
                        elif 'prompt' in json_keys: _input = str(input_output.get('prompt', '')).strip()
                        elif 'Prompt' in json_keys: _input = str(input_output.get('Prompt', '')).strip()
                        elif 'PROMPT' in json_keys: _input = str(input_output.get('PROMPT', '')).strip()
                        if 'output' in json_keys: _output = str(input_output.get('output', '')).strip()
                        elif 'Output' in json_keys: _output = str(input_output.get('Output', '')).strip()
                        elif 'OUTPUT' in json_keys: _output = str(input_output.get('OUTPUT', '')).strip()
                        elif 'answer' in json_keys: _output = str(input_output.get('answer', '')).strip()
                        elif 'Answer' in json_keys: _output = str(input_output.get('Answer', '')).strip()
                        elif 'ANSWER' in json_keys: _output = str(input_output.get('ANSWER', '')).strip()
                        elif 'response' in json_keys: _output = str(input_output.get('response', '')).strip()
                        elif 'Response' in json_keys: _output = str(input_output.get('Response', '')).strip()
                        elif 'RESPONSE' in json_keys: _output = str(input_output.get('RESPONSE', '')).strip()
                        if not _input: _input = str(input_output[json_keys[0]]).strip()
                        if not _output: _output = str(input_output[json_keys[1]]).strip()
                    progress_bar.update(1)
                    if not _input or not _output: continue
                q_ids = self.__encode(_input.strip() + '\n')
                a_ids = self.__encode(_output.strip() + end_tag)
                full_ids = q_ids + a_ids
                if len(full_ids) > self.block_size:
                    full_ids = full_ids[:self.block_size]
                    len_useful_answer = len(full_ids) - len(q_ids)
                    if len_useful_answer <= 0: continue
                else: len_useful_answer = len(a_ids)
                target_offset = self.__randint(0, len_useful_answer, (1,)).item()
                target_abs_idx = len(q_ids) + target_offset
                target_token = full_ids[target_abs_idx]
                input_ids = full_ids[:target_abs_idx] + [self.mask_token_id]
                if len(input_ids) > self.block_size: input_ids = input_ids[-self.block_size:]
                input_tensor = self.__tensor([input_ids], dtype=self.__int64).to(self.device)
                target_tensor = self.__tensor([target_token], dtype=self.__int64).to(self.device)
                logits = self.__model(input_tensor)
                last_token_logits = logits[0, -1, :].unsqueeze(0)
                loss = self.__Function.cross_entropy(last_token_logits, target_tensor)
                optimizer.zero_grad()
                loss.backward()
                self.__utils.clip_grad_norm_(self.__model.parameters(), 1.0)
                optimizer.step()
                total_loss += loss.item()
                loss_value = loss.item()
                current_precision = min(mean(current_precisions), abs(self.precision-0.0001)) if current_precisions else min(current_precision, abs(self.precision-0.0001))
                str_epoch, str_loss, str_precision = f'{_epoch}', f'{loss_value:.4f}', f'{current_precision:.4f}'
                str_epoch = str_epoch.rjust(len(str(epochs)), '0')
                str_loss = str_loss.rjust(8, '0')
                str_precision = str_precision.rjust(6, '0')
                progress_bar.set_description(f'Epoch {str_epoch} | Loss: {str_loss} | Precision: {str_precision}')
                current_precision = max(0.0, 1-loss_value)
                current_precisions.append(current_precision)
                training_metrics['loss'] = loss_value
                training_metrics['generalization_rate'] = 1-current_precision
                training_metrics['precision'] = current_precision
                count += 1
                avg_loss = total_loss / count if count > 0 else 0
                if avg_loss < best_loss: best_loss = avg_loss
                val_loss = avg_loss
                current_precision = mean(current_precisions) if current_precisions else current_precision
                if current_precision >= self.precision: exit = True
                if exit:
                    current_precision = max(current_precision, self.precision)
                    break
                else: current_precision = min(mean(current_precisions), abs(self.precision-0.0001))
            str_epoch, str_loss, str_precision = f'{_epoch}', f'{avg_loss:.4f}', f'{current_precision:.4f}'
            str_epoch = str_epoch.rjust(len(str(epochs)), '0')
            str_loss = str_loss.rjust(8, '0')
            str_precision = str_precision.rjust(6, '0')
            progress_bar.set_description(f'Final {str_epoch} | Loss: {str_loss} | Precision: {str_precision}')
            training_metrics['loss'] = loss_value
            training_metrics['generalization_rate'] = 1-current_precision
            training_metrics['precision'] = current_precision
            if val_loss: training_metrics['val_loss'] = val_loss
            if progress: 
                progress_bar.n = total_iterations
                progress_bar.refresh()
                progress_bar.close()
            if self.device.type == 'cuda': self.__cuda.empty_cache()
            if training_metrics: self.__train = True
            return training_metrics
        except Exception as error:
            try:
                if self.__show_errors:
                    error_message = 'ERROR in SapiensBERT.fineTuning: '+str(error)
                    print(error_message)
                    try: self.__print_exc() if self.__display_error_point else None
                    except: pass
            except: pass
            return {'val_loss': 0.0, 'loss': 0.0, 'generalization_rate': 0.0, 'precision': 0.0}
    def inference(self, prompt='', stream=True):
        try:
            prompt = str(prompt).strip()
            max_tokens = max((1, int(self.max_tokens))) if type(self.max_tokens) in (bool, int, float) else 500
            temperature = max((0, float(self.temperature))) if type(self.temperature) in (bool, int, float) else 0.5
            top_k = max((0, int(self.top_k))) if type(self.top_k) in (bool, int, float) else 0
            top_p = min((1.0, max((0.0, float(self.top_p))))) if type(self.top_p) in (bool, int, float) else 1.0
            stream = bool(stream) if type(stream) in (bool, int, float) else False
            if self.__model is None: raise ValueError('Model is not initialized. Call train or loadModel first.')
            if stream: return self.__generate_tokens(prompt=prompt, max_tokens=max_tokens, temperature=temperature, top_k=top_k, top_p=top_p)
            tokens = list(self.__generate_tokens(prompt=prompt, max_tokens=max_tokens, temperature=temperature, top_k=top_k, top_p=top_p))
            return ''.join(tokens)
        except Exception as error:
            try:
                if self.__show_errors:
                    error_message = 'ERROR in SapiensBERT.inference: '+str(error)
                    print(error_message)
                    try: self.__print_exc() if self.__display_error_point else None
                    except: pass
            except: pass
            return ''
    def completeMessages(self, messages=[], stream=False):
        try:
            complete_messages = {'answer': '', 'messages': [], 'next_token': ''}
            messages = list(messages) if type(messages) in (tuple, list, dict) else []
            stream = bool(stream) if type(stream) in (bool, int, float) else False
            complete_messages['messages'] = messages
            if messages:
                prompt_template, assistant = '', 'assistant'
                if len(messages) < 2: prompt_template = str(messages[-1].get('content', '?')).strip()
                else:
                    end_tag, system_tags = self.end_tag if self.end_tag else '', ('system', 'model', 'instruction')
                    assistant = str(messages[-2].get('role', '')).strip() if len(messages) > 1 else 'assistant'
                    if assistant.lower() in system_tags: assistant = 'assistant'
                    for message in messages:
                        role = str(message.get('role', '')).strip()
                        content = str(message.get('content', '')).strip()
                        if role and content and (role == assistant or role.lower() in system_tags): prompt_template += f'{role}:\n{content}{end_tag}\n'
                        elif role and content: prompt_template += f'{role}:\n{content}\n'
                prompt_template = prompt_template.strip()
                prompt_template += f'\n{assistant}:\n'
                def _get_stream(prompt_template='', complete_messages={}):
                    current_answer, message = '', {'role': 'assistant', 'content': ''}
                    complete_messages['messages'].append(message)
                    inference_function = self.inference(prompt=prompt_template, stream=True)
                    for token in inference_function:
                        current_answer += token
                        complete_messages['answer'] = current_answer
                        complete_messages['messages'][-1]['content'] = current_answer
                        complete_messages['next_token'] = token
                        yield complete_messages
                def _get_string(prompt_template='', complete_messages={}):
                    inference_function = self.inference(prompt=prompt_template, stream=False)
                    complete_messages['answer'] = inference_function
                    message = {'role': 'assistant', 'content': inference_function}
                    complete_messages['messages'].append(message)
                    token = inference_function.split(chr(32))[-1].rstrip()
                    complete_messages['next_token'] = token
                    return complete_messages
                if stream: complete_messages = _get_stream(prompt_template=prompt_template, complete_messages=complete_messages)
                else: complete_messages = _get_string(prompt_template=prompt_template, complete_messages=complete_messages)
            return complete_messages
        except Exception as error:
            try:
                if self.__show_errors:
                    error_message = 'ERROR in SapiensBERT.completeMessages: '+str(error)
                    print(error_message)
                    try: self.__print_exc() if self.__display_error_point else None
                    except: pass
            except: pass
            return {'answer': '', 'messages': [], 'next_token': ''}
    def printInference(self, prompt='', stream=True):
        try:
            prompt = str(prompt).strip()
            max_tokens = max((1, int(self.max_tokens))) if type(self.max_tokens) in (bool, int, float) else 500
            temperature = max((0, float(self.temperature))) if type(self.temperature) in (bool, int, float) else 0.5
            top_k = max((0, int(self.top_k))) if type(self.top_k) in (bool, int, float) else 0
            top_p = min((1.0, max((0.0, float(self.top_p))))) if type(self.top_p) in (bool, int, float) else 1.0
            stream = bool(stream) if type(stream) in (bool, int, float) else False
            if self.__model is None: raise ValueError('Model is not initialized. Call train or loadModel first.')
            if stream:
                [print(token, end='', flush=True) for token in self.__generate_tokens(prompt=prompt, max_tokens=max_tokens, temperature=temperature, top_k=top_k, top_p=top_p)]
                print()
            else: print(self.predict(prompt=prompt, max_tokens=max_tokens, temperature=temperature, stream=stream))
        except Exception as error:
            try:
                if self.__show_errors:
                    error_message = 'ERROR in SapiensBERT.printInference: '+str(error)
                    print(error_message)
                    try: self.__print_exc() if self.__display_error_point else None
                    except: pass
            except: pass
    def printCompleteMessages(self, messages=[], stream=True):
        try:
            inference = self.completeMessages(messages=messages, stream=stream)
            if stream:
                from time import sleep
                for token in inference:
                    print(token['next_token'], end='', flush=True)
                    sleep(self.delay)
                print()
            else: print(inference['answer'])
        except Exception as error:
            try:
                if self.__show_errors:
                    error_message = 'ERROR in SapiensBERT.printCompleteMessages: '+str(error)
                    print(error_message)
                    try: self.__print_exc() if self.__display_error_point else None
                    except: pass
            except: pass
    def close(self):
        try:
            self.__init__()
            return True
        except Exception as error:
            try:
                if self.__show_errors:
                    error_message = 'ERROR in SapiensBERT.close: '+str(error)
                    print(error_message)
                    try: self.__print_exc() if self.__display_error_point else None
                    except: pass
            except: pass
            return False
# This is a Transformer algorithm of the BERT type (Bidirectional Encoder Representations from Transformers) developed by Sapiens Technology®️
# with the purpose of training, saving, fine-tuning, loading, and inferring bidirectional language models.
# The code includes a wide range of pre-established functions that make it easier to develop conversational Artificial Intelligence models of any scale.
# --------------------------> A SAPIENS TECHNOLOGY®️ PRODUCTION) <--------------------------
