"""
This algorithm is a proposal from Sapiens Technology®️ for a new concept of language model architecture focused on direct CPU training.
Its concept aims to establish a new type of paradigm that enables cheaper and more accessible training.
The CPUModel architecture allows for fast and efficient training on home hardware with low computational power.
It is useful for students and developers looking to create specialized models quickly, cheaply, and accurately.
Although large language models can be created with this architecture, it will perform better on small language models that are hyper-specialized.
Any alteration, disclosure, or public comment about the technical and theoretical concepts involved in this algorithm without prior authorization
from Sapiens Technology®️ is strictly prohibited and subject to legal action by our legal team.
"""
# --------------------------> A SAPIENS TECHNOLOGY®️ PRODUCTION) <--------------------------
class CPUModel:
    def __init__(self, show_errors=True, display_error_point=False):
        try:
            self.__show_errors = bool(show_errors) if type(show_errors) in (bool, int, float) else True
            self.__display_error_point = bool(display_error_point) if type(display_error_point) in (bool, int, float) else False
            from traceback import print_exc
            self.__print_exc = print_exc
            from unicodedata import normalize, combining
            from re import sub, split, findall
            from numpy import zeros, array, sqrt, sum as npsum, random
            from pickle import dump, load
            from tqdm import tqdm
            from collections import Counter
            from sapiens_attention import SapiensAttention
            from sapiens_tokenizer import SapiensTokenizer
            from os.path import exists
            from random import choice
            self.__normalize = normalize
            self.__combining = combining
            self.__sub = sub
            self.__split = split
            self.__findall = findall
            self.__zeros = zeros
            self.__array = array
            self.__sqrt = sqrt
            self.__npsum = npsum
            self.__random = random
            self.__dump = dump
            self.__load = load
            self.__tqdm = tqdm
            self.__Counter = Counter
            self.__SapiensAttention = SapiensAttention
            self.__SapiensTokenizer = SapiensTokenizer
            self.__exists = exists
            self.__choice = choice
            self.__training_tokens = None
            self.__token_embeddings = None
            self.__sequence_norms = None
            self.__token_lengths = None
            self.__word_to_id = {}
            self.__id_to_word = {}
            self.__normalization_map = None
            self.__experts_data = []
            self.__context_window = 128
            self.END_TAG = None
            self.N_EXPERTS = None
            self.__fine_tuning = []
        except Exception as error:
            try:
                if self.__show_errors:
                    error_message = 'ERROR in CPUModel.__init__: '+str(error)
                    print(error_message)
                    try: self.__print_exc() if self.__display_error_point else None
                    except: pass
            except: pass
    def __normalize_text(self, text=''):
        try:
            nfkd_form = self.__normalize('NFKD', text)
            text = ''.join([character for character in nfkd_form if not self.__combining(character)])
            text = text.lower()
            text = self.__sub(r'[^\w\s]', '', text)
            return text
        except Exception as error:
            if self.__show_errors:
                error_message = 'ERROR in CPUModel.__normalize_text: '+str(error)
                print(error_message)
                try: self.__print_exc() if self.__display_error_point else None
                except: pass
            return text
    def __build_normalization_map(self):
        try:
            if not self.__id_to_word:
                self.__normalization_map = None
                return
            vocab_size = len(self.__id_to_word)
            self.__normalization_map = self.__zeros(vocab_size, dtype=int)
            norm_str_to_id, next_norm_id = {}, 0
            for tid, word in self.__id_to_word.items():
                norm_word = self.__normalize_text(word)
                if norm_word not in norm_str_to_id:
                    norm_str_to_id[norm_word] = next_norm_id
                    next_norm_id += 1
                self.__normalization_map[tid] = norm_str_to_id[norm_word]
        except Exception as error:
            if self.__show_errors:
                error_message = 'ERROR in CPUModel.__build_normalization_map: '+str(error)
                print(error_message)
                try: self.__print_exc() if self.__display_error_point else None
                except: pass
    def __encoder(self, text=''):
        try:
            token_ids = []
            tokens = self.__findall(r'[^\s]+|\s+', text)
            for token in tokens:
                if token not in self.__word_to_id:
                    new_id = len(self.__word_to_id)
                    self.__word_to_id[token] = new_id
                    self.__id_to_word[new_id] = token
                token_ids.append(self.__word_to_id[token])
            return token_ids
        except Exception as error:
            if self.__show_errors:
                error_message = 'ERROR in CPUModel.__encoder: '+str(error)
                print(error_message)
                try: self.__print_exc() if self.__display_error_point else None
                except: pass
            return []
    def __decoder(self, token_ids=[]):
        try:
            result_words = []
            for token_id in token_ids:
                if token_id in self.__id_to_word: result_words.append(self.__id_to_word[token_id])
                else:
                    closest_id = min(self.__id_to_word.keys(), key=lambda x: abs(x - token_id))
                    result_words.append(self.__id_to_word[closest_id])
            return ''.join(result_words)
        except Exception as error:
            if self.__show_errors:
                error_message = 'ERROR in CPUModel.__decoder: '+str(error)
                print(error_message)
                try: self.__print_exc() if self.__display_error_point else None
                except: pass
            return ''
    def __build_token_lengths(self):
        try:
            if not self.__id_to_word:
                self.__token_lengths = None
                return
            vocab_size = len(self.__id_to_word)
            self.__token_lengths = self.__zeros(vocab_size, dtype='float32')
            for tid, word in self.__id_to_word.items(): self.__token_lengths[tid] = len(word)
        except Exception as error:
            if self.__show_errors:
                error_message = 'ERROR in CPUModel.__build_token_lengths: '+str(error)
                print(error_message)
                try: self.__print_exc() if self.__display_error_point else None
                except: pass
    def __split_sentences(self, text=''):
        try:
            if not text: return []
            return self.__split(r'(?<=[.?!;])(?=\s)', str(text))
        except Exception as error:
            if self.__show_errors:
                error_message = 'ERROR in CPUModel.__split_sentences: '+str(error)
                print(error_message)
                try: self.__print_exc() if self.__display_error_point else None
                except: pass
            return []
    def __infer_adjustment(self, prompt='', min_prob=0.0):
        try:
            infer_adjustment = []
            if not self.__fine_tuning: return infer_adjustment
            prompt = str(prompt).strip()
            min_prob = min(1.0, max(0.0, float(min_prob))) if type(min_prob) in (bool, int, float) else 0.0
            coded_prompt = self.__encoder(text=self.__normalize_text(text=prompt))
            coded_prompt_len = len(coded_prompt)
            for fine_tuning in self.__fine_tuning:
                coded_input, coded_output, hits = fine_tuning[0][:coded_prompt_len-1], fine_tuning[1], 0
                min_len = min(coded_prompt_len, len(coded_input))
                if coded_prompt[:min_len] == coded_input[:min_len]:
                    infer_adjustment = [coded_output]
                    break
                for code in coded_prompt:
                    if code in coded_input: hits += 1
                probability = hits/coded_prompt_len
                if probability >= min_prob: infer_adjustment.append(coded_output)
            return infer_adjustment
        except Exception as error:
            if self.__show_errors:
                error_message = 'ERROR in CPUModel.__infer_adjustment: '+str(error)
                print(error_message)
                try: self.__print_exc() if self.__display_error_point else None
                except: pass
            return []
    def train(self, dataset_path='', progress=True):
        try:
            dataset_path = str(dataset_path).strip()
            progress = bool(progress) if type(progress) in (bool, int, float) else True
            sapiens_tokenizer = self.__SapiensTokenizer()
            tokens_counted, n_experts = sapiens_tokenizer.count_tokens_txt(file_path=dataset_path), self.N_EXPERTS
            if not n_experts and tokens_counted > 40000: n_experts = tokens_counted//75
            else: n_experts = self.N_EXPERTS if self.N_EXPERTS else 1
            self.__word_to_id, self.__id_to_word, self.__experts_data, content = {}, {}, [], ''
            if dataset_path.startswith(('https://', 'http://')):
                from urllib.request import urlopen
                def _read_remote_file(remote_path=''):
                    remote_stream = urlopen(remote_path)
                    content = remote_stream.read().decode('utf-8')
                    return str(content).strip()
                content = _read_remote_file(remote_path=dataset_path)
            else:
                with open(dataset_path, 'r', encoding='utf-8') as file: content = str(file.read()).strip()
            total_len, experts_temp_store = len(content), [{'tokens': [], 'text_parts': [], 'sentences_vocab_sets': []} for _ in range(n_experts)]
            sentences, target_chunk_size = self.__split_sentences(content), total_len // n_experts if n_experts > 0 else total_len
            current_expert_idx, current_expert_size = 0, 0
            progress_bar = self.__tqdm(total=total_len, unit='chars', desc='Encoding') if progress else None
            for sentence in sentences:
                sent_tokens, sent_len = self.__encoder(sentence), len(sentence)
                sent_tokens_len = len(sent_tokens)
                if sent_tokens_len > self.__context_window: self.__context_window = sent_tokens_len
                norm_sent = self.__normalize_text(sentence)
                sent_word_set = set(norm_sent.split())
                experts_temp_store[current_expert_idx]['tokens'].extend(sent_tokens)
                experts_temp_store[current_expert_idx]['text_parts'].append(sentence)
                if sent_word_set: experts_temp_store[current_expert_idx]['sentences_vocab_sets'].append(sent_word_set)
                current_expert_size += sent_len
                if progress_bar: progress_bar.update(sent_len)
                if current_expert_size >= target_chunk_size and current_expert_idx < n_experts - 1:
                    current_expert_idx += 1
                    current_expert_size = 0
            if progress_bar: progress_bar.close()
            all_tokens_global, lengths = [], [len(exp['tokens']) for exp in experts_temp_store if len(exp['tokens']) > 0]
            min_len = min(lengths) if lengths else 0
            expert_max_len = max(1, int(min_len * 0.01))
            sapiens_attention = self.__SapiensAttention()
            iterator = self.__tqdm(range(len(experts_temp_store)), desc='Training') if progress else range(len(experts_temp_store))
            for index in iterator:
                exp_data = experts_temp_store[index]
                full_text = ''.join(exp_data['text_parts'])
                all_tokens_global.extend(exp_data['tokens'])
                if not full_text.strip(): attentions = []
                else: attentions = sapiens_attention.get_attention_words(text=full_text, maximum_length=expert_max_len)
                full_vocab_set = set(self.__findall(r'\w+', self.__normalize_text(full_text)))
                self.__experts_data.append({'tokens': self.__array(exp_data['tokens'], dtype=int), 'attentions': set(attentions), 'full_vocab': full_vocab_set, 'sentences_vocab_sets': exp_data['sentences_vocab_sets']})
            global_tokens_array = self.__array(all_tokens_global, dtype=int)
            if len(global_tokens_array) > 0: vocab_size = int(global_tokens_array.max()) + 1
            else: vocab_size = 1
            self.__build_token_lengths()
            self.__build_normalization_map()
            embedding_dim = min(256, vocab_size)
            self.__random.seed(42)
            self.__token_embeddings = self.__random.randn(vocab_size, embedding_dim).astype('float32')
            token_counts = self.__Counter(global_tokens_array.tolist())
            for token_id, count in token_counts.items():
                weight = self.__sqrt(count)
                self.__token_embeddings[token_id] *= weight
            norms = self.__sqrt(self.__npsum(self.__token_embeddings ** 2, axis=1, keepdims=True))
            norms[norms == 0] = 1
            self.__token_embeddings = self.__token_embeddings / norms
            if self.__experts_data: self.__training_tokens = self.__experts_data[0]['tokens']
            else: self.__training_tokens = self.__array([], dtype=int)
            return True
        except Exception as error:
            if self.__show_errors:
                error_message = 'ERROR in CPUModel.train: '+str(error)
                print(error_message)
                try: self.__print_exc() if self.__display_error_point else None
                except: pass
            return False
    def fineTuning(self, dataset_path='', progress=True):
        try:
            fine_tuned = False
            dataset_path = str(dataset_path).strip()
            progress = bool(progress) if type(progress) in (bool, int, float) else True
            content, json_object, data = '', [], []
            if not self.__exists(dataset_path): return fine_tuned
            try:
                with open(dataset_path, 'r', errors='replace') as file: content = str(file.read()).strip()
            except:
                for encoding in ('utf-8', 'latin-1', 'windows-1252'):
                    try:
                        with open(dataset_path, 'r', encoding=encoding, errors='ignore') as file: content = str(file.read()).strip()
                        if content: break
                    except: pass
            if content:
                try:
                    from json import loads
                    json_object = loads(content)
                except:
                    from ast import literal_eval
                    json_object = literal_eval(content)
            if type(json_object) == dict:
                key_names, key_data = list(json_object.keys()), ''
                possible_key_names = ('data', 'Data', 'DATA')
                for possible_key_name in possible_key_names:
                    if possible_key_name in key_names:
                        key_data = possible_key_name
                        break
                if not key_data: key_data = key_names[0]
                data = list(json_object[key_data])
            else: data = list(json_object)
            input_priority = ['input', 'Input', 'INPUT', 'question', 'Question', 'QUESTION', 'prompt', 'Prompt', 'PROMPT']
            output_priority = ['output', 'Output', 'OUTPUT', 'answer', 'Answer', 'ANSWER', 'response', 'Response', 'RESPONSE']
            progress_bar = self.__tqdm(total=len(data), desc='Fine-tuning') if progress else None
            for input_output in data:
                _input = _output = ''
                if input_output:
                    columns = list(input_output.keys())
                    for name in input_priority:
                        if name in columns:
                            input_column = name
                            break
                    for name in output_priority:
                        if name in columns:
                            output_column = name
                            break
                    _input, _output = str(input_output.get(input_column, '')).strip(), str(input_output.get(output_column, '')).strip()
                if _input and _output:
                    coded_input, coded_output = self.__encoder(text=self.__normalize_text(text=_input)), self.__encoder(text=_output)
                    self.__fine_tuning.append([coded_input, coded_output])
                if progress_bar: progress_bar.update(1)
            if progress_bar: progress_bar.close()
            fine_tuned = True
            return fine_tuned
        except Exception as error:
            if self.__show_errors:
                error_message = 'ERROR in CPUModel.fineTuning: '+str(error)
                print(error_message)
                try: self.__print_exc() if self.__display_error_point else None
                except: pass
            return False
    def infer(self, prompt='', temperature=0.1, max_tokens=None):
        try:
            prompt = str(prompt).strip() if prompt else '?'
            temperature = min(1.0, max(0.0, float(temperature))) if type(temperature) in (bool, int, float) else 0.1
            max_tokens = max(1, int(max_tokens)) if type(max_tokens) in (bool, int, float) else self.__context_window
            infer_adjustment = self.__infer_adjustment(prompt=prompt, min_prob=1.0-temperature)
            if infer_adjustment:
                adjustment = self.__choice(infer_adjustment)
                if not adjustment: return
                for code in adjustment: yield self.__decoder(token_ids=[code])
                return
            if not self.__experts_data: return
            sapiens_attention = self.__SapiensAttention()
            prompt_len_ids, first_token = len(self.__encoder(prompt)), True
            prompt_kw_max_len = max(1, prompt_len_ids // 2)
            if prompt.strip():
                raw_attentions = sapiens_attention.get_attention_words(text=prompt, maximum_length=prompt_kw_max_len)
                normalized_attentions = set()
                for kw in raw_attentions: normalized_attentions.add(self.__normalize_text(kw))
            else: raw_attentions, normalized_attentions = [], set()
            selected_expert = None
            if normalized_attentions:
                phase0_candidates = []
                for expert in self.__experts_data:
                    for sent_set in expert.get('sentences_vocab_sets', []):
                        if normalized_attentions.issubset(sent_set):
                            phase0_candidates.append(expert)
                            break
                if phase0_candidates: selected_expert = self.__random.choice(phase0_candidates)
            if selected_expert is None:
                prompt_all_words = self.__findall(r'\w+', self.__normalize_text(prompt))
                prompt_all_set = set(prompt_all_words)
                if prompt_all_set:
                    exact_matches = []
                    for expert in self.__experts_data:
                        if prompt_all_set.issubset(expert.get('full_vocab', set())): exact_matches.append(expert)
                    if exact_matches: selected_expert = self.__random.choice(exact_matches)
            if selected_expert is None:
                prompt_kw_set_raw, best_experts, max_overlap = set(raw_attentions), [], -1
                for expert in self.__experts_data:
                    overlap = len(prompt_kw_set_raw.intersection(expert['attentions']))
                    if overlap > max_overlap: max_overlap, best_experts = overlap, [expert]
                    elif overlap == max_overlap: best_experts.append(expert)
                if max_overlap <= 0: selected_expert = self.__random.choice(self.__experts_data)
                else: selected_expert = self.__random.choice(best_experts)
            self.__training_tokens = selected_expert['tokens']
            similarity_threshold, epsilon = min(1.0, max(0.0, 1.0 - temperature)), 1e-6
            if self.__token_lengths is None and self.__id_to_word: self.__build_token_lengths()
            if self.__normalization_map is None and self.__id_to_word: self.__build_normalization_map()
            prompt_tokens = list(self.__encoder(prompt))
            if len(self.__normalization_map) < len(self.__id_to_word): self.__build_normalization_map()
            generated_count, termination_chars = 0, ['.', '?', '!', ';']
            buffer_last_char, first_word_generated = prompt[-1] if prompt else '', False
            while True:
                max_window = min(self.__context_window, len(prompt_tokens))
                if max_window == 0: window_tokens = []
                else: window_tokens = prompt_tokens[-max_window:]
                window_length, stride = len(window_tokens), 1
                next_tokens, similarities = [], []
                best_global_similarity, best_global_token = -1.0, None
                if window_length == 0:
                    if len(self.__training_tokens) > 0:
                        start_idx = self.__random.randint(0, len(self.__training_tokens) - 1)
                        best_global_token = self.__training_tokens[start_idx]
                        next_tokens.append(best_global_token)
                    else: break
                else:
                    prompt_window_arr = self.__array(window_tokens, dtype=int)
                    prompt_window_norm = self.__normalization_map[prompt_window_arr]
                    limit = len(self.__training_tokens) - window_length
                    for index in range(0, limit, stride):
                        candidate_seq = self.__training_tokens[index:index + window_length]
                        next_val = self.__training_tokens[index + window_length]
                        candidate_seq_norm = self.__normalization_map[candidate_seq]
                        matches = (prompt_window_norm == candidate_seq_norm).sum()
                        similarity = matches / window_length
                        if similarity >= (similarity_threshold - epsilon):
                            next_tokens.append(int(next_val))
                            similarities.append(similarity)
                        if similarity > best_global_similarity: best_global_similarity, best_global_token = similarity, int(next_val)
                        if similarity == 1.0: break
                if not next_tokens:
                    if best_global_token is not None: selected_token = best_global_token
                    else: break
                else:
                    similarities = self.__array(similarities, dtype='float32')
                    if temperature < 0.1:
                        best_idx = similarities.argmax()
                        selected_token = next_tokens[best_idx]
                    else:
                        if similarities.sum() > 0:
                            probs = similarities ** (1 / (temperature + 0.1))
                            probs = probs / probs.sum()
                            selected_token = self.__random.choice(next_tokens, p=probs)
                        else: selected_token = self.__random.choice(next_tokens)
                token_text = self.__decoder([selected_token])
                if first_token or (buffer_last_char == '\n' and token_text.startswith(' ')): token_text = token_text.lstrip()
                first_token, should_yield = not len(token_text) > 0, True
                if not first_word_generated:
                    stripped = token_text.lstrip()
                    if not stripped and '\n' not in token_text: should_yield = False
                    else:
                        if stripped: token_text = token_text[0].upper() + token_text[1:]
                        first_word_generated = True
                if should_yield:
                    generated_count += 1
                    is_end_tag = False
                    if self.END_TAG and self.END_TAG in token_text: token_text, is_end_tag = token_text.split(self.END_TAG)[0].strip(), True
                    yield token_text
                    if is_end_tag: break
                    if generated_count > max_tokens:
                        for character in termination_chars:
                            if character in token_text:
                                if token_text.find(character) != -1: return
                        if generated_count >= max_tokens * 2: return
                if token_text: buffer_last_char = token_text[-1]
                prompt_tokens.append(selected_token)
        except Exception as error:
            if self.__show_errors:
                error_message = 'ERROR in CPUModel.infer: '+str(error)
                print(error_message)
                try: self.__print_exc() if self.__display_error_point else None
                except: pass
            return ''
    def printInference(self, prompt='', temperature=0.1, max_tokens=None):
        try:
            prompt = str(prompt).strip()
            temperature = min(1.0, max(0.0, float(temperature))) if type(temperature) in (bool, int, float) else 0.1
            max_tokens = max(1, int(max_tokens)) if type(max_tokens) in (bool, int, float) else self.__context_window
            tokens = self.infer(prompt=prompt, temperature=temperature, max_tokens=max_tokens)
            token_positions = 0
            for token in tokens:
                if token_positions < 1: token = token.lstrip()
                print(token, end='', flush=True)
                token_positions += 1
            print()
        except Exception as error:
            if self.__show_errors:
                error_message = 'ERROR in CPUModel.printInference: '+str(error)
                print(error_message)
                try: self.__print_exc() if self.__display_error_point else None
                except: pass
    def saveModel(self, model_path=''):
        try:
            saved_model = False
            model_path = str(model_path).strip()
            if not model_path: model_path = 'model.cpu'
            if not model_path.endswith('.cpu'): model_path += '.cpu'
            data_dump = {
                'experts_data': self.__experts_data,
                'fine_tuning': self.__fine_tuning,
                'token_embeddings': self.__token_embeddings,
                'word_to_id': self.__word_to_id,
                'id_to_word': self.__id_to_word,
                'context_window': self.__context_window,
                'end_tag': self.END_TAG if self.END_TAG else ''
            }
            with open(model_path, 'wb') as file: self.__dump(data_dump, file)
            saved_model = self.__exists(model_path)
            return saved_model
        except Exception as error:
            if self.__show_errors:
                error_message = 'ERROR in CPUModel.saveModel: '+str(error)
                print(error_message)
                try: self.__print_exc() if self.__display_error_point else None
                except: pass
            return False
    def loadModel(self, model_path=''):
        try:
            loaded_model = False
            model_path = str(model_path).strip()
            if not model_path: model_path = 'model.cpu'
            if not model_path.endswith('.cpu'): model_path += '.cpu'
            if not self.__exists(model_path): return loaded_model
            with open(model_path, 'rb') as file:
                data = self.__load(file)
                self.__experts_data = data.get('experts_data', [])
                self.__fine_tuning = data.get('fine_tuning', [])
                self.__token_embeddings = data['token_embeddings']
                self.__word_to_id = data.get('word_to_id', {})
                self.__id_to_word = data.get('id_to_word', {})
                self.__context_window = data.get('context_window', 1)
                self.END_TAG = data.get('end_tag', None)
                if self.END_TAG == '': self.END_TAG = None
                if 'training_tokens' in data and not self.__experts_data:
                    old_tokens = data['training_tokens']
                    self.__experts_data = [{'tokens': old_tokens, 'attentions': set(), 'full_vocab': set(), 'sentences_vocab_sets': []}]
                for expert in self.__experts_data:
                    if 'full_vocab' not in expert:
                        rebuilt_vocab = set()
                        for token in expert['tokens']:
                            word = self.__id_to_word.get(token, '')
                            if word: rebuilt_vocab.add(self.__normalize_text(word))
                        expert['full_vocab'] = rebuilt_vocab
                    if 'sentences_vocab_sets' not in expert: expert['sentences_vocab_sets'] = []
                self.__build_token_lengths()
                self.__build_normalization_map()
                if self.__experts_data: self.__training_tokens = self.__experts_data[0]['tokens']
            loaded_model = True
            return loaded_model
        except Exception as error:
            if self.__show_errors:
                error_message = 'ERROR in CPUModel.loadModel: '+str(error)
                print(error_message)
                try: self.__print_exc() if self.__display_error_point else None
                except: pass
            return False
"""
This algorithm is a proposal from Sapiens Technology®️ for a new concept of language model architecture focused on direct CPU training.
Its concept aims to establish a new type of paradigm that enables cheaper and more accessible training.
The CPUModel architecture allows for fast and efficient training on home hardware with low computational power.
It is useful for students and developers looking to create specialized models quickly, cheaply, and accurately.
Although large language models can be created with this architecture, it will perform better on small language models that are hyper-specialized.
Any alteration, disclosure, or public comment about the technical and theoretical concepts involved in this algorithm without prior authorization
from Sapiens Technology®️ is strictly prohibited and subject to legal action by our legal team.
"""
# --------------------------> A SAPIENS TECHNOLOGY®️ PRODUCTION) <--------------------------
