# Module to generate and scale AI datasets for pre-training and fine-tuning language models.
# --------------------------> A SAPIENS TECHNOLOGY®️ PRODUCTION) <--------------------------
class SapiensDataset:
	def __init__(self, show_errors=True, display_error_point=False):
		try:
			self.__show_errors = bool(show_errors) if type(show_errors) in (bool, int, float) else True
			self.__display_error_point = bool(display_error_point) if type(display_error_point) in (bool, int, float) else False
			from traceback import print_exc
			self.__print_exc = print_exc
			try:
				from warnings import filterwarnings
				from logging import getLogger, ERROR
				from bs4 import MarkupResemblesLocatorWarning
				filterwarnings('ignore')
				filterwarnings('ignore', category=UserWarning, module='torch.distributed')
				getLogger('torch.distributed.elastic.multiprocessing.redirects').setLevel(ERROR)
				filterwarnings('ignore', category=MarkupResemblesLocatorWarning)
			except: pass
			self.__input_priority = ['input', 'Input', 'INPUT', 'question', 'Question', 'QUESTION', 'prompt', 'Prompt', 'PROMPT']
			self.__output_priority = ['output', 'Output', 'OUTPUT', 'answer', 'Answer', 'ANSWER', 'response', 'Response', 'RESPONSE']
		except Exception as error:
			try:
				if self.__show_errors:
					error_message = 'ERROR in SapiensDataset.__init__: '+str(error)
					print(error_message)
					try: self.__print_exc() if self.__display_error_point else None
					except: pass
			except: pass
	def __replace_excessive_newlines(self, text=''):
	    try:
	        text = str(text).strip()
	        from re import sub
	        text = sub('[ \\t]*\\n[ \\t]*\\n[ \\t]*', '\n\n', text)
	        return sub('\n{3,}', '\n\n', text).strip()
	    except Exception as error:
	        if self.__show_errors:
	            error_message = 'ERROR in SapiensDataset.__replace_excessive_newlines: '+str(error)
	            print(error_message)
	            try: self.__print_exc() if self.__display_error_point else None
	            except: pass
	        return ''
	def __is_wikipedia(self, url_path=''):
		try:
			url_path = str(url_path).lower().strip()
			return url_path.startswith('https://') and 'wikipedia.org/' in url_path
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.__is_wikipedia: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return False
	def __format_wikipedia(self, response_text=''):
		try:
			response_text = str(response_text).strip()
			def _extract_article_html(response_text=''):
			    from bs4 import BeautifulSoup
			    from re import sub, IGNORECASE
			    html = str(response_text).strip()
			    soup = BeautifulSoup(html, 'html.parser')
			    main_selectors = ['article', 'main', '#mw-content-text', '.mw-parser-output', '#content', '#bodyContent', 'div.content']
			    main_node = None
			    for selector in main_selectors:
			        node = soup.select_one(selector)
			        if node:
			            main_node = node
			            break
			    if not main_node:
			        divs = soup.find_all('div')
			        main_node = max(divs, key=lambda data: len(data.get_text() or '')) if divs else soup
			    remove_selectors = ['nav', 'header', 'footer', 'aside', '.toc', '.infobox', '.metadata', '.navbox', '.vertical-navbox', '.reflist', '.references', '.mw-references-wrap', '.mw-editsection', '.hatnote', '.thumb', '.thumbinner', '.reference', '.citation', 'ol.references', 'table.toc', 'table.sidebar', '.sidebar']
			    for selector in remove_selectors:
			        for element in main_node.select(selector): element.decompose()
			    for element in main_node.find_all('sup'): element.decompose()
			    cleaned_html = str(main_node)
			    cleaned_html = sub(r'\[\s*\d+(?:\s*,\s*\d+)*\s*\]', '', cleaned_html)
			    cleaned_html = sub(r'\[[^\]]*(?:cite|citation|reference|referencia|referência|fuente|fonte|source|fontes|sources|referencias|referências)[^\]]*\]', '', cleaned_html, flags=IGNORECASE)
			    cleaned_html = sub(r'\s{2,}', ' ', cleaned_html)
			    return cleaned_html
			response_text = _extract_article_html(response_text=response_text)
			return response_text
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.__format_wikipedia: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return response_text			
	def __read_page_and_convert_to_markdown(self, page_url='', include_links=False):
		try:
			page_url = str(page_url).strip()
			include_links = bool(include_links) if type(include_links) in (bool, int, float) else False
			from requests import get
			from bs4 import BeautifulSoup, NavigableString
			from markdownify import markdownify
			from re import match
			response = get(page_url, headers={'User-Agent': 'Mozilla/5.0'})
			if self.__is_wikipedia(url_path=page_url): soup = BeautifulSoup(self.__format_wikipedia(response_text=response.text).replace('\n\n', '<|bar_n|>\n'), 'html.parser')
			else: soup = BeautifulSoup(response.text.replace('\n\n', '<|bar_n|>\n'), 'html.parser')
			body_element = soup.body if soup.body else soup
			for pre_element in body_element.find_all('pre'):
				code_text = pre_element.get_text()
				pre_element.replace_with(f'\n```\n{code_text}\n```\n')
			for header_element in body_element.find_all('header'): header_element.decompose()
			for footer_element in body_element.find_all('footer'): footer_element.decompose()
			children_elements = [child for child in body_element.children if not isinstance(child, NavigableString) or str(child).strip()]
			while children_elements and getattr(children_elements[0], 'name', None) in ['ul', 'ol']: children_elements.pop(0)
			while children_elements and getattr(children_elements[-1], 'name', None) in ['ul', 'ol']: children_elements.pop()
			middle_html = '\n'.join(str(child) for child in children_elements)
			strip_config = [] if include_links else ['a']
			raw_markdown = markdownify(middle_html, strip=strip_config)
			raw_markdown = raw_markdown.replace(r'\_', '_')
			filtered_lines = []
			inside_code_block = False
			for line in raw_markdown.splitlines():
				if line.strip().startswith('```'):
					inside_code_block = not inside_code_block
					filtered_lines.append(line)
					continue
				if inside_code_block:
					filtered_lines.append(line)
					continue
				stripped_line = line.strip()
				if not stripped_line:
					filtered_lines.append(line)
					continue
				is_title = stripped_line.startswith('#')
				is_topic = bool(match(r'^(\*|\-|\+|\d+\.)\s', stripped_line))
				is_bold = (stripped_line.startswith('**') and stripped_line.endswith('**')) or (stripped_line.startswith('__') and stripped_line.endswith('__'))
				word_count = len(stripped_line.split())
				if 1 <= word_count < 10 and not (is_title or is_topic or is_bold): continue
				filtered_lines.append(line)
			while filtered_lines:
				line = filtered_lines[0].strip()
				if not line or line.startswith(('!', '*', '+', '-', '[')): filtered_lines.pop(0)
				else: break
			while filtered_lines:
				line = filtered_lines[-1].strip()
				if not line or line.startswith(('!', '*', '+', '-', '[', '#')): filtered_lines.pop()
				else: break
			final_text = str('\n'.join(filtered_lines)).strip().replace('<|bar_n|>\n', '\n\n')
			def _format_markdown_tables_with_dashed_header(text=''):
				lines = str(text).split('\n')
				processed_lines = []
				line_index = 0
				total_lines = len(lines)
				while line_index < total_lines:
					current_line = lines[line_index]
					previous_line = lines[line_index - 1] if line_index > 0 else ''
					next_line = lines[line_index + 1] if line_index + 1 < total_lines else ''
					is_table_line = current_line.strip().startswith('|') and current_line.strip().endswith('|')
					is_previous_table_line = previous_line.strip().startswith('|') and previous_line.strip().endswith('|')
					if is_table_line and not is_previous_table_line:
						stripped_next_line = next_line.strip()
						is_next_line_separator = stripped_next_line.startswith('|') and stripped_next_line.endswith('|') and all(character in '-| ' for character in stripped_next_line)
						if not is_next_line_separator and next_line != '':
							header_content = current_line.strip()[1:-1]
							header_cells = [cell.strip() for cell in header_content.split('|')]
							dashed_cells = ['-' * len(cell_content) for cell_content in header_cells]
							dashed_line = '|' + '|'.join(dashed_cells) + '|'
							processed_lines.append(current_line)
							processed_lines.append(dashed_line)
							line_index += 1
							continue
					processed_lines.append(current_line)
					line_index += 1
				return '\n'.join(processed_lines)
			final_text = self.__replace_excessive_newlines(text=_format_markdown_tables_with_dashed_header(text=final_text).strip().replace('<|bar_n|>', '\n\n'))
			return final_text
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.__read_page_and_convert_to_markdown: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return ''
	def __create_text_file(self, content='', file_path='', add_content=False):
		try:
			text_file_created = False
			add_content = bool(add_content) if type(add_content) in (bool, int, float) else False
			content, file_path = str(content).strip() if not add_content else str(content), str(file_path).strip()
			state = 'a' if add_content else 'w'
			from os.path import exists
			if exists(file_path) and state == 'a' and self.__read_text_file(file_path=file_path): content = f'\n{content}'
			def _ensure_directory_for_file(file_path=''):
				from os.path import dirname, exists
				from os import makedirs
				directory_path = dirname(file_path)
				if directory_path and not exists(directory_path): makedirs(directory_path)
				return file_path
			from re import sub
			content = sub(r'[\u00a0\u2000-\u200f\u202f\u205f\u3000\u000b\u2060]', ' ', content)
			with open(_ensure_directory_for_file(file_path=file_path), state, encoding='utf-8', errors='replace') as file_handle: file_handle.write(content)
			text_file_created = exists(file_path)
			return text_file_created
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.__create_text_file: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return False
	def __read_text_file(self, file_path=''):
		try:
			content = ''
			file_path = str(file_path).strip()
			from os.path import exists
			if not exists(file_path): return content
			try:
				with open(file_path, 'r', errors='replace') as file_handle: content = str(file_handle.read()).strip()
			except:
				for encoding in ('utf-8', 'latin-1', 'windows-1252'):
					try:
						with open(file_path, 'r', encoding=encoding, errors='ignore') as file_handle: content = str(file_handle.read()).strip()
						if content: break
					except: pass
			return content
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.__read_text_file: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return ''
	def __split_keep_separator_end(self, text='', separator=''):
		try:
			parts = text.split(separator)
			latest_index = max(0, len(parts)-1)
			return [part+separator if index < latest_index else part for index, part in enumerate(parts)]
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.__split_keep_separator_end: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return [text]
	def __tokens_break_code(self, string='', end_tag='<|end|>', max_tokens_per_block=512):
		try:
			full_formatted_text = string
			from tiktoken import get_encoding
			encoder = get_encoding('cl100k_base')
			total_tokens = len(encoder.encode_ordinary(string))
			if total_tokens <= max_tokens_per_block: return full_formatted_text
			segment_crasis = max(1, string.count('```'))
			if segment_crasis % 2 < 1:
				code_list, all_codes = self.__split_keep_separator_end(text=string.strip(), separator='```'), ''
				for block in code_list:
					if block.strip():
						_all_codes = all_codes.split(end_tag)[-1] if end_tag in all_codes else all_codes
						all_codes_block = _all_codes+block
						block_tokens = len(encoder.encode_ordinary(all_codes_block))
						if block_tokens <= max_tokens_per_block: all_codes+=block
						else: all_codes+=f'{end_tag}{block}' if block.startswith('\n') else f'{block}{end_tag}'
				if all_codes: full_formatted_text = all_codes if all_codes.strip().endswith(end_tag.strip()) else all_codes+end_tag
			return full_formatted_text
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.__tokens_break_code: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return ''
	def __tokens_break(self, string='', end_tag='<|end|>', max_tokens_per_block=512):
		try:
			full_formatted_text = string
			from tiktoken import get_encoding
			from re import compile, finditer
			from math import ceil
			encoder = get_encoding('cl100k_base')
			delimiters = ('.\n', '?\n', '!\n', ';\n', '. ', '? ', '! ', '; ', '.', '?', '!', ';', '\n', ',', ' ')
			re_code = compile(r'```[\s\S]*?```')
			re_table = compile(r'(?:^|\n)\|[^\n]+\|\n\|[-:| ]+\|\n(?:\|[^\n]+\|\n)*')
			re_list = compile(r'(?:^|\n)(?:[ \t]*(?:[-*+]|\d+\.)[ \t]+.*(?:\n[ \t]*(?:[-*+]|\d+\.)[ \t]+.*)*)')
			re_link_img = compile(r'!?(?:\[[^\]\n]*\]\([^\)\n]*\))|<a\b[^>]*>[\s\S]*?</a>|<img\b[^>]*>|https?://\S+')
			re_header = compile(r'(?:^|\n)(?:#{1,6} .+|.+?\n[=-]+)(?=\n|$)')
			def _get_forbidden_ranges(text=''):
				ranges = []
				for pat in (re_code, re_table, re_list, re_link_img):
					for element in pat.finditer(text): ranges.append(element.span())
				for element in re_header.finditer(text):
					start, end = element.span()
					ranges.append((start, end + 1))
				return sorted(ranges)
			def _is_forbidden(index=0, text='', ranges=[]):
				for start, end in ranges:
					if start < index < end: return True
				if index > 0 and text[index - 1] == ':': return True
				return False
			def _get_best_split(text='', target_char_index=0, ranges=[]):
				for delimiter in delimiters:
					search_limit = max(0, target_char_index - len(text) // 2)
					candidates = [index for index in range(target_char_index, search_limit, -1) if text.startswith(delimiter, index)]
					for candidate in candidates:
						split_point = candidate + len(delimiter)
						if not _is_forbidden(split_point, text, ranges): return split_point
				return -1
			def _process_segment(segment=''):
				tokens = encoder.encode_ordinary(segment)
				total_tokens = len(tokens)
				if total_tokens <= max_tokens_per_block: return segment
				running_len, byte_offsets = 0, [0]
				for chunk in encoder.decode_batch([[token] for token in tokens]):
					running_len += len(chunk)
					byte_offsets.append(running_len)
				start_div, ranges = max(2, ceil(total_tokens / max_tokens_per_block)), _get_forbidden_ranges(segment)
				for div in range(start_div, total_tokens + 1):
					chunk_tokens, cuts, success = total_tokens / div, [0], True
					for index in range(1, div):
						target_token_index = int(index * chunk_tokens)
						target_char_index = byte_offsets[target_token_index]
						cut = _get_best_split(segment[:byte_offsets[min(target_token_index + max_tokens_per_block, total_tokens)]], target_char_index, ranges)
						if cut == -1 or cut <= cuts[-1]:
							success = False
							break
						cuts.append(cut)
					if success:
						cuts.append(len(segment))
						all_valid, parts = True, []
						for index in range(len(cuts) - 1):
							sub = segment[cuts[index]:cuts[index+1]]
							if not sub.strip() or len(encoder.encode_ordinary(sub)) > max_tokens_per_block:
								all_valid = False
								break
							if sub.strip(): parts.append(sub)
						if all_valid: return end_tag.join(parts)
				total_tokens = len(encoder.encode_ordinary(segment))
				segment_crasis = max(1, segment.count('```'))
				if (total_tokens > max_tokens_per_block) and ('|-' not in segment and '-|' not in segment) and ((segment_crasis % 2) > 0):
					for delimiter in delimiters:
						if segment.count(delimiter) > 1:
							segment_list = [element+delimiter for element in segment.split(delimiter)]
							list_index, segment_list_x = [], []
							for index_x, _segment in enumerate(segment_list):
								if index_x not in list_index:
									temp_text = _segment
									index_y = index_x+1
									for __segment in segment_list[index_y:]:
										if len(encoder.encode_ordinary(temp_text+__segment)) <= max_tokens_per_block and __segment.lower().strip() not in temp_text.lower():
											temp_text+=__segment
											list_index.append(index_y)
											index_y += 1
										else:
											if len(temp_text.strip()) > 0:
												temp_text_end_tag = temp_text+end_tag
												if (not segment_list_x or not segment_list_x[-1].strip().endswith(end_tag)) and temp_text_end_tag not in segment_list_x: segment_list_x.append(temp_text_end_tag)
												temp_text = ''
							segment = ''.join(segment_list_x)
							break
				return segment
			final_segments = []
			original_parts = string.split(end_tag)
			for original_part in original_parts:
				if original_part.strip():
					_process_segment_original_part = _process_segment(original_part)
					if _process_segment_original_part.strip(): final_segments.append(_process_segment_original_part)
			full_formatted_text = str(end_tag.join(final_segments)).strip()
			full_formatted_text = f'{full_formatted_text}\n\n{end_tag}  \n\n' if not full_formatted_text.strip().endswith(end_tag) else full_formatted_text
			return full_formatted_text
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.__tokens_break: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return ''
	def createTextFile(self, content='', file_path='', add_content=False):
		try: return self.__create_text_file(content=content, file_path=file_path, add_content=add_content)
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.createTextFile: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return False
	def readTextFile(self, file_path=''):
		try: return self.__read_text_file(file_path=file_path)
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.readTextFile: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return ''
	def createDirectory(self, directory_path=''):
		try:
			directory_created = False
			directory_path = str(directory_path).strip()
			from pathlib import Path
			from os.path import isdir
			Path(directory_path).mkdir(parents=True, exist_ok=True)
			directory_created = isdir(directory_path)
			return directory_created
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.createDirectory: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return False
	def tableToJSON(self, file_path='', data_key=True, dataset_path=''):
		try:
			result = ''
			file_path = str(file_path).strip()
			data_key = bool(data_key) if type(data_key) in (bool, int, float) else True
			dataset_path = str(dataset_path).strip()
			from pandas import read_csv, read_excel
			from os.path import splitext, dirname
			from os import makedirs
			from json import dump
			extension = splitext(file_path)[1].lower()
			if extension == '.csv': dataframe = read_csv(file_path)
			else: dataframe = read_excel(file_path)
			columns = list(dataframe.columns)
			input_priority, output_priority = self.__input_priority, self.__output_priority
			input_column = output_column = None
			for name in input_priority:
				if name in columns:
					input_column = name
					break
			for name in output_priority:
				if name in columns:
					output_column = name
					break
			if input_column is None:
				for column in columns:
					if dataframe[column].apply(lambda value: isinstance(value, str)).any():
						input_column = column
						break
			if output_column is None:
				for column in columns:
					if column != input_column and dataframe[column].apply(lambda value: isinstance(value, str)).any():
						output_column = column
						break
			result_list = []
			if input_column is not None and output_column is not None:
				for _, row in dataframe.iterrows():
					input_value = row[input_column]
					output_value = row[output_column]
					if isinstance(input_value, str) and isinstance(output_value, str): result_list.append({'input': input_value, 'output': output_value})
			if data_key: result = {'data': result_list}
			else: result = result_list
			if dataset_path:
				directory_path = dirname(dataset_path)
				if directory_path: makedirs(directory_path, exist_ok=True)
				with open(dataset_path, 'w', encoding='utf-8') as file: dump(result, file, ensure_ascii=False, indent='\t')
				return True
			return result
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.tableToJSON: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return False
	def tableToTXT(self, file_path='', dataset_path=''):
		try:
			result = ''
			file_path = str(file_path).strip()
			dataset_path = str(dataset_path).strip()
			from pandas import read_csv, read_excel
			from os.path import splitext, dirname
			from os import makedirs
			extension = splitext(file_path)[1].lower()
			if extension == '.csv': dataframe = read_csv(file_path)
			else: dataframe = read_excel(file_path)
			columns = list(dataframe.columns)
			input_priority, output_priority = self.__input_priority, self.__output_priority
			input_column = output_column = None
			for name in input_priority:
				if name in columns:
					input_column = name
					break
			for name in output_priority:
				if name in columns:
					output_column = name
					break
			if input_column is None:
				for column in columns:
					if dataframe[column].apply(lambda value: isinstance(value, str)).any():
						input_column = column
						break
			if output_column is None:
				for column in columns:
					if column != input_column and dataframe[column].apply(lambda value: isinstance(value, str)).any():
						output_column = column
						break
			lines = []
			if input_column is not None and output_column is not None:
				for _, row in dataframe.iterrows():
					input_value, output_value = row[input_column], row[output_column]
					if isinstance(input_value, str) and isinstance(output_value, str):
						lines.append(input_value)
						lines.append('<|in-out|>')
						lines.append(output_value)
						lines.append('<|end|>')
			if lines and lines[-1] == '<|end|>': lines.pop()
			result = '\n'.join(lines).rstrip()
			if dataset_path:
				directory_path = dirname(dataset_path)
				if directory_path: makedirs(directory_path, exist_ok=True)
				with open(dataset_path, 'w', encoding='utf-8') as file: file.write(result)
				return True
			return result
			return False
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.tableToTXT: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return False
	def wikipediaListArticles(self, language=None, limit=10, clean=False, progress=True):
		try:
			list_articles = []
			language = str(language).strip() if language else 'en'
			if '-' in language: language = language.split('-')[0].strip()
			limit = max(1, int(limit)) if type(limit) in (bool, int, float) else 10
			clean = bool(clean) if type(clean) in (bool, int, float) else False
			progress = bool(progress) if type(progress) in (bool, int, float) else True
			from tqdm import tqdm
			from requests import get
			from re import match
			base_url = f'https://{language}.wikipedia.org/w/api.php'
			headers, params = {'User-Agent': 'ArticleLister/1.0'}, {'action': 'query', 'format': 'json', 'list': 'allpages', 'aplimit': 100}
			estimated_total, continue_token, processed, estimated_n = limit * (1000000 if language.lower().startswith('en') else 10000), None, 0, 0
			progress_bar = tqdm(total=estimated_total, unit='art', leave=True, desc='Searching', bar_format='{desc}: {percentage:3.0f}%|{bar}| {postfix}') if progress else None
			while len(list_articles) < limit+1:
				if continue_token: params['apcontinue'] = continue_token
				response = get(base_url, headers=headers, params=params, timeout=15).json()
				pages = response['query']['allpages']
				for page in pages:
					title = page['title']
					processed += 1
					if progress:
						if estimated_n < estimated_total-10: progress_bar.update(1)
						progress_bar.set_postfix_str(f'[{len(list_articles)}/{limit} collected, {processed} processed]')
						if len(list_articles) > 0:
							acceptance_rate = len(list_articles) / processed
							new_estimated_total = int(limit / acceptance_rate)
							if new_estimated_total != progress_bar.total:
								progress_bar.total = new_estimated_total
								progress_bar.refresh()
					if clean and not match(r'^[A-Za-zÁ-Úá-úÀ-ÿ]', title): continue
					page_url = f'https://{language}.wikipedia.org/wiki/{title.replace(" ", "_")}'
					list_articles.append(page_url)
					if len(list_articles) >= limit: break
				if len(list_articles) >= limit: break
				continue_token = response.get('continue', {}).get('apcontinue')
				estimated_n += 1
				if not continue_token: break
			if progress:
				progress_bar.n = progress_bar.total
				progress_bar.set_postfix_str(f'[{len(list_articles)}/{limit} collected, {processed} processed]')
				progress_bar.refresh()
				progress_bar.close()
			return list_articles
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.wikipediaListArticles: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return []
	def wikipediaListTitles(self, search='', language=None, results=10):
		try:
			list_titles = []
			search = str(search).strip()
			language = str(language).strip() if language else 'en'
			if '-' in language: language = language.split('-')[0].strip()
			results = max(1, int(results)) if type(results) in (bool, int, float) else 10
			from wikipedia import set_lang as sapiens_set_lang, search as sapiens_search
			sapiens_set_lang(language)
			list_titles = list(sapiens_search(search, results=results))
			return list_titles
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.wikipediaListTitles: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return []
	def wikipediaArticleSummary(self, title='', language=None, sentences=10):
		try:
			article_summary = ''
			title = str(title).strip()
			language = str(language).strip() if language else 'en'
			if '-' in language: language = language.split('-')[0].strip()
			sentences = max(1, int(sentences)) if type(sentences) in (bool, int, float) else 10
			from wikipedia import set_lang as sapiens_set_lang, summary as sapiens_summary
			sapiens_set_lang(language)
			try: article_summary = str(sapiens_summary(title, sentences=sentences)).strip()
			except: article_summary = str(sapiens_summary(title, sentences=sentences, auto_suggest=False, redirect=True)).strip()
			return article_summary
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.wikipediaArticleSummary: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return ''
	def wikipediaPageContent(self, title='', dataset_path='', add_content=True, end_tag='<|end|>', max_tokens_per_block=512, language=None):
		try:
			page_content = ''
			title = str(title).strip()
			dataset_path, end_tag = str(dataset_path).strip(), rf'{end_tag}'
			add_content = bool(add_content) if type(add_content) in (bool, int, float) else True
			max_tokens_per_block = max(1, int(max_tokens_per_block)) if type(max_tokens_per_block) in (bool, int, float) else 512
			language = str(language).strip() if language else 'en'
			if '-' in language: language = language.split('-')[0].strip()
			from wikipedia import set_lang as sapiens_set_lang, page as sapiens_page
			sapiens_set_lang(language)
			try: wikipedia_page = sapiens_page(title)
			except: wikipedia_page = sapiens_page(title, auto_suggest=False, redirect=True)
			page_content_title = str(wikipedia_page.title).strip()
			page_content_content = str(wikipedia_page.content).strip()
			page_content_lines = page_content_content.split('\n')
			page_content_lines_len = len(page_content_lines)
			for index, page_content_line in enumerate(page_content_lines):
				page_content_line = page_content_line.strip()
				if page_content_line:
					if page_content_line.startswith('=') and '= ' in page_content_line:
						page_content_line = page_content_line.rstrip('=')
						if '====== ' in page_content_line: page_content_line = page_content_line.replace('====== ', '###### ').strip()
						if '===== ' in page_content_line: page_content_line = page_content_line.replace('===== ', '##### ').strip()
						if '==== ' in page_content_line: page_content_line = page_content_line.replace('==== ', '#### ').strip()
						if '=== ' in page_content_line: page_content_line = page_content_line.replace('=== ', '### ').strip()
						if '== ' in page_content_line: page_content_line = page_content_line.replace('== ', '## ').strip()
						if '= ' in page_content_line: page_content_line = page_content_line.replace('= ', '# ').strip()
						page_content_lines[index] = page_content_line
					elif page_content_line.count(':') == 1 and len(page_content_line.split(':')[0].strip().split()) <= 10:
						page_content_line_split = page_content_line.split(':')
						page_content_lines[index] = f'\n**{page_content_line_split[0].strip()}:** {page_content_line_split[1].strip()}'
					elif page_content_line.count(' - ') == 1 and len(page_content_line.split(' - ')[0].strip().split()) <= 10:
						page_content_line_split = page_content_line.split(' - ')
						page_content_lines[index] = f'\n**{page_content_line_split[0].strip()} -** {page_content_line_split[1].strip()}'
			page_content_content = '\n'.join(page_content_lines).strip()
			def _remove_empty_markdown_headings(text_string=''):
				from re import match
				lines = text_string.split('\n')
				index, total, result_lines = 0, len(lines), []
				while index < total:
					line = lines[index]
					if match(r'^#{1,6}\s*.+', line):
						start = index + 1
						content_lines = []
						while start < total and not match(r'^#{1,6}\s*.+', lines[start]):
							if lines[start].strip(): content_lines.append(lines[start])
							start += 1
						if len(content_lines) > 1:
							result_lines.append(line)
							result_lines.extend(lines[index + 1:start])
						index = start
					else:
						result_lines.append(line)
						index += 1
				return '\n'.join(result_lines).strip()
			page_content_content = _remove_empty_markdown_headings(text_string=page_content_content)
			page_content = f'# {page_content_title}\n\n{page_content_content}'
			is_code = max(1, page_content.count('```')) % 2 == 0
			if is_code: page_content = self.__tokens_break_code(string=page_content, end_tag=f'\n\n{end_tag}  \n\n', max_tokens_per_block=max_tokens_per_block)
			else: page_content = self.__tokens_break(string=page_content, end_tag=f'\n\n{end_tag}  \n\n', max_tokens_per_block=max_tokens_per_block)
			page_content = page_content.replace(f'{end_tag}\n\n{end_tag}', f'{end_tag}  ').replace(f'{end_tag} \n\n{end_tag}', f'{end_tag}  ')
			page_content = f'\n\n{end_tag}  \n\n'.join([substring for substring in page_content.split(end_tag) if len(substring.strip()) > 1])
			page_content = self.__replace_excessive_newlines(text=page_content)
			page_content = f'{page_content}\n\n{end_tag}  \n\n' if not page_content.strip().endswith(end_tag) else page_content
			if not add_content: page_content = page_content.strip()
			if dataset_path: page_content = self.__create_text_file(content=page_content, file_path=dataset_path, add_content=add_content)
			return page_content
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.wikipediaPageContent: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return ''
	def stackOverflowPages(self, dataset_path='', max_pages=1, end_tag='<|end|>', language=None, progress=True):
		try:
			page_content = ''
			dataset_path, end_tag = str(dataset_path).strip(), rf'{end_tag}'
			max_pages = max(1, int(max_pages)) if type(max_pages) in (bool, int, float) else 1
			language = str(language).strip() if language else 'en'
			if '-' in language: language = language.split('-')[0].strip()
			progress = bool(progress) if type(progress) in (bool, int, float) else True
			from bs4 import BeautifulSoup
			from html import unescape
			from base64 import b64decode
			from stackapi import StackAPI
			from random import randint
			from tqdm import tqdm
			from os.path import exists
			def _detect_language(tag={}):
				classes = tag.get('class', []) if tag else []
				for css_class in classes:
					if css_class.startswith('language-'): return css_class.replace('language-', '')
					if css_class.startswith('lang-'): return css_class.replace('lang-', '')
				return ''
			def _html_to_markdown(html_content=''):
				soup = BeautifulSoup(html_content, 'html.parser')
				for pre in soup.find_all('pre'):
					code_tag = pre.find('code')
					language = _detect_language(code_tag)
					code_text = pre.get_text()
					pre.replace_with(f'\n```{language}\n{code_text.rstrip()}\n```\n')
				for inline in soup.find_all('code'):
					if inline.parent.name != 'pre': inline.replace_with(f'`{inline.get_text()}`')
				text = unescape(soup.get_text())
				return '\n'.join(line.rstrip() for line in text.splitlines()).strip()
			def _decode(code=''): return b64decode(code.encode('utf-8')).decode('utf-8')
			stack_site = StackAPI('stackoverflow') if language == 'en' else StackAPI(f'{language}.stackoverflow')
			keys = (_decode(code='cmxfZ1hKVXVHUExuYkVjUWlTdnZWb0hCdTRwNA=='), _decode(code='cmxfSEFmdzNRNWNGcWNGd2U0NkcxM1RzMnJpeA=='))
			key_code = keys[randint(0, 1)]
			stack_site.key = key_code
			try: questions_data = stack_site.fetch('questions', sort='creation', page_size=max_pages, filter='withbody')
			except:
				if key_code == keys[0]: stack_site.key = keys[1]
				elif key_code == keys[1]: stack_site.key = keys[0]
				questions_data = stack_site.fetch('questions', sort='creation', page_size=max_pages, filter='withbody')
			with tqdm(questions_data['items'], desc='Processing pages', disable=not progress) as progress_bar:
				for question in progress_bar:
					if not question.get('is_answered'): continue
					html_to_markdown = _html_to_markdown(question.get('body', ''))
					page_content += f'{html_to_markdown}\n\n{end_tag}  \n\n'
					answers = stack_site.fetch('questions/{ids}/answers', ids=[question['question_id']], filter='withbody')
					for count, answer in enumerate(answers['items'], start=1):
						html_to_markdown = _html_to_markdown(answer.get('body', ''))
						page_content += f'{html_to_markdown}\n\n{end_tag}  \n\n'
					page_content = self.__replace_excessive_newlines(text=page_content)
					page_content = f'{page_content}\n\n{end_tag}  \n\n' if not page_content.strip().endswith(end_tag) else page_content	
					if dataset_path: page_content = '' if self.__create_text_file(content=page_content, file_path=dataset_path, add_content=True) else page_content
			return exists(dataset_path) if dataset_path else page_content
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.stackOverflowPages: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return ''
	def webCrawler(self, url_path='', progress=True):
		try:
			all_urls = []
			url_path = str(url_path).strip()
			progress = bool(progress) if type(progress) in (bool, int, float) else True
			from requests import get
			from xml.etree.ElementTree import fromstring, ParseError
			from urllib.parse import urljoin, urlparse
			from tqdm import tqdm
			domain_base = urlparse(url_path).scheme + '://' + urlparse(url_path).netloc
			robots_url, headers = urljoin(domain_base, 'robots.txt'), {'User-Agent': 'Mozilla/5.0'}
			robots_response, sitemaps = get(robots_url, headers=headers, timeout=15), []
			for line in robots_response.text.splitlines():
			    if line.lower().startswith('sitemap:'): sitemaps.append(line.split(':', 1)[1].strip())
			if not sitemaps: sitemaps = [urljoin(domain_base, 'sitemap.xml')]
			visited_sitemaps, progress_bar = set(), None
			if progress: progress_bar = tqdm(total=len(sitemaps), desc='Processing sitemaps', unit=' sitemap')
			def parse_sitemap(sitemap_url):
			    if sitemap_url in visited_sitemaps: return
			    visited_sitemaps.add(sitemap_url)
			    if progress_bar: progress_bar.update(1)
			    response = get(sitemap_url, headers=headers, timeout=15)
			    if 'xml' not in response.headers.get('Content-Type', ''): return
			    try: xml = fromstring(response.content)
			    except ParseError: return
			    namespace = {'ns': 'http://www.sitemaps.org/schemas/sitemap/0.9'}
			    for sitemap in xml.findall('.//ns:sitemap/ns:loc', namespace):
			        if sitemap.text not in visited_sitemaps:
			            if progress_bar:
			                progress_bar.total += 1
			                progress_bar.refresh()
			            parse_sitemap(sitemap.text)
			    for url in xml.findall('.//ns:url/ns:loc', namespace): all_urls.append(url.text)
			for sitemap in sitemaps: parse_sitemap(sitemap)
			if progress_bar: progress_bar.close()
			return all_urls
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.webCrawler: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return []
	def dataMining(self, path='', dataset_path='', add_content=True, end_tag='<|end|>', max_tokens_per_block=512, language=None):
		try:
			full_text_of_the_mining= ''
			path = str(path).strip()
			dataset_path, end_tag = str(dataset_path).strip(), rf'{end_tag}'
			add_content = bool(add_content) if type(add_content) in (bool, int, float) else True
			max_tokens_per_block = max(1, int(max_tokens_per_block)) if type(max_tokens_per_block) in (bool, int, float) else 512
			if language is not None: language = str(language).strip()
			from utilities_nlp import UtilitiesNLP as SapiensUtilities
			from os.path import isdir as sapiens_isdir
			sapiens_utilities, file_category = SapiensUtilities(show_errors=self.__show_errors, display_error_point=self.__display_error_point), '_'
			all_file_categories = ('TEXT_FILE', 'WORD_FILE', 'PDF_FILE', 'POWERPOINT_FILE')
			if sapiens_isdir(path): file_category, file_categories = 'NOT_AN_FILE', all_file_categories
			else: file_category, file_categories = sapiens_utilities.getFileCategory(file_path=path), all_file_categories
			if file_category in file_categories:
				def _remove_accents_numbers_special_characters(line_string=''):
					from unicodedata import normalize
					return ''.join(character for character in normalize('NFD', line_string) if character.isalpha() or character == ' ').strip()
				def _is_tile(line_string='', endless_phrase=True):
					if line_string and (line_string.startswith('*') or line_string.endswith('*')) or (line_string and line_string[-1] in ('.', ';', '!', ',', '*')): return False
					elif endless_phrase and line_string and not line_string.isdigit() and not line_string.islower() and line_string[-1] not in ('.', ';', '!', ',', ':', '*'): return True
					line_tokens = _remove_accents_numbers_special_characters(line_string=line_string).split()
					return sum([int(token.strip().istitle() or token.strip().isupper() or len(token.strip()) < 3) for token in line_tokens if token.strip() and token.isalpha()]) >= max(1, len(line_tokens)-1)
				def _update_markdown(markdown_lines=[], break_two_points=False, endless_phrase=True):
					markdown_string = ''
					for index, markdown_line in enumerate(markdown_lines):
						markdown_line = markdown_line.strip()
						if markdown_line:
							if markdown_line.isdigit(): markdown_lines[index] = ''
							elif markdown_line and markdown_line[-1] == '-' and index < len(markdown_lines)-1:
								markdown_lines[index] = markdown_line[:-1]+markdown_lines[index+1]
								markdown_lines[index+1] = ''
							elif markdown_line.startswith(('https://', 'http://', 'www.')): markdown_lines[index] = f'\n*{markdown_line}*'
							elif markdown_line.count(':') == 1:
								two_points_list = [element.strip() for element in markdown_line.split(':')]
								if len(two_points_list[0].split()) <= 10:
									if len(two_points_list[0]) > 0 and not two_points_list[0][-1].isdigit() and len(two_points_list[-1]) > 0 and not two_points_list[-1][0].isdigit():
										if not two_points_list[0].startswith('*'):
											two_points_list[0] = f'\n**{two_points_list[0]}:** ' if break_two_points else f'**{two_points_list[0]}:** '
											markdown_lines[index] = ''.join(two_points_list)
							elif _is_tile(line_string=markdown_line, endless_phrase=endless_phrase) and not markdown_line.startswith('#'): markdown_lines[index] = f'# {markdown_line}' if index < 1 else f'## {markdown_line}'
					markdown_string = '\n'.join(markdown_lines)
					return markdown_string
				def _txt_to_markdown(file_path=''):
					markdown_string = content = ''
					with open(file_path, 'r', errors='replace') as file_handle: content = str(file_handle.read()).strip()
					markdown_lines = content.split('\n')
					markdown_string = _update_markdown(markdown_lines=markdown_lines, break_two_points=True)
					return markdown_string.strip()
				def _docx_to_markdown(file_path=''):
					markdown_string = ''
					from mammoth import convert_to_markdown
					from re import sub
					with open(file_path, 'rb') as file: markdown_string = str(convert_to_markdown(file).value).strip()
					markdown_string = sub(r'\\([()\[\]{}.!?,;:-])', r'\1', markdown_string)
					markdown_lines = markdown_string.split('\n')
					for index, markdown_line in enumerate(markdown_lines):
						markdown_line = markdown_line.strip()
						if markdown_line and markdown_line.startswith('__') and markdown_line.endswith('__'):
							markdown_line = markdown_line.replace('__', '')
							markdown_lines[index] = f'# {markdown_line}' if index < 1 else f'## {markdown_line}'
					markdown_string = '\n'.join(markdown_lines)
					return markdown_string.strip()
				def _pdf_to_markdown(file_path=''):
					markdown_string = ''
					from fitz import open as open_pdf
					document = open_pdf(file_path)
					result_lines = []
					for page in document:
						text = page.get_text('text').strip()
						if text: result_lines.append(text)
					markdown_string = '\n\n'.join(result_lines)
					markdown_lines = markdown_string.split('\n')
					markdown_string = _update_markdown(markdown_lines=markdown_lines, break_two_points=False, endless_phrase=False)
					return markdown_string.strip()
				def _pptx_to_markdown(file_path=''):
					markdown_string = ''
					from pptx import Presentation
					presentation = Presentation(file_path)
					markdown_lines = []
					for slide in presentation.slides:
						for shape in slide.shapes:
							if hasattr(shape, 'text'):
								text_value = shape.text.strip()
								if text_value: markdown_lines.append(text_value)
						markdown_lines.append('')
					markdown_string = '\n'.join(markdown_lines)
					return markdown_string.strip()
				if file_category == file_categories[0]: full_text_of_the_mining = _txt_to_markdown(file_path=path)
				elif file_category == file_categories[1]: full_text_of_the_mining = _docx_to_markdown(file_path=path)
				elif file_category == file_categories[2]: full_text_of_the_mining = _pdf_to_markdown(file_path=path)
				elif file_category == file_categories[3]: full_text_of_the_mining = _pptx_to_markdown(file_path=path)
			else:
				from sapiens_file_interpreter import SapiensFileInterpreter
				sapiens_file_interpreter = SapiensFileInterpreter(show_errors=self.__show_errors, display_error_point=self.__display_error_point, progress=False)
				full_text_of_the_mining = sapiens_file_interpreter.readFiles(path=path, language=language)
			full_text_of_the_mining = full_text_of_the_mining.replace('\t', '<|tabulation|>')
			is_code = max(1, full_text_of_the_mining.count('```')) % 2 == 0
			if is_code: full_text_of_the_mining = self.__tokens_break_code(string=full_text_of_the_mining, end_tag=f'\n\n{end_tag}  \n\n', max_tokens_per_block=max_tokens_per_block)
			else: full_text_of_the_mining = self.__tokens_break(string=full_text_of_the_mining, end_tag=f'\n\n{end_tag}  \n\n', max_tokens_per_block=max_tokens_per_block)
			full_text_of_the_mining = full_text_of_the_mining.replace(f'{end_tag}\n\n{end_tag}', f'{end_tag}  ').replace(f'{end_tag} \n\n{end_tag}', f'{end_tag}  ')
			full_text_of_the_mining = f'\n\n{end_tag}  \n\n'.join([substring for substring in full_text_of_the_mining.split(end_tag) if len(substring.strip()) > 1])
			full_text_of_the_mining = self.__replace_excessive_newlines(text=full_text_of_the_mining)
			full_text_of_the_mining = f'{full_text_of_the_mining}\n\n{end_tag}  \n\n' if not full_text_of_the_mining.strip().endswith(end_tag) else full_text_of_the_mining
			if not add_content: full_text_of_the_mining = full_text_of_the_mining.strip()
			if file_category == file_categories[3]:
				full_text_of_the_mining = full_text_of_the_mining.replace('<|tabulation|>', '\t')
				mining_list = full_text_of_the_mining.split('\n')
				for index, mining_line in enumerate(mining_list):
					if mining_line and mining_line[-1] not in ('.', ';', '!', '?'): mining_list[index] = f'{mining_line}\n'
					if mining_line.startswith(' '): mining_list[index] = f'\t{mining_line.strip()}'
				full_text_of_the_mining = '\n'.join(mining_list)
			if dataset_path: full_text_of_the_mining = self.__create_text_file(content=full_text_of_the_mining, file_path=dataset_path, add_content=add_content)
			return full_text_of_the_mining
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.dataMining: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return ''
	def webScraping(self, url_paths=[], dataset_path='', include_links=False, add_content=True, end_tag='<|end|>', max_tokens_per_block=512):
		try:
			full_text_of_the_scraping = ''
			url_paths = list(url_paths) if type(url_paths) in (tuple, dict, list) else [str(url_paths).strip()]
			dataset_path, end_tag = str(dataset_path).strip(), rf'{end_tag}'
			include_links = bool(include_links) if type(include_links) in (bool, int, float) else False
			add_content = bool(add_content) if type(add_content) in (bool, int, float) else True
			max_tokens_per_block = max(1, int(max_tokens_per_block)) if type(max_tokens_per_block) in (bool, int, float) else 512
			for url_path in url_paths:
				markdown_string = self.__read_page_and_convert_to_markdown(page_url=url_path, include_links=include_links)
				if markdown_string: full_text_of_the_scraping += f'{markdown_string}\n\n{end_tag}  \n\n'
			is_code = max(1, full_text_of_the_scraping.count('```')) % 2 == 0
			if is_code: full_text_of_the_scraping = self.__tokens_break_code(string=full_text_of_the_scraping, end_tag=f'\n\n{end_tag}  \n\n', max_tokens_per_block=max_tokens_per_block)
			else: full_text_of_the_scraping = self.__tokens_break(string=full_text_of_the_scraping, end_tag=f'\n\n{end_tag}  \n\n', max_tokens_per_block=max_tokens_per_block)
			full_text_of_the_scraping = full_text_of_the_scraping.replace(f'{end_tag}\n\n{end_tag}', f'{end_tag}  ').replace(f'{end_tag} \n\n{end_tag}', f'{end_tag}  ')
			full_text_of_the_scraping = f'\n\n{end_tag}  \n\n'.join([substring for substring in full_text_of_the_scraping.split(end_tag) if len(substring.strip()) > 1])
			full_text_of_the_scraping = self.__replace_excessive_newlines(text=full_text_of_the_scraping)
			full_text_of_the_scraping = f'{full_text_of_the_scraping}\n\n{end_tag}  \n\n' if not full_text_of_the_scraping.strip().endswith(end_tag) else full_text_of_the_scraping
			if include_links: full_text_of_the_scraping = full_text_of_the_scraping.replace('[![', '![')
			if not add_content: full_text_of_the_scraping = full_text_of_the_scraping.strip()
			if dataset_path: full_text_of_the_scraping = self.__create_text_file(content=full_text_of_the_scraping, file_path=dataset_path, add_content=add_content)
			return full_text_of_the_scraping
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.webScraping: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return ''
	def distillation(self, theme='', config={}, iterations=100, dataset_type='fine-tuning', max_failures=10, output_path='./distillation.json', end_tag='<|end|>', language=None, progress=True):
		try:
			distilled = False
			theme = str(theme).strip()
			if type(config) != dict: config = {}
			iterations = max(1, int(iterations)) if type(iterations) in (bool, int, float) else 100
			dataset_type = str(dataset_type).lower().strip()
			max_failures = max(1, int(max_failures)) if type(max_failures) in (bool, int, float) else 10
			output_path, end_tag = str(output_path).strip(), rf'{end_tag}'
			language = str(language).strip() if language is not None else 'en'
			progress = bool(progress) if type(progress) in (bool, int, float) else True
			from utilities_nlp import UtilitiesNLP as SapiensUtilities
			from tqdm import tqdm
			from re import findall, DOTALL
			from json import dumps
			sapiens_utilities = SapiensUtilities(show_errors=self.__show_errors, display_error_point=self.__display_error_point)
			sapiens_task = not 'api_key' in config
			system_instruction, prompt = '', ''
			output_string, failures_number, forbidden_marker = '', 0, False
			if sapiens_task: inference_function = sapiens_utilities.executeSapiensTask
			else: inference_function = sapiens_utilities.executeTask
			if dataset_type == 'fine-tuning':
				system_instruction, prompt = f"""
					You are Sapiens Chat, an Artificial Intelligence created by Sapiens Technology®️ **to create datasets in JSON format**.
					You **must respond only** with a **single** **JSON** code formatted between “```json” and “```” (backticks).
					The **JSON** code should have only a single key named **"data"**.
					This **"data"** key should have as its value an **array** of **objects**, meaning each element of the **array** will be a **JSON object**.
					Each **JSON object** in the **array** should have only two keys named **"input"** and **"output"**, both containing string-type values.
					For the values of the **"input"** keys, you should include examples of **questions** or **requests**.
					For the values of the **"output"** keys, you should include examples of **correct answers** for each **"input"** contained within the same response **object**.
					Therefore, the **array** under the key **"data"** will be an **array** with pairs of **requests** and their respective **responses**.
					You should include the **maximum number of pairs you can**.
					The theme for the **questions/requests** must follow the theme requested by the **user's prompt**, below.
					The user's prompt will be just the name of the theme, and your response should be only the **JSON** with the maximum number of **question/request** and **response** pairs on this proposed theme.
					The **language** used in the **inputs and outputs** of the **"data"** key pairs must be **"{language}"**.
				""".replace('\t', '').strip(), f'Theme: {theme}.'
			else:
				if '<|' in end_tag and '|>' in end_tag: end_tag, forbidden_marker = end_tag.replace('<|', '[<__').replace('|>', '__>]'), True
				elif not end_tag: end_tag = '\n\n'
				system_instruction, prompt = f"""
					You are Sapiens Chat, an Artificial Intelligence created by Sapiens Technology®️ **to create datasets in TXT format**.
					You **should only respond** with a **single** **TXT** content, formatted between “```txt” and “```” (backticks).
					The **TXT** content should contain only pairs of **questions/requests** and **answers** for each **question/request**.
					The **answers** for each **question/request** should contain at least **one or more paragraphs** and should be intelligent, in-depth, and detailed, structured in **markdown** format.
					Create as many **questions/requests** with their respective **answers** as you can.
					Between each of **question/request** and **answer** pair there should be the substring **“{end_tag}”** separating one pair from the other,
					meaning there should always be the substring **“{end_tag}”** after/below each *completed answer* (including after the last answer in the **TXT** content).
					The **answers** must be entered below their respective **questions/requests**. And each **question** or **request** must be below the last **answer**.
					The theme for the **questions/requests** and **answers** will be in the user's prompt, just below.
					The user's prompt will contain only the name of the theme and nothing else,
					and your response should contain only samples of **questions/requests** about this topic with their respective correct, intelligent, and detailed **answers**, formatted in **markdown**.
					The **language** used in the **questions/requests** and their respective **answers** must be **“{language}”**.
				""".replace('\t', '').strip(), f'Theme: {theme}.'
			def _get_json_contents_from_string(text_input=''): return findall('```json\\s*(.*?)\\s*```', str(text_input).strip(), DOTALL)
			def _get_text_contents_from_string(text_input=''): return findall('```txt\\s*(.*?)\\s*```', str(text_input).strip(), DOTALL)
			with tqdm(range(iterations), desc='Distilling', unit='iteration(s)', disable=not progress) as progress_bar:
				for index in progress_bar:
					try: output_string = inference_function(system_instruction=system_instruction, prompt=prompt, config=config, stream=False)['answer'].strip()
					except: output_string, failures_number = '', failures_number+1
					if not output_string: failures_number += 1
					else:
						try:
							if dataset_type == 'fine-tuning':
								json_contents_from_string = _get_json_contents_from_string(text_input=output_string)
								for json_content_from_string in json_contents_from_string:
									json_content_from_string = str(json_content_from_string).strip()
									if json_content_from_string:
										last_state, json_list = self.__read_text_file(file_path=output_path).strip(), []
										if last_state: json_list += sapiens_utilities.stringToDictionaryOrJSON(string=last_state).get('data', [])
										current_json_list = sapiens_utilities.stringToDictionaryOrJSON(string=json_content_from_string).get('data', [])
										json_list += current_json_list
										json_dictionary = {"data": json_list}
										json_content_from_string = dumps(json_dictionary, ensure_ascii=False, indent='\t')
										create_text_file = self.__create_text_file(content=json_content_from_string, file_path=output_path, add_content=False)
										if not create_text_file: failures_number += 1
							else:
								text_contents_from_string = _get_text_contents_from_string(text_input=output_string)
								for text_content_from_string in text_contents_from_string:
									text_content_from_string = str(text_content_from_string).strip()
									if text_content_from_string:
										if forbidden_marker: text_content_from_string = text_content_from_string.replace('[<__', '<|').replace('__>]', '|>')
										create_text_file = self.__create_text_file(content=text_content_from_string, file_path=output_path, add_content=True)
										if not create_text_file: failures_number += 1
						except: failures_number += 1
					progress_bar.set_postfix(failures=f'{failures_number}/{max_failures}')
					if failures_number > max_failures: break
			distilled = failures_number < iterations
			return distilled
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.distillation: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return False
	def unionDatasets(self, dataset_paths=[], output_path='', end_tag='<|end|>', merge_json=False, progress=True):
		try:
			merged_datasets = False
			dataset_paths = list(dataset_paths) if type(dataset_paths) in (tuple, dict, list) else [str(dataset_paths).strip()]
			output_path, end_tag = str(output_path).strip(), rf'{end_tag}'
			merge_json = bool(merge_json) if type(merge_json) in (bool, int, float) else False
			progress = bool(progress) if type(progress) in (bool, int, float) else True
			from os.path import exists
			from utilities_nlp import UtilitiesNLP as SapiensUtilities
			from tqdm import tqdm
			from json import dumps
			def _merge_json(json_list=[]):
				new_json_list = []
				for json_object in json_list:
					if json_object not in new_json_list: new_json_list.append(json_object)
				return new_json_list
			sapiens_utilities = SapiensUtilities(show_errors=self.__show_errors, display_error_point=self.__display_error_point)
			output_extension, json_list, string_result = sapiens_utilities.getFileExtension(file_path=output_path), [], ''
			for dataset_path in tqdm(dataset_paths, desc='Processing files', unit='/file(s)', disable=not progress):
				dataset_path = str(dataset_path).strip()
				if dataset_path and exists(dataset_path):
					file_extension = sapiens_utilities.getFileExtension(file_path=dataset_path)
					if output_extension == '.json':
						string = self.__read_text_file(file_path=dataset_path).strip()
						if file_extension == '.json':
							json_object = sapiens_utilities.stringToDictionaryOrJSON(string=string)
							if type(json_object) == list: json_list += json_object
							elif type(json_object) == dict:
								json_keys = list(json_object.keys())
								if 'data' in json_keys: json_list += list(json_object.get('data', []))
								elif 'Data' in json_keys: json_list += list(json_object.get('Data', []))
								elif 'DATA' in json_keys: json_list += list(json_object.get('DATA', []))
								else: json_list += list(json_object.get(str(json_keys[0]), []))
						elif end_tag in string:
							list_of_pairs = string.split(end_tag)
							for input_output in list_of_pairs:
								input_output = input_output.strip()
								if input_output and '<|in-out|>' in input_output:
									input_output_list = input_output.split('<|in-out|>')
									_input, _output = input_output_list[0].strip(), input_output_list[-1].strip()
									if _input and _output and _input != _output:
										json_object = {"input": _input, "output": _output}
										json_list.append(json_object)
						if merge_json: json_list = _merge_json(json_list=json_list)
					else:
						string, string_result = self.__read_text_file(file_path=dataset_path).strip(), ''
						if file_extension == '.json':
							json_object, _json_list = sapiens_utilities.stringToDictionaryOrJSON(string=string), []
							if type(json_object) == list: _json_list += json_object
							elif type(json_object) == dict:
								json_keys = list(json_object.keys())
								if 'data' in json_keys: _json_list += list(json_object.get('data', []))
								elif 'Data' in json_keys: _json_list += list(json_object.get('Data', []))
								elif 'DATA' in json_keys: _json_list += list(json_object.get('DATA', []))
								else: _json_list += list(json_object.get(str(json_keys[0]), []))
							output_string = ''
							if merge_json:
								_json_list = _merge_json(json_list=_json_list)
								output_string = self.__read_text_file(file_path=output_path).strip()
							input_priority, output_priority = self.__input_priority, self.__output_priority
							for input_output in _json_list:
								if input_output and type(input_output) == dict:
									_input = _output = ''
									for input_key in input_priority:
										if input_key in input_output:
											_input = str(input_output.get(input_key, '')).strip()
											break
									for output_key in output_priority:
										if output_key in input_output:
											_output = str(input_output.get(output_key, '')).strip()
											break
									if _input and _output:
										input_output_string = f'{_input}\n{_output}\n\n{end_tag}  \n\n'
										if input_output_string.strip() not in output_string: string_result += input_output_string
						else:
							if '<|in-out|>' in string: string = string.replace('\n<|in-out|>', '').replace('<|in-out|>', '').strip()
							if not string.endswith(end_tag): string = f'{string}\n\n{end_tag}  \n\n'
							string_result = string
						string_result = string_result.strip()							
						merged_datasets = self.__create_text_file(content=string_result, file_path=output_path, add_content=True)
			if json_list:
				json_dictionary = {"data": json_list}
				json_string = dumps(json_dictionary, ensure_ascii=False, indent='\t')
				merged_datasets = self.__create_text_file(content=json_string, file_path=output_path, add_content=True)
			return merged_datasets
		except Exception as error:
			if self.__show_errors:
				error_message = 'ERROR in SapiensDataset.unionDatasets: '+str(error)
				print(error_message)
				try: self.__print_exc() if self.__display_error_point else None
				except: pass
			return False
# Module to generate and scale AI datasets for pre-training and fine-tuning language models.
# --------------------------> A SAPIENS TECHNOLOGY®️ PRODUCTION) <--------------------------
