# HuggingFace Dataset Examples

This guide provides practical examples of using the HuggingFace dataset application with CaDaR for various real-world scenarios.

## Example 1: Basic Transliteration

Convert Arabic text to Latin script in a dataset:

```python
from cadar.hf_dataset import CaDaRProcessor
from datasets import load_dataset

# Load a dataset with Arabic text
dataset = load_dataset("arabic_speech_corpus", split="train")

# Initialize the processor
processor = CaDaRProcessor(darija="Ma", batch_size=64)

# Process the dataset
processed = processor.process_dataset(
    dataset=dataset,
    text_column="text",
    target_script="latin",
    output_column="text_latin"
)

# Check the results
print(processed[0]["text"])        # Original Arabic
print(processed[0]["text_latin"])  # Transliterated Latin
```

## Example 2: Creating a Parallel Corpus

Create a bidirectional corpus with both Arabic and Latin versions:

```python
from cadar.hf_dataset import DatasetLoader, CaDaRProcessor

# Load monolingual dataset
loader = DatasetLoader()
dataset = loader.load_from_hub("arabic_news", split="train")

# Create parallel version
processor = CaDaRProcessor(darija="Ma", batch_size=32)
parallel_corpus = processor.process_bidirectional(
    dataset=dataset,
    text_column="text",
    arabic_output="text_arabic",
    latin_output="text_latin"
)

# Save the parallel corpus
parallel_corpus.save_to_disk("./parallel_corpus")
```

## Example 3: Processing Social Media Data

Handle messy social media text with special characters:

```python
from cadar.hf_dataset import CaDaRProcessor
from datasets import load_dataset

# Load social media dataset
dataset = load_dataset("arabic_tweets", split="train")

# Preprocessing function to clean up social media text
def preprocess_tweet(example):
    import re
    
    text = example["text"]
    
    # Remove URLs
    text = re.sub(r'http\S+', '', text)
    
    # Remove hashtags and mentions but keep the text
    text = re.sub(r'#([^\s]+)', r'\1', text)  # #داريجا -> داريجا
    text = re.sub(r'@([^\s]+)', r'\1', text)  # @user -> user
    
    # Store original and cleaned version
    example["original_text"] = example["text"]
    example["cleaned_text"] = text
    
    return example

# Apply preprocessing
cleaned_dataset = dataset.map(preprocess_tweet)

# Process the cleaned text
processor = CaDaRProcessor(
    darija="Ma",
    batch_size=128,
    standardize_input=True,
    detect_script=True
)

processed = processor.process_dataset(
    dataset=cleaned_dataset,
    text_column="cleaned_text",
    target_script="latin",
    output_column="text_latin"
)

# Example results
print("Original:", processed[0]["original_text"])
print("Cleaned:", processed[0]["cleaned_text"])
print("Latin:", processed[0]["text_latin"])
```

## Example 4: Large-Scale Processing

Process a very large dataset using streaming:

```python
from cadar.hf_dataset import DatasetLoader, CaDaRProcessor
import os

# For large datasets, use streaming
loader = DatasetLoader()
streaming_dataset = loader.load_streaming(
    dataset_name="massive_arabic_corpus",
    subset="default"
)

# Configure processing for large-scale
processor = CaDaRProcessor(
    darija="Ma",
    batch_size=256,
    num_proc=8,
    progress=True
)

# Output directory
output_dir = "./processed_large_dataset"
os.makedirs(output_dir, exist_ok=True)

# Process in batches and save
batch_size = 1000
batch_num = 0
current_batch = []

for example in streaming_dataset:
    current_batch.append(example)
    
    if len(current_batch) >= batch_size:
        # Convert batch to Dataset
        from datasets import Dataset
        batch_dataset = Dataset.from_list(current_batch)
        
        # Process the batch
        processed_batch = processor.process_dataset(
            dataset=batch_dataset,
            text_column="text",
            target_script="latin"
        )
        
        # Save batch
        batch_path = f"{output_dir}/batch_{batch_num:05d}.parquet"
        processed_batch.to_parquet(batch_path)
        
        print(f"Processed batch {batch_num}, saved to {batch_path}")
        
        # Reset for next batch
        current_batch = []
        batch_num += 1

# Process the remaining items if any
if current_batch:
    from datasets import Dataset
    batch_dataset = Dataset.from_list(current_batch)
    processed_batch = processor.process_dataset(
        dataset=batch_dataset,
        text_column="text",
        target_script="latin"
    )
    batch_path = f"{output_dir}/batch_{batch_num:05d}.parquet"
    processed_batch.to_parquet(batch_path)
    print(f"Processed final batch {batch_num}, saved to {batch_path}")
```

## Example 5: Quality Assessment

Assess the quality of transliteration and filter low-quality results:

```python
from cadar.hf_dataset import CaDaRProcessor
from datasets import load_dataset
import re

# Load dataset
dataset = load_dataset("mixed_script_darija", split="test")

# Process the dataset
processor = CaDaRProcessor(darija="Ma")
processed = processor.process_dataset(
    dataset=dataset,
    text_column="text",
    target_script="latin",
    output_column="text_latin"
)

# Define quality assessment function
def assess_quality(example):
    original = example["text"]
    transliterated = example["text_latin"]
    
    # 1. Script consistency check
    # Good transliteration should not contain Arabic characters
    arabic_chars = sum(1 for c in transliterated if '\u0600' <= c <= '\u06FF')
    script_consistency = 1.0 - (arabic_chars / len(transliterated)) if len(transliterated) > 0 else 0
    
    # 2. Length consistency
    # Transliteration should be similar in length to original
    length_ratio = min(len(transliterated), len(original)) / max(len(transliterated), len(original))
    length_consistency = length_ratio
    
    # 3. Special character preservation
    # Preserve important special characters like numbers, punctuation
    original_special = re.findall(r'[0-9\.,;:!?]', original)
    translit_special = re.findall(r'[0-9\.,;:!?]', transliterated)
    special_preservation = len(set(original_special) & set(translit_special)) / max(len(set(original_special)), 1)
    
    # 4. Darija-specific checks
    # Common Darija words should be properly transliterated
    darija_words = ["السلام", "كيفاش", "مزيان", "شحال", "فين"]
    darija_translit = ["salam", "kifash", "mezyan", "sh7al", "fin"]
    
    darija_score = 0
    for word, translit in zip(darija_words, darija_translit):
        if word in original and translit in transliterated:
            darija_score += 1
    darija_score = darija_score / len(darija_words)
    
    # Calculate overall quality score
    example["quality_score"] = (
        script_consistency * 0.4 +
        length_consistency * 0.2 +
        special_preservation * 0.2 +
        darija_score * 0.2
    )
    
    return example

# Apply quality assessment
assessed = processed.map(assess_quality)

# Filter by quality
high_quality = assessed.filter(lambda x: x["quality_score"] > 0.9)
medium_quality = assessed.filter(lambda x: 0.7 < x["quality_score"] <= 0.9)
low_quality = assessed.filter(lambda x: x["quality_score"] <= 0.7)

print(f"High quality: {len(high_quality)} examples")
print(f"Medium quality: {len(medium_quality)} examples")
print(f"Low quality: {len(low_quality)} examples")

# Save high-quality examples
high_quality.save_to_disk("./high_quality_transliterations")
```

## Example 6: Custom Configuration

Use a YAML configuration file for reproducible processing:

```yaml
# darija_processing_config.yaml
dataset:
  name: "arabic_speech_corpus"
  subset: "default"
  split: "train"
  cache_dir: "./cache"

processing:
  darija: "Ma"
  target_script: "latin"
  text_column: "text"
  output_column: "text_latin"
  batch_size: 64
  num_proc: 4
  standardize_input: true
  detect_script: true

output:
  format: "parquet"
  save_path: "./processed_data"
  overwrite: true

advanced:
  progress: true
  filter_func: null  # No filtering
  preprocess_callback: null  # No preprocessing
```

```python
from cadar.hf_dataset import ConfigurableProcessor

# Load configuration and process
processor = ConfigurableProcessor.from_config("darija_processing_config.yaml")
dataset = processor.load_dataset()
processed = processor.process()
processor.save()

print(f"Processed {len(processed)} examples")
```

## Example 7: Training Data Preparation

Prepare data for training an NLP model with multiple features:

```python
from cadar.hf_dataset import CaDaRProcessor
from datasets import load_dataset

# Load raw dataset
dataset = load_dataset("darija_qa", split="train")

# Initialize processor
processor = CaDaRProcessor(darija="Ma")

# Define feature engineering function
def prepare_features(example):
    # Standardize Arabic
    example["question_arabic"] = processor.standardize_dataset(
        dataset=[{"text": example["question"]}],
        text_column="text"
    )[0]["text"]
    
    # Transliterate to Latin
    example["question_latin"] = processor.process_dataset(
        dataset=[{"text": example["question"]}],
        text_column="text",
        target_script="latin"
    )[0]["text_latin"]
    
    # Same for answer
    example["answer_arabic"] = processor.standardize_dataset(
        dataset=[{"text": example["answer"]}],
        text_column="text"
    )[0]["text"]
    
    example["answer_latin"] = processor.process_dataset(
        dataset=[{"text": example["answer"]}],
        text_column="text",
        target_script="latin"
    )[0]["text_latin"]
    
    # Add length features
    example["question_length_ar"] = len(example["question_arabic"])
    example["question_length_la"] = len(example["question_latin"])
    example["answer_length_ar"] = len(example["answer_arabic"])
    example["answer_length_la"] = len(example["answer_latin"])
    
    # Add script detection (mixed vs pure)
    has_arabic = bool(re.search(r'[\u0600-\u06FF]', example["question"]))
    has_latin = bool(re.search(r'[a-zA-Z]', example["question"]))
    
    if has_arabic and has_latin:
        example["question_script_type"] = "mixed"
    elif has_arabic:
        example["question_script_type"] = "arabic"
    else:
        example["question_script_type"] = "latin"
    
    return example

# Apply feature engineering
training_data = dataset.map(prepare_features)

# Save for training
training_data.save_to_disk("./qa_training_data")

# Example of prepared features
example = training_data[0]
print("Original:", example["question"])
print("Arabic Standardized:", example["question_arabic"])
print("Latin Transliterated:", example["question_latin"])
print("Script Type:", example["question_script_type"])
```

## Example 8: Command Line Usage

Using the CLI for batch processing:

```bash
# Basic processing
cadar-dataset-process \
    --dataset-name "arabic_speech_corpus" \
    --dataset-config "default" \
    --split "train" \
    --text-column "text" \
    --target-script "latin" \
    --output-dir "./processed_data" \
    --darija "Ma"

# With custom batch size and processes
cadar-dataset-process \
    --dataset-name "large_arabic_corpus" \
    --split "train" \
    --text-column "text" \
    --target-script "latin" \
    --output-dir "./processed_data" \
    --darija "Ma" \
    --batch-size 128 \
    --num-proc 8 \
    --format "parquet"

# Using configuration file
cadar-dataset-process \
    --config "darija_processing_config.yaml"

# Creating a parallel corpus
cadar-dataset-process \
    --dataset-name "arabic_news" \
    --split "train" \
    --text-column "text" \
    --bidirectional \
    --arabic-output "text_arabic_standardized" \
    --latin-output "text_latin" \
    --output-dir "./parallel_corpus" \
    --darija "Ma"
```

## Example 9: Integration with ML Pipeline

Integrate into a machine learning pipeline:

```python
from transformers import AutoTokenizer, AutoModelForSequenceClassification
from cadar.hf_dataset import CaDaRProcessor
from datasets import load_dataset
from torch.utils.data import DataLoader

# Load model and tokenizer
model_name = "bert-base-multilingual-cased"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSequenceClassification.from_pretrained(model_name)

# Load and process dataset
dataset = load_dataset("darija_sentiment", split="train")
processor = CaDaRProcessor(darija="Ma")

# Create standardized versions for better model performance
def prepare_for_model(example):
    # Standardize text first
    standardized = processor.standardize_dataset(
        dataset=[{"text": example["text"]}],
        text_column="text"
    )[0]["text"]
    
    # Then tokenize
    tokenized = tokenizer(
        standardized,
        truncation=True,
        padding="max_length",
        max_length=128,
        return_tensors="pt"
    )
    
    # Add labels
    tokenized["labels"] = example["label"]
    
    return tokenized

# Prepare the dataset
tokenized_dataset = dataset.map(prepare_for_model, batched=False)

# Convert to PyTorch format
tokenized_dataset.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])

# Create data loader
dataloader = DataLoader(tokenized_dataset, batch_size=16, shuffle=True)

# Now ready for training
# for batch in dataloader:
#     outputs = model(**batch)
#     # ... training logic
```

## Example 10: Analyzing Script Usage

Analyze script usage patterns in a corpus:

```python
from cadar.hf_dataset import DatasetLoader
from collections import Counter
import re

# Load dataset
loader = DatasetLoader()
dataset = loader.load_from_hub("darija_social_media", split="train")

# Define script analysis function
def analyze_scripts(example):
    text = example["text"]
    
    # Count characters by script
    arabic_chars = sum(1 for c in text if '\u0600' <= c <= '\u06FF')
    latin_chars = sum(1 for c in text if 'a' <= c.lower() <= 'z')
    digits = sum(1 for c in text if c.isdigit())
    other_chars = len(text) - arabic_chars - latin_chars - digits
    
    # Calculate percentages
    total = max(len(text), 1)  # Avoid division by zero
    example["arabic_pct"] = arabic_chars / total
    example["latin_pct"] = latin_chars / total
    example["digits_pct"] = digits / total
    example["other_pct"] = other_chars / total
    
    # Determine script type
    if example["arabic_pct"] > 0.8:
        example["script_type"] = "arabic"
    elif example["latin_pct"] > 0.8:
        example["script_type"] = "latin"
    elif example["arabic_pct"] > 0.2 and example["latin_pct"] > 0.2:
        example["script_type"] = "mixed"
    else:
        example["script_type"] = "other"
    
    return example

# Apply analysis
analyzed = dataset.map(analyze_scripts)

# Get statistics
script_types = Counter(ex["script_type"] for ex in analyzed)
print("Script distribution:")
for script_type, count in script_types.items():
    print(f"  {script_type}: {count} ({count/len(analyzed)*100:.1f}%)")

# Calculate average percentages
avg_arabic_pct = sum(ex["arabic_pct"] for ex in analyzed) / len(analyzed)
avg_latin_pct = sum(ex["latin_pct"] for ex in analyzed) / len(analyzed)
print(f"\nAverage script usage:")
print(f"  Arabic: {avg_arabic_pct*100:.1f}%")
print(f"  Latin: {avg_latin_pct*100:.1f}%")

# Find examples of each type
examples = {}
for script_type in ["arabic", "latin", "mixed", "other"]:
    examples[script_type] = next(
        (ex for ex in analyzed if ex["script_type"] == script_type), 
        None
    )

for script_type, example in examples.items():
    if example:
        print(f"\n{script_type.capitalize()} example:")
        print(f"  Text: {example['text']}")
        print(f"  Arabic: {example['arabic_pct']*100:.1f}%")
        print(f"  Latin: {example['latin_pct']*100:.1f}%")
```

These examples demonstrate various ways to integrate CaDaR with HuggingFace datasets for different real-world scenarios. You can adapt them to your specific use case and requirements.
