{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Quality with DuckGuard - 10x Faster Than Pandas\n",
    "\n",
    "**DuckGuard** is a Python-native data quality tool built on DuckDB.\n",
    "\n",
    "In this notebook, we'll use DuckGuard to:\n",
    "1. Profile the Titanic dataset\n",
    "2. Get instant quality scores\n",
    "3. Detect semantic types and PII\n",
    "4. Create validation rules\n",
    "5. Detect anomalies\n",
    "\n",
    "[![GitHub](https://img.shields.io/badge/GitHub-XDataHubAI%2Fduckguard-blue)](https://github.com/XDataHubAI/duckguard)\n",
    "[![PyPI](https://img.shields.io/pypi/v/duckguard.svg)](https://pypi.org/project/duckguard/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Install DuckGuard\n",
    "!pip install duckguard -q\n",
    "print(\"DuckGuard installed!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Connect to Kaggle dataset (adjust path as needed)\n",
    "# For Titanic competition:\n",
    "# data = connect(\"/kaggle/input/titanic/train.csv\")\n",
    "# For this demo, we'll create sample data\n",
    "import pandas as pd\n",
    "\n",
    "from duckguard import connect, detect_anomalies, execute_rules, load_rules_from_string\n",
    "from duckguard.semantic import SemanticAnalyzer\n",
    "\n",
    "df = pd.DataFrame({\n",
    "    'PassengerId': range(1, 101),\n",
    "    'Survived': [0, 1] * 50,\n",
    "    'Pclass': [1, 2, 3, 1, 2] * 20,\n",
    "    'Name': [f'Passenger {i}' for i in range(1, 101)],\n",
    "    'Sex': ['male', 'female'] * 50,\n",
    "    'Age': [25, 30, None, 45, 22] * 20,\n",
    "    'Fare': [50.0, 75.5, 10.0, 200.0, 15.5] * 20,\n",
    "    'Embarked': ['S', 'C', 'Q', 'S', None] * 20\n",
    "})\n",
    "df.to_csv('titanic_sample.csv', index=False)\n",
    "\n",
    "data = connect('titanic_sample.csv')\n",
    "print(f\"Loaded {data.row_count} rows, {data.column_count} columns\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Instant Quality Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "quality = data.score()\n",
    "\n",
    "print(\"=\" * 50)\n",
    "print(\"DATA QUALITY REPORT\")\n",
    "print(\"=\" * 50)\n",
    "print(f\"\\nOverall Score: {quality.overall:.1f}/100\")\n",
    "print(f\"Grade: {quality.grade}\")\n",
    "print(\"\\nDimension Scores:\")\n",
    "print(f\"  Completeness: {quality.completeness:.1f}\")\n",
    "print(f\"  Uniqueness: {quality.uniqueness:.1f}\")\n",
    "print(f\"  Validity: {quality.validity:.1f}\")\n",
    "print(f\"  Consistency: {quality.consistency:.1f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Column Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check for missing values\n",
    "print(\"Missing Values:\")\n",
    "for col in data.columns:\n",
    "    null_pct = getattr(data, col).null_percent\n",
    "    if null_pct > 0:\n",
    "        print(f\"  {col}: {null_pct:.1f}% null\")\n",
    "\n",
    "print(\"\\nFare Statistics:\")\n",
    "print(f\"  Min: {data.Fare.min}\")\n",
    "print(f\"  Max: {data.Fare.max}\")\n",
    "print(f\"  Mean: {data.Fare.mean:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Semantic Type Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from duckguard import detect_types_for_dataset\n",
    "\n",
    "types = detect_types_for_dataset(data)\n",
    "print(\"Detected Semantic Types:\")\n",
    "for col, sem_type in types.items():\n",
    "    type_name = sem_type.value if sem_type else \"generic\"\n",
    "    print(f\"  {col}: {type_name}\")\n",
    "\n",
    "# Check for PII\n",
    "analyzer = SemanticAnalyzer()\n",
    "analysis = analyzer.analyze(data)\n",
    "if analysis.pii_columns:\n",
    "    print(f\"\\n⚠️ Potential PII in: {analysis.pii_columns}\")\n",
    "else:\n",
    "    print(\"\\n✓ No obvious PII detected\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. YAML Validation Rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rules_yaml = \"\"\"\n",
    "dataset: titanic\n",
    "rules:\n",
    "  - PassengerId is not null\n",
    "  - PassengerId is unique\n",
    "  - Survived in [0, 1]\n",
    "  - Pclass in [1, 2, 3]\n",
    "  - Age >= 0\n",
    "  - Fare >= 0\n",
    "  - Sex in ['male', 'female']\n",
    "\"\"\"\n",
    "\n",
    "rules = load_rules_from_string(rules_yaml)\n",
    "result = execute_rules(rules, dataset=data)\n",
    "\n",
    "print(f\"Results: {result.passed_count}/{result.total_checks} passed\")\n",
    "print(f\"Quality Score: {result.quality_score:.1f}%\")\n",
    "print(\"\\nDetails:\")\n",
    "for r in result.results:\n",
    "    status = \"✓ PASS\" if r.passed else \"✗ FAIL\"\n",
    "    print(f\"  [{status}] {r.check.expression}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Anomaly Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "report = detect_anomalies(data, method=\"iqr\", threshold=1.5)\n",
    "\n",
    "print(\"Anomaly Detection (IQR method):\")\n",
    "print(f\"Columns checked: {report.statistics.get('columns_checked', 0)}\")\n",
    "print(f\"Anomalies found: {report.anomaly_count}\")\n",
    "\n",
    "for a in report.anomalies:\n",
    "    status = \"⚠️ ANOMALY\" if a.is_anomaly else \"✓ OK\"\n",
    "    print(f\"  {status} {a.column}: {a.message}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Auto-Generate Rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from duckguard import generate_rules\n",
    "\n",
    "# Auto-generate rules from data\n",
    "generated = generate_rules(data, dataset_name=\"titanic\")\n",
    "print(\"Auto-Generated Rules:\")\n",
    "print(generated)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary\n",
    "\n",
    "DuckGuard provides:\n",
    "- **Instant quality scores** with A-F grades\n",
    "- **Semantic type detection** including PII\n",
    "- **YAML-based validation rules**\n",
    "- **Anomaly detection** with multiple methods\n",
    "- **10x faster** than pandas-based tools\n",
    "\n",
    "### Install & Learn More\n",
    "```bash\n",
    "pip install duckguard\n",
    "```\n",
    "\n",
    "- GitHub: https://github.com/XDataHubAI/duckguard\n",
    "- PyPI: https://pypi.org/project/duckguard/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"=\" * 70)\n",
    "print(\"DUCKGUARD 3.0 - TITANIC DATASET VALIDATION SUITE\")\n",
    "print(\"=\" * 70)\n",
    "\n",
    "validation_results = {}\n",
    "\n",
    "# 1. Basic Quality Checks\n",
    "print(\"\\n[1/4] Basic Quality Checks...\")\n",
    "validation_results['unique_ids'] = data.PassengerId.is_unique()\n",
    "validation_results['valid_survival'] = data.Survived.isin([0, 1])\n",
    "print(f\"  ✓ Unique IDs: {validation_results['unique_ids'].passed}\")\n",
    "print(f\"  ✓ Valid survival values: {validation_results['valid_survival'].passed}\")\n",
    "\n",
    "# 2. Conditional Validation\n",
    "print(\"\\n[2/4] Conditional Validation...\")\n",
    "validation_results['fare_range'] = data.Fare.between_when(\n",
    "    min_value=50.0,\n",
    "    max_value=1000.0,\n",
    "    condition=\"Pclass = 1\"\n",
    ")\n",
    "print(f\"  ✓ First class fare range: {validation_results['fare_range'].passed}\")\n",
    "\n",
    "# 3. Multi-Column Relationships\n",
    "print(\"\\n[3/4] Multi-Column Relationships...\")\n",
    "validation_results['pclass_valid'] = data.expect_column_pair_satisfy(\n",
    "    column_a=\"Pclass\",\n",
    "    column_b=\"Pclass\",\n",
    "    expression=\"Pclass IN (1, 2, 3)\",\n",
    "    threshold=1.0\n",
    ")\n",
    "print(f\"  ✓ Valid passenger class: {validation_results['pclass_valid'].passed}\")\n",
    "\n",
    "# 4. Query-Based Business Logic\n",
    "print(\"\\n[4/4] Query-Based Business Logic...\")\n",
    "validation_results['no_dups'] = data.expect_query_to_return_no_rows(\n",
    "    query=\"SELECT PassengerId, COUNT(*) FROM table GROUP BY PassengerId HAVING COUNT(*) > 1\"\n",
    ")\n",
    "validation_results['survival_rate'] = data.expect_query_result_to_be_between(\n",
    "    query=\"SELECT AVG(Survived) * 100 FROM table\",\n",
    "    min_value=20.0,\n",
    "    max_value=60.0\n",
    ")\n",
    "print(f\"  ✓ No duplicates: {validation_results['no_dups'].passed}\")\n",
    "print(f\"  ✓ Survival rate in range: {validation_results['survival_rate'].passed}\")\n",
    "\n",
    "# Summary\n",
    "print(\"\\n\" + \"=\" * 70)\n",
    "passed = sum(1 for r in validation_results.values() if r.passed)\n",
    "total = len(validation_results)\n",
    "quality_score = (passed / total) * 100\n",
    "\n",
    "print(f\"VALIDATION COMPLETE: {passed}/{total} checks passed ({quality_score:.0f}%)\")\n",
    "print(\"=\" * 70)\n",
    "\n",
    "if quality_score == 100:\n",
    "    print(\"\\n🎉 Perfect! Dataset is ready for ML modeling.\")\n",
    "elif quality_score >= 80:\n",
    "    print(\"\\n✅ Good! Minor issues detected, but dataset is usable.\")\n",
    "else:\n",
    "    print(\"\\n⚠️ Warning! Significant data quality issues detected.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": "## 🎯 Complete 3.0 Validation Pipeline\n\nCombine all features for production-grade data validation:"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Distributional tests require scipy\n",
    "!pip install scipy -q\n",
    "\n",
    "try:\n",
    "    # Test if Age follows normal distribution\n",
    "    result = data.Age.expect_distribution_normal(significance_level=0.05)\n",
    "\n",
    "    print(f\"Age follows normal distribution: {result.passed}\")\n",
    "    print(f\"P-value: {result.details.get('pvalue', 0):.4f}\")\n",
    "    print(f\"Mean age: {result.details.get('mean', 0):.1f}\")\n",
    "    print(f\"Std dev: {result.details.get('std', 0):.1f}\")\n",
    "\n",
    "    # Test if Fare distribution matches expected\n",
    "    result2 = data.Fare.expect_ks_test(distribution='expon')\n",
    "    print(f\"\\nFare follows exponential: {result2.passed}\")\n",
    "\n",
    "    # Chi-square test for categorical distribution\n",
    "    # Test if passenger class distribution is as expected\n",
    "    result3 = data.Pclass.expect_chi_square_test(\n",
    "        expected_frequencies={1: 0.25, 2: 0.25, 3: 0.50},  # Expected proportions\n",
    "        significance_level=0.05\n",
    "    )\n",
    "\n",
    "    print(f\"\\nClass distribution matches expected: {result3.passed}\")\n",
    "    print(f\"P-value: {result3.details.get('pvalue', 0):.4f}\")\n",
    "\n",
    "except ImportError:\n",
    "    print(\"scipy not available for distributional tests\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": "## 🆕 New in 3.0: Distributional Testing\n\nPerfect for ML feature validation - test if your features follow expected distributions:"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check 1: No duplicate passenger IDs\n",
    "result = data.expect_query_to_return_no_rows(\n",
    "    query=\"\"\"\n",
    "        SELECT PassengerId, COUNT(*) as cnt \n",
    "        FROM table \n",
    "        GROUP BY PassengerId \n",
    "        HAVING cnt > 1\n",
    "    \"\"\"\n",
    ")\n",
    "\n",
    "print(f\"No duplicates: {result.passed}\")\n",
    "\n",
    "# Check 2: Verify we have both survivors and non-survivors\n",
    "result2 = data.expect_query_to_return_rows(\n",
    "    query=\"SELECT * FROM table WHERE Survived = 1\"\n",
    ")\n",
    "\n",
    "print(f\"Have survivors: {result2.passed}\")\n",
    "\n",
    "# Check 3: Check survival rate is in reasonable range\n",
    "result3 = data.expect_query_result_to_be_between(\n",
    "    query=\"SELECT AVG(Survived) * 100 FROM table\",\n",
    "    min_value=20.0,  # At least 20% survived\n",
    "    max_value=60.0   # At most 60% survived\n",
    ")\n",
    "\n",
    "print(f\"Survival rate in range: {result3.passed}\")\n",
    "print(f\"Actual rate: {result3.actual_value:.1f}%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": "## 🆕 New in 3.0: Query-Based Checks\n\nWrite custom SQL to validate business logic that's hard to express otherwise:"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Business rule: Survivors (Survived=1) should be recorded\n",
    "result = data.expect_column_pair_satisfy(\n",
    "    column_a=\"Survived\",\n",
    "    column_b=\"PassengerId\",\n",
    "    expression=\"Survived IN (0, 1)\",\n",
    "    threshold=1.0  # 100% must satisfy\n",
    ")\n",
    "\n",
    "print(f\"Survival data validity: {result.passed}\")\n",
    "print(f\"Message: {result.message}\")\n",
    "\n",
    "# More complex: Validate Pclass and Fare relationship\n",
    "# Higher class (lower Pclass number) generally means higher fare\n",
    "result2 = data.expect_column_pair_satisfy(\n",
    "    column_a=\"Pclass\",\n",
    "    column_b=\"Fare\",\n",
    "    expression=\"(Pclass = 3 AND Fare < 50) OR (Pclass IN (1, 2))\",\n",
    "    threshold=0.8  # Allow 20% exceptions\n",
    ")\n",
    "\n",
    "print(f\"\\nClass-Fare relationship: {result2.passed}\")\n",
    "print(f\"Satisfaction rate: {result2.details.get('satisfaction_rate', 0):.1%}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": "## 🆕 New in 3.0: Multi-Column Validation\n\nValidate complex relationships between columns using SQL expressions:"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example: Age should not be null for adult passengers (Age >= 18)\n",
    "# (Some children may have missing ages, but adults should have them)\n",
    "result = data.Age.not_null_when(\"Age >= 18 OR Age IS NULL\")\n",
    "\n",
    "print(f\"Conditional check: {result.passed}\")\n",
    "print(f\"Details: {result.message}\")\n",
    "\n",
    "# Example 2: First class passengers (Pclass=1) should have cabin assignments\n",
    "# result2 = data.Cabin.not_null_when(\"Pclass = 1\")\n",
    "# print(f\"\\nCabin check: {result2.passed}\")\n",
    "\n",
    "# Example 3: Fare validation based on class\n",
    "result3 = data.Fare.between_when(\n",
    "    min_value=50.0,\n",
    "    max_value=1000.0,\n",
    "    condition=\"Pclass = 1\"\n",
    ")\n",
    "\n",
    "print(f\"\\nFirst class fare range: {result3.passed}\")\n",
    "print(f\"Violations: {result3.details.get('violations', 0)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": "## 🆕 New in 3.0: Conditional Expectations\n\nDuckGuard 3.0 adds powerful conditional validation - check values only when conditions are met:"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}