# Template: HTTP Extraction with Source Validation
# Copy to your pipeline directory as extract.yaml
#
# This template demonstrates source schema validation after extraction.
# Invalid records can be skipped, quarantined (sent to DLQ), or cause failures.
#
# Usage:
#   pipeline = Pipeline.from_config_dir("path/to/pipeline")
#   result = await pipeline.run()
#   print(f"Valid records: {result.rows_transformed}")
#   print(f"Quarantined at extract: {result.rows_quarantined_extract}")

title: http_extraction_with_validation
description: Extract from HTTP API with source schema validation
version: "1.0.0"

# Source type (required)
type: http

# API URL
url: https://api.example.com/v1/orders

# HTTP method
method: GET

# Request headers
headers:
  Accept: application/json
  Authorization: Bearer ${API_TOKEN}

# Response handling
response_format: json
response_path: data.orders  # Path to data array in response

# Batch size
batch_size: 1000

# ============================================================================
# Source Schema Validation (Optional)
# ============================================================================
# Validate extracted data against a JSON/YAML schema before transformation.
# This catches malformed data early in the pipeline.

validation:
  # Path to source schema file (JSON Schema or Pydantic schema YAML)
  # Can be absolute or relative to config directory
  schema: ./source_schema.yaml
  
  # Action when validation fails:
  # - fail: Stop pipeline immediately (default)
  # - warn: Log warning but continue with all records
  # - skip: Skip invalid records silently
  # - quarantine: Send invalid records to DLQ
  on_error: quarantine
  
  # DLQ configuration (optional, overrides settings.yaml)
  # Can be true (use settings.yaml defaults), false (disable), or object
  dlq: true  # Use default DLQ from settings.yaml
  
  # Or customize per-stage:
  # dlq:
  #   enabled: true
  #   backend: file
  #   path: ./dlq/extract

# ============================================================================
# Example Source Schema (create as source_schema.yaml)
# ============================================================================
# version: "1.0.0"
# type: object
# properties:
#   order_id:
#     type: string
#     pattern: "^ORD-[0-9]{6}$"
#   customer_id:
#     type: integer
#   status:
#     type: string
#     enum: ["pending", "processing", "shipped", "delivered"]
#   total_amount:
#     type: number
#     minimum: 0
# required:
#   - order_id
#   - customer_id
#   - status
