# Template: HTTP/API Extraction (Single Request)
# Copy to your pipeline directory as extract.yaml
#
# Usage:
#   pipeline = Pipeline.from_config("path/to/pipeline")
#   result = await pipeline.run()

title: http_extraction
description: Extract from HTTP API
version: "1.0.0"

# Source type (required)
type: http

# Option 1: Full URL
url: https://api.example.com/v1/users

# Option 2: Base URL + endpoint (uncomment to use)
# base_url: https://api.example.com
# api_endpoint: /v1/users

# HTTP method
method: GET

# Query parameters
params:
  api_key: ${API_KEY:?API_KEY is required}
  # limit: 100
  # offset: 0

# Request headers
headers:
  Accept: application/json
  User-Agent: PyCharter-ETL/1.0
  # Authorization: Bearer ${AUTH_TOKEN}

# Batch size for yielding records
batch_size: 1000

# Response handling
response_format: json
response_path: null  # e.g. "data.items" for nested response

# Retry configuration
retry:
  max_attempts: 3
  backoff_factor: 2
  retry_on_status: [429, 500, 502, 503, 504]

# Timeout configuration (seconds)
timeout:
  connect: 10
  read: 30
  write: 10
  pool: 10

# Rate limiting delay between requests (seconds)
rate_limit_delay: 0.2

# Pagination (disabled for simple extraction)
pagination:
  enabled: false
