# Template: Incremental HTTP/API Extraction
# Uses watermark to only fetch records updated since the last run.
#
# The ${__watermark__} variable is automatically injected by the pipeline
# from persisted state. On the first run it uses the initial_value below.
#
# Usage:
#   pipeline = Pipeline.from_config_dir("path/to/pipeline")
#   result = await pipeline.run()

type: http

url: https://api.example.com/v1/users

method: GET

params:
  api_key: ${API_KEY:?API_KEY is required}
  updated_since: ${__watermark__}

headers:
  Accept: application/json
  User-Agent: PyCharter-ETL/1.0

response_format: json
response_path: data

batch_size: 1000

retry:
  max_attempts: 3
  backoff_factor: 2
  retry_on_status: [429, 500, 502, 503, 504]

incremental:
  enabled: true
  watermark_field: updated_at
  initial_value: "1970-01-01T00:00:00Z"
