# Template: Incremental Database Extraction
# Uses watermark to only extract new/updated records since last run.
#
# The ${__watermark__} variable is automatically injected by the pipeline
# from persisted state. On the first run it uses the initial_value below.
#
# Usage:
#   pipeline = Pipeline.from_config_dir("path/to/pipeline")
#   result = await pipeline.run()

type: database

query: |
  SELECT id, name, email, updated_at
  FROM users
  WHERE updated_at > '${__watermark__}'
  ORDER BY updated_at ASC

connection_string: ${DATABASE_URL:?DATABASE_URL is required}

batch_size: 1000

incremental:
  enabled: true
  watermark_field: updated_at
  initial_value: "1970-01-01T00:00:00Z"
