Scheduling & Automation

# First run: collects everything
anysite dataset collect dataset.yaml

# Subsequent runs: only collects new inputs
anysite dataset collect dataset.yaml --incremental

sources:
  - id: companies
    endpoint: /api/linkedin/search/companies
    params: { keywords: "AI startup", count: 100 }
    refresh: always    # Always re-collect (time-sensitive data)

  - id: employees
    endpoint: /api/linkedin/company/employees
    dependency: { from_source: companies, field: urn.value }
    input_key: companies
    refresh: auto      # Respect --incremental flag (default)

anysite dataset reset-cursor dataset.yaml

name: daily-monitoring
description: Daily competitor monitoring pipeline

sources:
  - id: competitor_posts
    endpoint: /api/linkedin/company/posts
    from_file: competitors.txt
    input_key: company
    parallel: 3

schedule:
  cron: "0 9 * * *"    # Every day at 9:00 AM

storage:
  format: parquet
  path: ./data/

anysite dataset schedule dataset.yaml --incremental --load-db pg

notifications:
  on_complete:
    - url: "https://hooks.slack.com/services/xxx/yyy/zzz"
  on_failure:
    - url: "https://alerts.example.com/anysite-failure"

anysite dataset history my-dataset

anysite dataset logs my-dataset --run 42

anysite dataset schedule dataset.yaml \
  --incremental \
  --load-db pg

name: lead-monitoring
description: Automated lead discovery and enrichment

sources:
  - id: search
    endpoint: /api/linkedin/search/users
    params:
      keywords: "Head of Engineering"
      count: 100
    refresh: always

  - id: profiles
    endpoint: /api/linkedin/user
    dependency: { from_source: search, field: urn.value }
    input_key: user
    parallel: 5
    rate_limit: "10/s"
    on_error: skip
    refresh: auto

  - id: enriched
    type: llm
    dependency: { from_source: profiles, field: name }
    llm:
      - type: classify
        categories: "high_priority,medium,low"
        output_column: lead_score

storage:
  format: parquet
  path: ./data/

schedule:
  cron: "0 9 * * 1"    # Every Monday at 9 AM

notifications:
  on_complete:
    - url: "https://hooks.slack.com/services/xxx"
  on_failure:
    - url: "https://alerts.example.com/fail"