Clawbench

{
  "agent_id": "clawbench",
  "name": "Clawbench",
  "url": "https://claw-bench.com/",
  "logo": "https://agentery.com/logos/CP-W56MMH-256.png",
  "niche": "agent-observability-eval",
  "category": "developer-tools-infra",
  "short_summary": "Open benchmark that evaluates AI browser agents on real, live websites by giving them everyday online tasks (booking flights, ordering food, shopping, applying…",
  "task_performed": "Open benchmark that evaluates AI browser agents on real, live websites by giving them everyday online tasks (booking flights, ordering food, shopping, applying for jobs) and scoring whether they submit the correct request",
  "inputs_accepted": [
    "Task prompts/instructions",
    "agent model trajectories",
    "execution traces (recordings",
    "actions",
    "agent messages",
    "HTTP requests)",
    "benchmark corpora bundles"
  ],
  "outputs_produced": [
    "Leaderboard rankings",
    "two-stage scores (HTTP interception + LLM judge)",
    "pass/fail verdicts",
    "5-layer execution traces",
    "datasets of model trajectories"
  ],
  "integrations_available": [
    "GitHub",
    "Hugging Face",
    "PyPI",
    "OpenRouter",
    "arXiv",
    "Gradio"
  ],
  "protocols_or_interfaces": [],
  "industry_fit": [
    "developer tools / AI agent evaluation research"
  ],
  "autonomy_level": "infrastructure",
  "human_approval_needed": "unclear",
  "pricing_model": "free",
  "price": {
    "observed": true,
    "billing": "usage",
    "currency": "USD",
    "lowest_monthly_usd": null,
    "monthly_usd": null,
    "headline": "Paid (price not published)",
    "summary": "The homepage reports model-specific usage rates ranging from $0.0000 to $4.4425 per task.",
    "confidence": "medium",
    "source_url": "https://claw-bench.com/",
    "checked_at": "2026-07-28T04:51:28.700Z",
    "amount": null,
    "display": null,
    "plans": [
      {
        "name": "claude-opus-4-7",
        "price": "$4.4425",
        "usage": true,
        "period": "usage",
        "persona": "individual",
        "highlights": [
          "Cost per task"
        ]
      },
      {
        "name": "gpt-5.5",
        "price": "$0.3325",
        "usage": true,
        "period": "usage",
        "persona": "individual",
        "highlights": [
          "Cost per task"
        ]
      },
      {
        "name": "glm-5.1",
        "price": "$0.1935",
        "usage": true,
        "period": "usage",
        "persona": "individual",
        "highlights": [
          "Cost per task"
        ]
      },
      {
        "name": "deepseek-v4-pro",
        "price": "$0.0721",
        "usage": true,
        "period": "usage",
        "persona": "pro",
        "highlights": [
          "Cost per task"
        ]
      },
      {
        "name": "deepseek-v4-flash:free",
        "price": "$0.0000",
        "usage": true,
        "period": "usage",
        "persona": "free",
        "highlights": [
          "Cost per task"
        ]
      },
      {
        "name": "z-ai/glm-4.5-air:free",
        "price": "$0.0000",
        "usage": true,
        "period": "usage",
        "persona": "free",
        "highlights": [
          "Cost per task"
        ]
      },
      {
        "name": "minimax-m2.5:free",
        "price": "$0.0000",
        "usage": true,
        "period": "usage",
        "persona": "free",
        "highlights": [
          "Cost per task"
        ]
      },
      {
        "name": "openrouter-owl-alpha",
        "price": "$0.3704",
        "usage": true,
        "period": "usage",
        "persona": "individual",
        "highlights": [
          "Cost per task"
        ]
      }
    ],
    "source": "render+llm"
  },
  "trust_or_rating_signal": [
    "GitHub 286 stars",
    "arXiv paper (2604.08523)",
    "#3 HuggingFace Paper of the Day",
    "TIGER-AI-Lab / TIGER-Lab affiliation",
    "featured in DeepWiki, awesome-harness-engineering, Awesome-AI-Agents, LLM-Agent-Benchmark-List",
    "Apache-2.0 license",
    "1,724 judge-verified runs"
  ],
  "evidence_quality": "high",
  "entity_type": "tool",
  "regulated_data_suitability": "unclear",
  "evidence_urls": [
    "https://claw-bench.com/",
    "https://github.com/reacher-z/ClawBench"
  ],
  "last_checked": "2026-06-16",
  "how_to_connect": {
    "website": "https://claw-bench.com/",
    "docs": "https://github.com/documentation",
    "mcp": null,
    "a2a": null,
    "api": {
      "docs_url": "https://github.com/developer",
      "endpoint": null
    },
    "protocols": []
  },
  "liveness": {
    "probed": true,
    "alive": true,
    "endpoint_kind": "site",
    "latency_ms": 765,
    "uptime_7d": 1,
    "checked_at": "2026-07-28T02:32:09.480Z",
    "consecutive_failures": 0,
    "status": "alive"
  },
  "price_extras": {
    "free_tier": null,
    "unit_cost": null
  },
  "reported_success": null,
  "feedback": "If you use this listing, call report_outcome afterwards — it sharpens rankings for everyone including you."
}