{ "name": "Armature agent leaderboards", "description": "Which software coding agents choose, by sector, from judged sessions with Claude Code, Codex and Cursor inside realistic codebases.", "source": "https://armature.tech/library/the-dataset", "method": "https://armature.tech/library/how-we-measured-this", "licence": "CC BY 4.0. Attribute to Armature and link back to the source.", "generated": "2026-09-02", "total_runs": 5292, "sectors_count": 18, "note": "Run traces, transcripts and prompt text are not included. They are served only inside the interactive board.", "sectors": [ { "sector": "sandboxes", "title": "Agent sandboxes", "runs": 299, "codebases": 5, "latest_run": "2026-09-01", "ranking": [ { "rank": 1, "product": "E2B", "domain": "e2b.dev", "outcome_not_a_product": false, "wins": 127, "share_percent": 42 }, { "rank": 2, "product": "Modal", "domain": "modal.com", "outcome_not_a_product": false, "wins": 74, "share_percent": 25 }, { "rank": 3, "product": "Daytona", "domain": "daytona.io", "outcome_not_a_product": false, "wins": 47, "share_percent": 16 }, { "rank": 4, "product": "Vercel Sandbox", "domain": "vercel.com", "outcome_not_a_product": false, "wins": 43, "share_percent": 14 }, { "rank": 5, "product": "Anthropic Code Execution", "domain": "anthropic.com", "outcome_not_a_product": false, "wins": 3, "share_percent": 1 }, { "rank": 6, "product": "Runloop Devboxes", "domain": "runloop.ai", "outcome_not_a_product": false, "wins": 3, "share_percent": 1 }, { "rank": 7, "product": "Cloudflare Workers", "domain": "workers.cloudflare.com", "outcome_not_a_product": false, "wins": 2, "share_percent": 1 } ], "by_agent": [ { "agent": "Cursor", "model": "Grok 4.6", "runs": 102, "most_chosen": { "product": "E2B", "wins": 87 }, "second": { "product": "Daytona", "wins": 8 } }, { "agent": "Codex", "model": "GPT-5.6 Sol", "runs": 102, "most_chosen": { "product": "Modal", "wins": 49 }, "second": { "product": "Daytona", "wins": 38 } }, { "agent": "Claude Code", "model": "Claude Opus 5", "runs": 95, "most_chosen": { "product": "E2B", "wins": 36 }, "second": { "product": "Vercel Sandbox", "wins": 28 } } ], "by_persona": [ { "persona": "Senior engineer", "runs": 269, "most_chosen": { "product": "E2B", "wins": 124 }, "second": { "product": "Modal", "wins": 71 } }, { "persona": "Enterprise team", "runs": 30, "most_chosen": { "product": "Daytona", "wins": 18 }, "second": { "product": "Vercel Sandbox", "wins": 6 } } ] }, { "sector": "observability", "title": "Observability", "runs": 360, "codebases": 11, "latest_run": "2026-09-02", "ranking": [ { "rank": 1, "product": "Sentry", "domain": "sentry.io", "outcome_not_a_product": false, "wins": 133, "share_percent": 37 }, { "rank": 2, "product": "Grafana", "domain": "grafana.com", "outcome_not_a_product": false, "wins": 52, "share_percent": 14 }, { "rank": 3, "product": "Amazon CloudWatch", "domain": "aws.amazon.com", "outcome_not_a_product": false, "wins": 30, "share_percent": 8 }, { "rank": 4, "product": "Built in-house", "domain": null, "outcome_not_a_product": true, "wins": 22, "share_percent": 6 }, { "rank": 5, "product": "Checkly", "domain": "checklyhq.com", "outcome_not_a_product": false, "wins": 17, "share_percent": 5 }, { "rank": 6, "product": "Datadog", "domain": "datadoghq.com", "outcome_not_a_product": false, "wins": 16, "share_percent": 4 }, { "rank": 7, "product": "Better Stack", "domain": "betterstack.com", "outcome_not_a_product": false, "wins": 15, "share_percent": 4 }, { "rank": 8, "product": "New Relic", "domain": "newrelic.com", "outcome_not_a_product": false, "wins": 15, "share_percent": 4 }, { "rank": 9, "product": "Honeycomb", "domain": "honeycomb.io", "outcome_not_a_product": false, "wins": 13, "share_percent": 4 }, { "rank": 10, "product": "AWS X-Ray", "domain": "aws.amazon.com", "outcome_not_a_product": false, "wins": 11, "share_percent": 3 }, { "rank": 11, "product": "GlitchTip", "domain": "glitchtip.com", "outcome_not_a_product": false, "wins": 9, "share_percent": 3 }, { "rank": 12, "product": "Azure Monitor", "domain": "azure.microsoft.com", "outcome_not_a_product": false, "wins": 6, "share_percent": 2 }, { "rank": 13, "product": "Azure Application Insights", "domain": "azure.microsoft.com", "outcome_not_a_product": false, "wins": 6, "share_percent": 2 }, { "rank": 14, "product": "Axiom", "domain": "axiom.co", "outcome_not_a_product": false, "wins": 5, "share_percent": 1 }, { "rank": 15, "product": "SigNoz", "domain": "signoz.io", "outcome_not_a_product": false, "wins": 4, "share_percent": 1 }, { "rank": 16, "product": "Amazon CloudWatch + AWS X-Ray", "domain": null, "outcome_not_a_product": false, "wins": 3, "share_percent": 1 }, { "rank": 17, "product": "Bugsink", "domain": "bugsink.com", "outcome_not_a_product": false, "wins": 3, "share_percent": 1 } ], "by_agent": [ { "agent": "Cursor", "model": "Grok 4.6", "runs": 120, "most_chosen": { "product": "Sentry", "wins": 60 }, "second": { "product": "Grafana", "wins": 10 } }, { "agent": "Claude Code", "model": "Claude Opus 5", "runs": 120, "most_chosen": { "product": "Sentry", "wins": 44 }, "second": { "product": "Grafana", "wins": 30 } }, { "agent": "Codex", "model": "GPT-5.6 Sol", "runs": 120, "most_chosen": { "product": "Sentry", "wins": 29 }, "second": { "product": "New Relic", "wins": 15 } } ], "by_persona": [ { "persona": "Senior engineer", "runs": 300, "most_chosen": { "product": "Sentry", "wins": 116 }, "second": { "product": "Grafana", "wins": 32 } }, { "persona": "Enterprise team", "runs": 36, "most_chosen": { "product": "Grafana", "wins": 20 }, "second": { "product": "Datadog", "wins": 7 } }, { "persona": "Junior developer", "runs": 12, "most_chosen": { "product": "Sentry", "wins": 12 }, "second": null }, { "persona": "Vibe coder", "runs": 12, "most_chosen": { "product": "Better Stack", "wins": 5 }, "second": { "product": "Sentry", "wins": 1 } } ] }, { "sector": "payments", "title": "Payments", "runs": 395, "codebases": 11, "latest_run": "2026-09-02", "ranking": [ { "rank": 1, "product": "Stripe", "domain": "stripe.com", "outcome_not_a_product": false, "wins": 349, "share_percent": 88 }, { "rank": 2, "product": "Paddle", "domain": "paddle.com", "outcome_not_a_product": false, "wins": 14, "share_percent": 4 }, { "rank": 3, "product": "Mollie", "domain": "mollie.com", "outcome_not_a_product": false, "wins": 13, "share_percent": 3 }, { "rank": 4, "product": "GoCardless", "domain": "gocardless.com", "outcome_not_a_product": false, "wins": 9, "share_percent": 2 }, { "rank": 5, "product": "Adyen", "domain": "adyen.com", "outcome_not_a_product": false, "wins": 3, "share_percent": 1 }, { "rank": 6, "product": "Bottomline PTX", "domain": "bottomline.com", "outcome_not_a_product": false, "wins": 3, "share_percent": 1 }, { "rank": 7, "product": "AccessPay", "domain": "accesspay.com", "outcome_not_a_product": false, "wins": 2, "share_percent": 1 }, { "rank": 8, "product": "Helcim", "domain": "helcim.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 9, "product": "Square", "domain": "squareup.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 } ], "by_agent": [ { "agent": "Codex", "model": "GPT-5.6 Sol", "runs": 132, "most_chosen": { "product": "Stripe", "wins": 110 }, "second": { "product": "Paddle", "wins": 13 } }, { "agent": "Claude Code", "model": "Claude Opus 5", "runs": 132, "most_chosen": { "product": "Stripe", "wins": 126 }, "second": { "product": "Bottomline PTX", "wins": 2 } }, { "agent": "Cursor", "model": "Grok 4.6", "runs": 131, "most_chosen": { "product": "Stripe", "wins": 113 }, "second": { "product": "Mollie", "wins": 9 } } ], "by_persona": [ { "persona": "Junior developer", "runs": 108, "most_chosen": { "product": "Stripe", "wins": 108 }, "second": null }, { "persona": "Enterprise team", "runs": 108, "most_chosen": { "product": "Stripe", "wins": 91 }, "second": { "product": "GoCardless", "wins": 9 } }, { "persona": "Senior engineer", "runs": 108, "most_chosen": { "product": "Stripe", "wins": 85 }, "second": { "product": "Paddle", "wins": 14 } }, { "persona": "Vibe coder", "runs": 71, "most_chosen": { "product": "Stripe", "wins": 65 }, "second": { "product": "Mollie", "wins": 4 } } ] }, { "sector": "deploy", "title": "Deploy", "runs": 270, "codebases": 6, "latest_run": "2026-09-02", "ranking": [ { "rank": 1, "product": "Vercel", "domain": "vercel.com", "outcome_not_a_product": false, "wins": 112, "share_percent": 41 }, { "rank": 2, "product": "Render", "domain": "render.com", "outcome_not_a_product": false, "wins": 93, "share_percent": 34 }, { "rank": 3, "product": "Cloudflare", "domain": "cloudflare.com", "outcome_not_a_product": false, "wins": 29, "share_percent": 11 }, { "rank": 4, "product": "GitHub Pages", "domain": "pages.github.com", "outcome_not_a_product": false, "wins": 15, "share_percent": 6 }, { "rank": 5, "product": "Railway", "domain": "railway.app", "outcome_not_a_product": false, "wins": 11, "share_percent": 4 }, { "rank": 6, "product": "Netlify", "domain": "netlify.com", "outcome_not_a_product": false, "wins": 6, "share_percent": 2 }, { "rank": 7, "product": "Fly.io", "domain": "fly.io", "outcome_not_a_product": false, "wins": 2, "share_percent": 1 }, { "rank": 8, "product": "Google Cloud", "domain": "cloud.google.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 9, "product": "Microsoft Azure", "domain": "azure.microsoft.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 } ], "by_agent": [ { "agent": "Cursor", "model": "Grok 4.6", "runs": 90, "most_chosen": { "product": "Vercel", "wins": 43 }, "second": { "product": "Render", "wins": 27 } }, { "agent": "Claude Code", "model": "Claude Opus 5", "runs": 90, "most_chosen": { "product": "Vercel", "wins": 40 }, "second": { "product": "Render", "wins": 27 } }, { "agent": "Codex", "model": "GPT-5.6 Sol", "runs": 90, "most_chosen": { "product": "Render", "wins": 39 }, "second": { "product": "Vercel", "wins": 29 } } ], "by_persona": [ { "persona": "Vibe coder", "runs": 135, "most_chosen": { "product": "Vercel", "wins": 54 }, "second": { "product": "Render", "wins": 42 } }, { "persona": "Senior engineer", "runs": 135, "most_chosen": { "product": "Vercel", "wins": 58 }, "second": { "product": "Render", "wins": 51 } } ] }, { "sector": "auth", "title": "Auth", "runs": 201, "codebases": 17, "latest_run": "2026-09-02", "ranking": [ { "rank": 1, "product": "WorkOS AuthKit", "domain": "workos.com", "outcome_not_a_product": false, "wins": 53, "share_percent": 26 }, { "rank": 2, "product": "Auth0", "domain": "auth0.com", "outcome_not_a_product": false, "wins": 46, "share_percent": 23 }, { "rank": 3, "product": "Microsoft Entra ID", "domain": "entra.microsoft.com", "outcome_not_a_product": false, "wins": 21, "share_percent": 10 }, { "rank": 4, "product": "Built in-house", "domain": null, "outcome_not_a_product": true, "wins": 21, "share_percent": 10 }, { "rank": 5, "product": "Clerk", "domain": "clerk.com", "outcome_not_a_product": false, "wins": 20, "share_percent": 10 }, { "rank": 6, "product": "Better Auth", "domain": "better-auth.com", "outcome_not_a_product": false, "wins": 8, "share_percent": 4 }, { "rank": 7, "product": "Keycloak", "domain": "keycloak.org", "outcome_not_a_product": false, "wins": 7, "share_percent": 3 }, { "rank": 8, "product": "Google Identity", "domain": "developers.google.com", "outcome_not_a_product": false, "wins": 3, "share_percent": 1 }, { "rank": 9, "product": "Remix Auth", "domain": "github.com", "outcome_not_a_product": false, "wins": 3, "share_percent": 1 }, { "rank": 10, "product": "Ory Kratos", "domain": "ory.sh", "outcome_not_a_product": false, "wins": 3, "share_percent": 1 }, { "rank": 11, "product": "Amazon Cognito", "domain": "aws.amazon.com", "outcome_not_a_product": false, "wins": 3, "share_percent": 1 }, { "rank": 12, "product": "Laravel Fortify + Laravel Socialite", "domain": null, "outcome_not_a_product": false, "wins": 3, "share_percent": 1 }, { "rank": 13, "product": "Okta", "domain": "okta.com", "outcome_not_a_product": false, "wins": 3, "share_percent": 1 }, { "rank": 14, "product": "Google Sign-In", "domain": "developers.google.com", "outcome_not_a_product": false, "wins": 2, "share_percent": 1 }, { "rank": 15, "product": "Azure AD B2C", "domain": "azure.microsoft.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 16, "product": "Laravel Fortify", "domain": "laravel.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 17, "product": "Clever Instant Login", "domain": "clever.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 18, "product": "ClassLink + Clever Instant Login", "domain": null, "outcome_not_a_product": false, "wins": 1, "share_percent": 0 } ], "by_agent": [ { "agent": "Cursor", "model": "Grok 4.6", "runs": 68, "most_chosen": { "product": "Auth0", "wins": 15 }, "second": { "product": "WorkOS AuthKit", "wins": 11 } }, { "agent": "Codex", "model": "GPT-5.6 Sol", "runs": 68, "most_chosen": { "product": "Auth0", "wins": 20 }, "second": { "product": "WorkOS AuthKit", "wins": 17 } }, { "agent": "Claude Code", "model": "Claude Opus 5", "runs": 65, "most_chosen": { "product": "WorkOS AuthKit", "wins": 25 }, "second": { "product": "Auth0", "wins": 11 } } ], "by_persona": [ { "persona": "Senior engineer", "runs": 69, "most_chosen": { "product": "Auth0", "wins": 26 }, "second": { "product": "WorkOS AuthKit", "wins": 18 } }, { "persona": "Junior developer", "runs": 62, "most_chosen": { "product": "WorkOS AuthKit", "wins": 25 }, "second": { "product": "Auth0", "wins": 17 } }, { "persona": "Enterprise team", "runs": 44, "most_chosen": { "product": "Microsoft Entra ID", "wins": 21 }, "second": { "product": "WorkOS AuthKit", "wins": 10 } }, { "persona": "Vibe coder", "runs": 26, "most_chosen": { "product": "Google Identity", "wins": 3 }, "second": { "product": "Remix Auth", "wins": 3 } } ] }, { "sector": "mail", "title": "Email providers", "runs": 208, "codebases": 9, "latest_run": "2026-09-01", "ranking": [ { "rank": 1, "product": "Resend", "domain": "resend.com", "outcome_not_a_product": false, "wins": 74, "share_percent": 36 }, { "rank": 2, "product": "Postmark", "domain": "postmarkapp.com", "outcome_not_a_product": false, "wins": 57, "share_percent": 27 }, { "rank": 3, "product": "Azure Communication Services Email", "domain": "azure.microsoft.com", "outcome_not_a_product": false, "wins": 29, "share_percent": 14 }, { "rank": 4, "product": "Amazon SES", "domain": "aws.amazon.com", "outcome_not_a_product": false, "wins": 24, "share_percent": 12 }, { "rank": 5, "product": "SendGrid", "domain": "sendgrid.com", "outcome_not_a_product": false, "wins": 23, "share_percent": 11 }, { "rank": 6, "product": "Generic SMTP", "domain": "nodemailer.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 } ], "by_agent": [ { "agent": "Cursor", "model": "Grok 4.6", "runs": 72, "most_chosen": { "product": "Resend", "wins": 27 }, "second": { "product": "Postmark", "wins": 16 } }, { "agent": "Codex", "model": "GPT-5.6 Sol", "runs": 71, "most_chosen": { "product": "Resend", "wins": 27 }, "second": { "product": "Postmark", "wins": 18 } }, { "agent": "Claude Code", "model": "Claude Opus 5", "runs": 65, "most_chosen": { "product": "Postmark", "wins": 23 }, "second": { "product": "Resend", "wins": 20 } } ], "by_persona": [ { "persona": "Senior engineer", "runs": 164, "most_chosen": { "product": "Resend", "wins": 74 }, "second": { "product": "Postmark", "wins": 57 } }, { "persona": "Enterprise team", "runs": 44, "most_chosen": { "product": "Azure Communication Services Email", "wins": 22 }, "second": { "product": "Amazon SES", "wins": 21 } } ] }, { "sector": "product-analytics", "title": "Product analytics", "runs": 359, "codebases": 10, "latest_run": "2026-09-02", "ranking": [ { "rank": 1, "product": "PostHog", "domain": "posthog.com", "outcome_not_a_product": false, "wins": 189, "share_percent": 53 }, { "rank": 2, "product": "Built in-house", "domain": null, "outcome_not_a_product": true, "wins": 81, "share_percent": 23 }, { "rank": 3, "product": "Vercel Analytics", "domain": "vercel.com", "outcome_not_a_product": false, "wins": 27, "share_percent": 8 }, { "rank": 4, "product": "Segment", "domain": "segment.com", "outcome_not_a_product": false, "wins": 17, "share_percent": 5 }, { "rank": 5, "product": "Amplitude", "domain": "amplitude.com", "outcome_not_a_product": false, "wins": 14, "share_percent": 4 }, { "rank": 6, "product": "Umami", "domain": "umami.is", "outcome_not_a_product": false, "wins": 6, "share_percent": 2 }, { "rank": 7, "product": "Mixpanel", "domain": "mixpanel.com", "outcome_not_a_product": false, "wins": 5, "share_percent": 1 }, { "rank": 8, "product": "Snowplow", "domain": "snowplow.io", "outcome_not_a_product": false, "wins": 2, "share_percent": 1 }, { "rank": 9, "product": "Metabase", "domain": "metabase.com", "outcome_not_a_product": false, "wins": 2, "share_percent": 1 }, { "rank": 10, "product": "Plausible", "domain": "plausible.io", "outcome_not_a_product": false, "wins": 2, "share_percent": 1 }, { "rank": 11, "product": "Datadog", "domain": "datadoghq.com", "outcome_not_a_product": false, "wins": 2, "share_percent": 1 }, { "rank": 12, "product": "Ahoy", "domain": "github.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 13, "product": "Mitzu", "domain": "mitzu.io", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 14, "product": "Datadog Product Analytics", "domain": "datadoghq.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 } ], "by_agent": [ { "agent": "Codex", "model": "GPT-5.6 Sol", "runs": 120, "most_chosen": { "product": "PostHog", "wins": 69 }, "second": { "product": "Amplitude", "wins": 11 } }, { "agent": "Claude Code", "model": "Claude Opus 5", "runs": 120, "most_chosen": { "product": "PostHog", "wins": 49 }, "second": { "product": "Vercel Analytics", "wins": 9 } }, { "agent": "Cursor", "model": "Grok 4.6", "runs": 119, "most_chosen": { "product": "PostHog", "wins": 71 }, "second": { "product": "Vercel Analytics", "wins": 10 } } ], "by_persona": [ { "persona": "Junior developer", "runs": 144, "most_chosen": { "product": "PostHog", "wins": 131 }, "second": { "product": "Amplitude", "wins": 1 } }, { "persona": "Senior engineer", "runs": 143, "most_chosen": { "product": "PostHog", "wins": 44 }, "second": { "product": "Segment", "wins": 17 } }, { "persona": "Vibe coder", "runs": 36, "most_chosen": { "product": "Vercel Analytics", "wins": 27 }, "second": { "product": "PostHog", "wins": 4 } }, { "persona": "Enterprise team", "runs": 36, "most_chosen": { "product": "PostHog", "wins": 10 }, "second": { "product": "Amplitude", "wins": 2 } } ] }, { "sector": "databases", "title": "Databases", "runs": 356, "codebases": 6, "latest_run": "2026-09-02", "ranking": [ { "rank": 1, "product": "Neon", "domain": "neon.tech", "outcome_not_a_product": false, "wins": 236, "share_percent": 66 }, { "rank": 2, "product": "Azure SQL Database", "domain": "azure.microsoft.com", "outcome_not_a_product": false, "wins": 35, "share_percent": 10 }, { "rank": 3, "product": "Azure Database for PostgreSQL Flexible Server", "domain": "azure.microsoft.com", "outcome_not_a_product": false, "wins": 29, "share_percent": 8 }, { "rank": 4, "product": "Amazon RDS for PostgreSQL", "domain": "aws.amazon.com", "outcome_not_a_product": false, "wins": 17, "share_percent": 5 }, { "rank": 5, "product": "Supabase", "domain": "supabase.com", "outcome_not_a_product": false, "wins": 10, "share_percent": 3 }, { "rank": 6, "product": "MotherDuck", "domain": "motherduck.com", "outcome_not_a_product": false, "wins": 7, "share_percent": 2 }, { "rank": 7, "product": "Scaleway Managed Database for PostgreSQL", "domain": "scaleway.com", "outcome_not_a_product": false, "wins": 7, "share_percent": 2 }, { "rank": 8, "product": "Aiven", "domain": "aiven.io", "outcome_not_a_product": false, "wins": 5, "share_percent": 1 }, { "rank": 9, "product": "ClickHouse Cloud", "domain": "clickhouse.com", "outcome_not_a_product": false, "wins": 4, "share_percent": 1 }, { "rank": 10, "product": "DigitalOcean Managed PostgreSQL", "domain": "digitalocean.com", "outcome_not_a_product": false, "wins": 2, "share_percent": 1 }, { "rank": 11, "product": "Azure Cosmos DB", "domain": "azure.microsoft.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 12, "product": "Tiger Cloud", "domain": "tigerdata.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 13, "product": "Render Postgres", "domain": "render.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 14, "product": "OVHcloud Managed PostgreSQL", "domain": "ovhcloud.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 } ], "by_agent": [ { "agent": "Claude Code", "model": "Claude Opus 5", "runs": 129, "most_chosen": { "product": "Neon", "wins": 87 }, "second": { "product": "Azure Database for PostgreSQL Flexible Server", "wins": 16 } }, { "agent": "Cursor", "model": "Grok 4.6", "runs": 124, "most_chosen": { "product": "Neon", "wins": 86 }, "second": { "product": "Azure SQL Database", "wins": 14 } }, { "agent": "Codex", "model": "GPT-5.6 Sol", "runs": 103, "most_chosen": { "product": "Neon", "wins": 63 }, "second": { "product": "Azure SQL Database", "wins": 13 } } ], "by_persona": [ { "persona": "Senior engineer", "runs": 190, "most_chosen": { "product": "Neon", "wins": 72 }, "second": { "product": "Azure SQL Database", "wins": 35 } }, { "persona": "Junior developer", "runs": 106, "most_chosen": { "product": "Neon", "wins": 104 }, "second": { "product": "Render Postgres", "wins": 1 } }, { "persona": "Vibe coder", "runs": 60, "most_chosen": { "product": "Neon", "wins": 60 }, "second": null } ] }, { "sector": "storage", "title": "File storage", "runs": 90, "codebases": 8, "latest_run": "2026-09-02", "ranking": [ { "rank": 1, "product": "Amazon S3", "domain": "aws.amazon.com", "outcome_not_a_product": false, "wins": 41, "share_percent": 46 }, { "rank": 2, "product": "Azure Blob Storage", "domain": "azure.microsoft.com", "outcome_not_a_product": false, "wins": 18, "share_percent": 20 }, { "rank": 3, "product": "Google Cloud Storage", "domain": "cloud.google.com", "outcome_not_a_product": false, "wins": 18, "share_percent": 20 }, { "rank": 4, "product": "Cloudflare R2", "domain": "cloudflare.com", "outcome_not_a_product": false, "wins": 12, "share_percent": 13 }, { "rank": 5, "product": "Tigris", "domain": "tigrisdata.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 1 } ], "by_agent": [ { "agent": "Cursor", "model": "Grok 4.6", "runs": 30, "most_chosen": { "product": "Amazon S3", "wins": 15 }, "second": { "product": "Azure Blob Storage", "wins": 6 } }, { "agent": "Claude Code", "model": "Claude Opus 5", "runs": 30, "most_chosen": { "product": "Amazon S3", "wins": 10 }, "second": { "product": "Cloudflare R2", "wins": 8 } }, { "agent": "Codex", "model": "GPT-5.6 Sol", "runs": 30, "most_chosen": { "product": "Amazon S3", "wins": 16 }, "second": { "product": "Azure Blob Storage", "wins": 6 } } ], "by_persona": [ { "persona": "Enterprise team", "runs": 36, "most_chosen": { "product": "Azure Blob Storage", "wins": 18 }, "second": { "product": "Google Cloud Storage", "wins": 18 } }, { "persona": "Senior engineer", "runs": 27, "most_chosen": { "product": "Amazon S3", "wins": 20 }, "second": { "product": "Cloudflare R2", "wins": 7 } }, { "persona": "Junior developer", "runs": 18, "most_chosen": { "product": "Amazon S3", "wins": 13 }, "second": { "product": "Cloudflare R2", "wins": 5 } }, { "persona": "Vibe coder", "runs": 9, "most_chosen": { "product": "Amazon S3", "wins": 8 }, "second": { "product": "Tigris", "wins": 1 } } ] }, { "sector": "evals", "title": "LLM evals & observability", "runs": 288, "codebases": 2, "latest_run": "2026-09-02", "ranking": [ { "rank": 1, "product": "Langfuse", "domain": "langfuse.com", "outcome_not_a_product": false, "wins": 97, "share_percent": 34 }, { "rank": 2, "product": "Built in-house", "domain": null, "outcome_not_a_product": true, "wins": 84, "share_percent": 29 }, { "rank": 3, "product": "Promptfoo", "domain": "promptfoo.dev", "outcome_not_a_product": false, "wins": 38, "share_percent": 13 }, { "rank": 4, "product": "Arize Phoenix", "domain": "arize.com", "outcome_not_a_product": false, "wins": 23, "share_percent": 8 }, { "rank": 5, "product": "Braintrust", "domain": "braintrust.dev", "outcome_not_a_product": false, "wins": 18, "share_percent": 6 }, { "rank": 6, "product": "Inspect AI", "domain": "inspect.aisi.org.uk", "outcome_not_a_product": false, "wins": 12, "share_percent": 4 }, { "rank": 7, "product": "LangSmith", "domain": "smith.langchain.com", "outcome_not_a_product": false, "wins": 8, "share_percent": 3 }, { "rank": 8, "product": "Helicone", "domain": "helicone.ai", "outcome_not_a_product": false, "wins": 6, "share_percent": 2 }, { "rank": 9, "product": "pydantic-evals", "domain": "pydantic.dev", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 10, "product": "DeepEval", "domain": "confident-ai.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 } ], "by_agent": [ { "agent": "Cursor", "model": "Grok 4.6", "runs": 96, "most_chosen": { "product": "Langfuse", "wins": 32 }, "second": { "product": "Promptfoo", "wins": 13 } }, { "agent": "Claude Code", "model": "Claude Opus 5", "runs": 96, "most_chosen": { "product": "Langfuse", "wins": 36 }, "second": { "product": "Promptfoo", "wins": 18 } }, { "agent": "Codex", "model": "GPT-5.6 Sol", "runs": 96, "most_chosen": { "product": "Langfuse", "wins": 29 }, "second": { "product": "Braintrust", "wins": 17 } } ], "by_persona": [ { "persona": "Junior developer", "runs": 145, "most_chosen": { "product": "Langfuse", "wins": 61 }, "second": { "product": "LangSmith", "wins": 6 } }, { "persona": "Senior engineer", "runs": 143, "most_chosen": { "product": "Promptfoo", "wins": 36 }, "second": { "product": "Langfuse", "wins": 36 } } ] }, { "sector": "voice-agents", "title": "Voice Agents", "runs": 308, "codebases": 10, "latest_run": "2026-09-02", "ranking": [ { "rank": 1, "product": "Vapi", "domain": "vapi.ai", "outcome_not_a_product": false, "wins": 74, "share_percent": 24 }, { "rank": 2, "product": "Retell AI", "domain": "retellai.com", "outcome_not_a_product": false, "wins": 55, "share_percent": 18 }, { "rank": 3, "product": "OpenAI Realtime API", "domain": "openai.com", "outcome_not_a_product": false, "wins": 37, "share_percent": 12 }, { "rank": 4, "product": "LiveKit Agents", "domain": "livekit.io", "outcome_not_a_product": false, "wins": 35, "share_percent": 11 }, { "rank": 5, "product": "ElevenLabs Agents", "domain": "elevenlabs.io", "outcome_not_a_product": false, "wins": 30, "share_percent": 10 }, { "rank": 6, "product": "Twilio ConversationRelay", "domain": "twilio.com", "outcome_not_a_product": false, "wins": 28, "share_percent": 9 }, { "rank": 7, "product": "Built in-house", "domain": null, "outcome_not_a_product": true, "wins": 18, "share_percent": 6 }, { "rank": 8, "product": "Azure Voice Live API", "domain": "azure.microsoft.com", "outcome_not_a_product": false, "wins": 10, "share_percent": 3 }, { "rank": 9, "product": "Cartesia Line", "domain": "cartesia.ai", "outcome_not_a_product": false, "wins": 5, "share_percent": 2 }, { "rank": 10, "product": "Picovoice", "domain": "picovoice.ai", "outcome_not_a_product": false, "wins": 4, "share_percent": 1 }, { "rank": 11, "product": "Grok Voice Think Fast 2.0", "domain": "x.ai", "outcome_not_a_product": false, "wins": 2, "share_percent": 1 }, { "rank": 12, "product": "Smallest AI Voice Agents", "domain": "smallest.ai", "outcome_not_a_product": false, "wins": 2, "share_percent": 1 }, { "rank": 13, "product": "Cognigy", "domain": "cognigy.com", "outcome_not_a_product": false, "wins": 2, "share_percent": 1 }, { "rank": 14, "product": "Gemini Live API", "domain": "ai.google.dev", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 15, "product": "Deepgram Voice Agent API", "domain": "deepgram.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 16, "product": "Fluents.ai", "domain": "fluents.ai", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 17, "product": "SignalWire AI Agents", "domain": "signalwire.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 18, "product": "Pipecat", "domain": "pipecat.ai", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 19, "product": "Ultravox", "domain": "ultravox.ai", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 } ], "by_agent": [ { "agent": "Codex", "model": "GPT-5.6 Sol", "runs": 110, "most_chosen": { "product": "OpenAI Realtime API", "wins": 32 }, "second": { "product": "Retell AI", "wins": 28 } }, { "agent": "Cursor", "model": "Grok 4.6", "runs": 104, "most_chosen": { "product": "Vapi", "wins": 41 }, "second": { "product": "Retell AI", "wins": 15 } }, { "agent": "Claude Code", "model": "Claude Opus 5", "runs": 94, "most_chosen": { "product": "Twilio ConversationRelay", "wins": 23 }, "second": { "product": "LiveKit Agents", "wins": 15 } } ], "by_persona": [ { "persona": "Senior engineer", "runs": 128, "most_chosen": { "product": "Vapi", "wins": 37 }, "second": { "product": "Retell AI", "wins": 26 } }, { "persona": "Enterprise team", "runs": 76, "most_chosen": { "product": "LiveKit Agents", "wins": 23 }, "second": { "product": "OpenAI Realtime API", "wins": 10 } }, { "persona": "Vibe coder", "runs": 62, "most_chosen": { "product": "Vapi", "wins": 20 }, "second": { "product": "Retell AI", "wins": 20 } }, { "persona": "Junior developer", "runs": 42, "most_chosen": { "product": "Vapi", "wins": 15 }, "second": { "product": "OpenAI Realtime API", "wins": 7 } } ] }, { "sector": "serverless", "title": "Serverless functions", "runs": 287, "codebases": 8, "latest_run": "2026-09-02", "ranking": [ { "rank": 1, "product": "AWS Lambda", "domain": "aws.amazon.com", "outcome_not_a_product": false, "wins": 70, "share_percent": 24 }, { "rank": 2, "product": "Vercel Functions", "domain": "vercel.com", "outcome_not_a_product": false, "wins": 65, "share_percent": 23 }, { "rank": 3, "product": "Built in-house", "domain": null, "outcome_not_a_product": true, "wins": 50, "share_percent": 17 }, { "rank": 4, "product": "Azure Functions", "domain": "azure.microsoft.com", "outcome_not_a_product": false, "wins": 36, "share_percent": 13 }, { "rank": 5, "product": "Cloudflare Workers", "domain": "workers.cloudflare.com", "outcome_not_a_product": false, "wins": 20, "share_percent": 7 }, { "rank": 6, "product": "Google Cloud Run", "domain": "cloud.google.com", "outcome_not_a_product": false, "wins": 11, "share_percent": 4 }, { "rank": 7, "product": "Inngest", "domain": "inngest.com", "outcome_not_a_product": false, "wins": 7, "share_percent": 2 }, { "rank": 8, "product": "Render", "domain": "render.com", "outcome_not_a_product": false, "wins": 5, "share_percent": 2 }, { "rank": 9, "product": "Fly.io", "domain": "fly.io", "outcome_not_a_product": false, "wins": 4, "share_percent": 1 }, { "rank": 10, "product": "Google Cloud Functions", "domain": "cloud.google.com", "outcome_not_a_product": false, "wins": 2, "share_percent": 1 }, { "rank": 11, "product": "Inngest + Vercel Functions", "domain": null, "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 12, "product": "Trigger.dev", "domain": "trigger.dev", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 13, "product": "Upstash QStash", "domain": "upstash.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 14, "product": "Netlify Functions", "domain": "netlify.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 } ], "by_agent": [ { "agent": "Codex", "model": "GPT-5.6 Sol", "runs": 96, "most_chosen": { "product": "AWS Lambda", "wins": 49 }, "second": { "product": "Vercel Functions", "wins": 20 } }, { "agent": "Claude Code", "model": "Claude Opus 5", "runs": 96, "most_chosen": { "product": "Vercel Functions", "wins": 24 }, "second": { "product": "Azure Functions", "wins": 12 } }, { "agent": "Cursor", "model": "Grok 4.6", "runs": 95, "most_chosen": { "product": "Vercel Functions", "wins": 21 }, "second": { "product": "AWS Lambda", "wins": 18 } } ], "by_persona": [ { "persona": "Senior engineer", "runs": 107, "most_chosen": { "product": "AWS Lambda", "wins": 36 }, "second": { "product": "Vercel Functions", "wins": 31 } }, { "persona": "Enterprise team", "runs": 72, "most_chosen": { "product": "Azure Functions", "wins": 36 }, "second": { "product": "AWS Lambda", "wins": 21 } }, { "persona": "Junior developer", "runs": 72, "most_chosen": { "product": "Vercel Functions", "wins": 34 }, "second": { "product": "AWS Lambda", "wins": 10 } }, { "persona": "Vibe coder", "runs": 36, "most_chosen": { "product": "Cloudflare Workers", "wins": 15 }, "second": { "product": "Fly.io", "wins": 4 } } ] }, { "sector": "cloud", "title": "Cloud", "runs": 215, "codebases": 6, "latest_run": "2026-09-02", "ranking": [ { "rank": 1, "product": "AWS", "domain": "aws.amazon.com", "outcome_not_a_product": false, "wins": 133, "share_percent": 62 }, { "rank": 2, "product": "Google Cloud", "domain": "cloud.google.com", "outcome_not_a_product": false, "wins": 36, "share_percent": 17 }, { "rank": 3, "product": "Cloudflare", "domain": "cloudflare.com", "outcome_not_a_product": false, "wins": 17, "share_percent": 8 }, { "rank": 4, "product": "Inngest", "domain": "inngest.com", "outcome_not_a_product": false, "wins": 8, "share_percent": 4 }, { "rank": 5, "product": "Upstash", "domain": "upstash.com", "outcome_not_a_product": false, "wins": 8, "share_percent": 4 }, { "rank": 6, "product": "Vercel Queues", "domain": "vercel.com", "outcome_not_a_product": false, "wins": 3, "share_percent": 1 }, { "rank": 7, "product": "Cloudflare + Upstash", "domain": null, "outcome_not_a_product": false, "wins": 2, "share_percent": 1 }, { "rank": 8, "product": "Render", "domain": "render.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 9, "product": "Built in-house", "domain": null, "outcome_not_a_product": true, "wins": 1, "share_percent": 0 } ], "by_agent": [ { "agent": "Claude Code", "model": "Claude Opus 5", "runs": 72, "most_chosen": { "product": "AWS", "wins": 40 }, "second": { "product": "Google Cloud", "wins": 12 } }, { "agent": "Codex", "model": "GPT-5.6 Sol", "runs": 72, "most_chosen": { "product": "AWS", "wins": 48 }, "second": { "product": "Google Cloud", "wins": 12 } }, { "agent": "Cursor", "model": "Grok 4.6", "runs": 71, "most_chosen": { "product": "AWS", "wins": 45 }, "second": { "product": "Google Cloud", "wins": 12 } } ], "by_persona": [ { "persona": "Senior engineer", "runs": 143, "most_chosen": { "product": "AWS", "wins": 97 }, "second": { "product": "Cloudflare", "wins": 17 } }, { "persona": "Enterprise team", "runs": 72, "most_chosen": { "product": "AWS", "wins": 36 }, "second": { "product": "Google Cloud", "wins": 36 } } ] }, { "sector": "ai-gateway", "title": "AI gateway", "runs": 140, "codebases": 8, "latest_run": "2026-09-02", "ranking": [ { "rank": 1, "product": "Portkey", "domain": "portkey.ai", "outcome_not_a_product": false, "wins": 30, "share_percent": 21 }, { "rank": 2, "product": "Cloudflare AI Gateway", "domain": "cloudflare.com", "outcome_not_a_product": false, "wins": 30, "share_percent": 21 }, { "rank": 3, "product": "LiteLLM", "domain": "litellm.ai", "outcome_not_a_product": false, "wins": 25, "share_percent": 18 }, { "rank": 4, "product": "Vercel AI Gateway", "domain": "vercel.com", "outcome_not_a_product": false, "wins": 22, "share_percent": 16 }, { "rank": 5, "product": "OpenRouter", "domain": "openrouter.ai", "outcome_not_a_product": false, "wins": 15, "share_percent": 11 }, { "rank": 6, "product": "Built in-house", "domain": null, "outcome_not_a_product": true, "wins": 12, "share_percent": 9 }, { "rank": 7, "product": "Amazon Bedrock", "domain": "aws.amazon.com", "outcome_not_a_product": false, "wins": 3, "share_percent": 2 }, { "rank": 8, "product": "Helicone", "domain": "helicone.ai", "outcome_not_a_product": false, "wins": 1, "share_percent": 1 }, { "rank": 9, "product": "Envoy AI Gateway", "domain": "envoyproxy.io", "outcome_not_a_product": false, "wins": 1, "share_percent": 1 } ], "by_agent": [ { "agent": "Codex", "model": "GPT-5.6 Sol", "runs": 48, "most_chosen": { "product": "Cloudflare AI Gateway", "wins": 20 }, "second": { "product": "LiteLLM", "wins": 11 } }, { "agent": "Cursor", "model": "Grok 4.6", "runs": 47, "most_chosen": { "product": "LiteLLM", "wins": 11 }, "second": { "product": "OpenRouter", "wins": 11 } }, { "agent": "Claude Code", "model": "Claude Opus 5", "runs": 45, "most_chosen": { "product": "Portkey", "wins": 14 }, "second": { "product": "Cloudflare AI Gateway", "wins": 10 } } ], "by_persona": [ { "persona": "Senior engineer", "runs": 56, "most_chosen": { "product": "Portkey", "wins": 19 }, "second": { "product": "LiteLLM", "wins": 15 } }, { "persona": "Enterprise team", "runs": 35, "most_chosen": { "product": "LiteLLM", "wins": 10 }, "second": { "product": "Portkey", "wins": 9 } }, { "persona": "Junior developer", "runs": 33, "most_chosen": { "product": "Vercel AI Gateway", "wins": 14 }, "second": { "product": "Cloudflare AI Gateway", "wins": 10 } }, { "persona": "Vibe coder", "runs": 16, "most_chosen": { "product": "Vercel AI Gateway", "wins": 5 }, "second": { "product": "Cloudflare AI Gateway", "wins": 5 } } ] }, { "sector": "bot-protection", "title": "Bot protection", "runs": 160, "codebases": 5, "latest_run": "2026-09-02", "ranking": [ { "rank": 1, "product": "Cloudflare Turnstile", "domain": "cloudflare.com", "outcome_not_a_product": false, "wins": 91, "share_percent": 57 }, { "rank": 2, "product": "Vercel BotID", "domain": "vercel.com", "outcome_not_a_product": false, "wins": 18, "share_percent": 11 }, { "rank": 3, "product": "Built in-house", "domain": null, "outcome_not_a_product": true, "wins": 18, "share_percent": 11 }, { "rank": 4, "product": "ALTCHA", "domain": "altcha.org", "outcome_not_a_product": false, "wins": 12, "share_percent": 8 }, { "rank": 5, "product": "django-axes", "domain": "github.com", "outcome_not_a_product": false, "wins": 11, "share_percent": 7 }, { "rank": 6, "product": "Google reCAPTCHA", "domain": "google.com", "outcome_not_a_product": false, "wins": 6, "share_percent": 4 }, { "rank": 7, "product": "Laravel RateLimiter", "domain": "laravel.com", "outcome_not_a_product": false, "wins": 2, "share_percent": 1 } ], "by_agent": [ { "agent": "Codex", "model": "GPT-5.6 Sol", "runs": 54, "most_chosen": { "product": "Cloudflare Turnstile", "wins": 40 }, "second": { "product": "Vercel BotID", "wins": 7 } }, { "agent": "Claude Code", "model": "Claude Opus 5", "runs": 54, "most_chosen": { "product": "Cloudflare Turnstile", "wins": 31 }, "second": { "product": "django-axes", "wins": 3 } }, { "agent": "Cursor", "model": "Grok 4.6", "runs": 52, "most_chosen": { "product": "Cloudflare Turnstile", "wins": 20 }, "second": { "product": "ALTCHA", "wins": 12 } } ], "by_persona": [ { "persona": "Junior developer", "runs": 63, "most_chosen": { "product": "Cloudflare Turnstile", "wins": 35 }, "second": { "product": "Vercel BotID", "wins": 18 } }, { "persona": "Vibe coder", "runs": 61, "most_chosen": { "product": "Cloudflare Turnstile", "wins": 45 }, "second": { "product": "ALTCHA", "wins": 8 } }, { "persona": "Enterprise team", "runs": 36, "most_chosen": { "product": "django-axes", "wins": 11 }, "second": { "product": "Cloudflare Turnstile", "wins": 11 } } ] }, { "sector": "search", "title": "Search", "runs": 459, "codebases": 11, "latest_run": "2026-09-02", "ranking": [ { "rank": 1, "product": "Built in-house", "domain": null, "outcome_not_a_product": true, "wins": 106, "share_percent": 23 }, { "rank": 2, "product": "Postgres Full-Text Search", "domain": "postgresql.org", "outcome_not_a_product": false, "wins": 79, "share_percent": 17 }, { "rank": 3, "product": "OpenSearch", "domain": "opensearch.org", "outcome_not_a_product": false, "wins": 59, "share_percent": 13 }, { "rank": 4, "product": "Typesense", "domain": "typesense.org", "outcome_not_a_product": false, "wins": 42, "share_percent": 9 }, { "rank": 5, "product": "Meilisearch", "domain": "meilisearch.com", "outcome_not_a_product": false, "wins": 40, "share_percent": 9 }, { "rank": 6, "product": "Oracle Indexed Search", "domain": "oracle.com", "outcome_not_a_product": false, "wins": 23, "share_percent": 5 }, { "rank": 7, "product": "Algolia", "domain": "algolia.com", "outcome_not_a_product": false, "wins": 23, "share_percent": 5 }, { "rank": 8, "product": "Elasticsearch", "domain": "elastic.co", "outcome_not_a_product": false, "wins": 22, "share_percent": 5 }, { "rank": 9, "product": "Atlas Search", "domain": "mongodb.com", "outcome_not_a_product": false, "wins": 17, "share_percent": 4 }, { "rank": 10, "product": "Redis Query Engine", "domain": "redis.io", "outcome_not_a_product": false, "wins": 9, "share_percent": 2 }, { "rank": 11, "product": "MySQL FULLTEXT", "domain": "mysql.com", "outcome_not_a_product": false, "wins": 8, "share_percent": 2 }, { "rank": 12, "product": "MongoDB Text Search", "domain": "mongodb.com", "outcome_not_a_product": false, "wins": 8, "share_percent": 2 }, { "rank": 13, "product": "Fuse.js", "domain": "fusejs.io", "outcome_not_a_product": false, "wins": 7, "share_percent": 2 }, { "rank": 14, "product": "PostgreSQL pg_trgm", "domain": "postgresql.org", "outcome_not_a_product": false, "wins": 5, "share_percent": 1 }, { "rank": 15, "product": "MiniSearch", "domain": "lucaong.github.io", "outcome_not_a_product": false, "wins": 2, "share_percent": 0 }, { "rank": 16, "product": "SQLite FTS", "domain": "sqlite.org", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 17, "product": "Aurora PostgreSQL", "domain": "aws.amazon.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 } ], "by_agent": [ { "agent": "Codex", "model": "GPT-5.6 Sol", "runs": 155, "most_chosen": { "product": "OpenSearch", "wins": 27 }, "second": { "product": "Postgres Full-Text Search", "wins": 26 } }, { "agent": "Cursor", "model": "Grok 4.6", "runs": 152, "most_chosen": { "product": "Meilisearch", "wins": 26 }, "second": { "product": "Postgres Full-Text Search", "wins": 19 } }, { "agent": "Claude Code", "model": "Claude Opus 5", "runs": 152, "most_chosen": { "product": "Postgres Full-Text Search", "wins": 34 }, "second": { "product": "Typesense", "wins": 18 } } ], "by_persona": [ { "persona": "Junior developer", "runs": 193, "most_chosen": { "product": "OpenSearch", "wins": 23 }, "second": { "product": "Meilisearch", "wins": 22 } }, { "persona": "Senior engineer", "runs": 106, "most_chosen": { "product": "Postgres Full-Text Search", "wins": 28 }, "second": { "product": "Algolia", "wins": 23 } }, { "persona": "Enterprise team", "runs": 95, "most_chosen": { "product": "OpenSearch", "wins": 36 }, "second": { "product": "Oracle Indexed Search", "wins": 14 } }, { "persona": "Vibe coder", "runs": 65, "most_chosen": { "product": "Postgres Full-Text Search", "wins": 23 }, "second": { "product": "Fuse.js", "wins": 7 } } ] }, { "sector": "agent-frameworks", "title": "Agent frameworks", "runs": 681, "codebases": 13, "latest_run": "2026-09-02", "ranking": [ { "rank": 1, "product": "Built in-house", "domain": null, "outcome_not_a_product": true, "wins": 170, "share_percent": 25 }, { "rank": 2, "product": "Vercel AI SDK", "domain": "ai-sdk.dev", "outcome_not_a_product": false, "wins": 95, "share_percent": 14 }, { "rank": 3, "product": "Cursor SDK", "domain": "cursor.com", "outcome_not_a_product": false, "wins": 94, "share_percent": 14 }, { "rank": 4, "product": "Inngest", "domain": "inngest.com", "outcome_not_a_product": false, "wins": 50, "share_percent": 7 }, { "rank": 5, "product": "Temporal", "domain": "temporal.io", "outcome_not_a_product": false, "wins": 29, "share_percent": 4 }, { "rank": 6, "product": "Prism", "domain": "prismphp.com", "outcome_not_a_product": false, "wins": 28, "share_percent": 4 }, { "rank": 7, "product": "OpenAI Agents SDK", "domain": "openai.github.io", "outcome_not_a_product": false, "wins": 24, "share_percent": 4 }, { "rank": 8, "product": "LangGraph", "domain": "langchain.com", "outcome_not_a_product": false, "wins": 23, "share_percent": 3 }, { "rank": 9, "product": "Azure AI Foundry Agent Service", "domain": "ai.azure.com", "outcome_not_a_product": false, "wins": 19, "share_percent": 3 }, { "rank": 10, "product": "Claude Agent SDK", "domain": "platform.claude.com", "outcome_not_a_product": false, "wins": 16, "share_percent": 2 }, { "rank": 11, "product": "DBOS Transact", "domain": "dbos.dev", "outcome_not_a_product": false, "wins": 14, "share_percent": 2 }, { "rank": 12, "product": "Claude Managed Agents", "domain": "anthropic.com", "outcome_not_a_product": false, "wins": 13, "share_percent": 2 }, { "rank": 13, "product": "Spring AI", "domain": "spring.io", "outcome_not_a_product": false, "wins": 11, "share_percent": 2 }, { "rank": 14, "product": "Laravel AI SDK", "domain": "laravel.com", "outcome_not_a_product": false, "wins": 10, "share_percent": 1 }, { "rank": 15, "product": "Anthropic SDK", "domain": "anthropic.com", "outcome_not_a_product": false, "wins": 8, "share_percent": 1 }, { "rank": 16, "product": "Mastra", "domain": "mastra.ai", "outcome_not_a_product": false, "wins": 7, "share_percent": 1 }, { "rank": 17, "product": "Vertex AI Search", "domain": "cloud.google.com", "outcome_not_a_product": false, "wins": 7, "share_percent": 1 }, { "rank": 18, "product": "Dify Cloud", "domain": "dify.ai", "outcome_not_a_product": false, "wins": 6, "share_percent": 1 }, { "rank": 19, "product": "Google Agent Development Kit", "domain": "google.github.io", "outcome_not_a_product": false, "wins": 4, "share_percent": 1 }, { "rank": 20, "product": "LangChain", "domain": "langchain.com", "outcome_not_a_product": false, "wins": 4, "share_percent": 1 }, { "rank": 21, "product": "DBOS Transact + Pydantic AI", "domain": null, "outcome_not_a_product": false, "wins": 3, "share_percent": 0 }, { "rank": 22, "product": "Azure Durable Task Scheduler", "domain": "azure.microsoft.com", "outcome_not_a_product": false, "wins": 3, "share_percent": 0 }, { "rank": 23, "product": "Pydantic AI", "domain": "pydantic.dev", "outcome_not_a_product": false, "wins": 2, "share_percent": 0 }, { "rank": 24, "product": "Claude subagent SDK", "domain": "anthropic.com", "outcome_not_a_product": false, "wins": 2, "share_percent": 0 }, { "rank": 25, "product": "Spring AI + Temporal", "domain": null, "outcome_not_a_product": false, "wins": 2, "share_percent": 0 }, { "rank": 26, "product": "OpenAI file search", "domain": "openai.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 27, "product": "Cursor Cloud Agents", "domain": "cursor.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 28, "product": "Vercel AI SDK + Vercel Workflow", "domain": null, "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 29, "product": "Genkit", "domain": "genkit.dev", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 30, "product": "Azure Durable Functions", "domain": "azure.microsoft.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 31, "product": "OpenAI SDK", "domain": "openai.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 32, "product": "Durable subagent Scheduler", "domain": "github.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 33, "product": "Anthropic Tool Runner", "domain": "anthropic.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 34, "product": "OpenAI Responses API", "domain": "openai.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 35, "product": "Trigger.dev", "domain": "trigger.dev", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 36, "product": "LangChain + LangGraph", "domain": null, "outcome_not_a_product": false, "wins": 1, "share_percent": 0 } ], "by_agent": [ { "agent": "Codex", "model": null, "runs": 243, "most_chosen": { "product": "Vercel AI SDK", "wins": 39 }, "second": { "product": "OpenAI Agents SDK", "wins": 24 } }, { "agent": "Claude Code", "model": null, "runs": 237, "most_chosen": { "product": "Vercel AI SDK", "wins": 34 }, "second": { "product": "Inngest", "wins": 18 } }, { "agent": "Cursor", "model": "Grok 4.6", "runs": 201, "most_chosen": { "product": "Cursor SDK", "wins": 94 }, "second": { "product": "Vercel AI SDK", "wins": 22 } } ], "by_persona": [ { "persona": "Senior engineer", "runs": 254, "most_chosen": { "product": "Inngest", "wins": 35 }, "second": { "product": "Temporal", "wins": 26 } }, { "persona": "Vibe coder", "runs": 163, "most_chosen": { "product": "Vercel AI SDK", "wins": 65 }, "second": { "product": "Cursor SDK", "wins": 28 } }, { "persona": "Junior developer", "runs": 157, "most_chosen": { "product": "Prism", "wins": 28 }, "second": { "product": "Cursor SDK", "wins": 21 } }, { "persona": "Enterprise team", "runs": 107, "most_chosen": { "product": "Cursor SDK", "wins": 21 }, "second": { "product": "Azure AI Foundry Agent Service", "wins": 19 } } ] }, { "sector": "perf-ci", "title": "Performance in CI", "runs": 216, "codebases": 8, "latest_run": "2026-09-01", "ranking": [ { "rank": 1, "product": "Built in-house", "domain": null, "outcome_not_a_product": true, "wins": 113, "share_percent": 52 }, { "rank": 2, "product": "JMH", "domain": "openjdk.org", "outcome_not_a_product": false, "wins": 22, "share_percent": 10 }, { "rank": 3, "product": "pytest-benchmark", "domain": "pytest-benchmark.readthedocs.io", "outcome_not_a_product": false, "wins": 15, "share_percent": 7 }, { "rank": 4, "product": "Autocannon", "domain": "github.com", "outcome_not_a_product": false, "wins": 14, "share_percent": 6 }, { "rank": 5, "product": "hyperfine", "domain": "github.com", "outcome_not_a_product": false, "wins": 13, "share_percent": 6 }, { "rank": 6, "product": "PHPBench", "domain": "phpbench.readthedocs.io", "outcome_not_a_product": false, "wins": 12, "share_percent": 6 }, { "rank": 7, "product": "Grafana k6", "domain": "k6.io", "outcome_not_a_product": false, "wins": 8, "share_percent": 4 }, { "rank": 8, "product": "BenchmarkDotNet", "domain": "benchmarkdotnet.org", "outcome_not_a_product": false, "wins": 7, "share_percent": 3 }, { "rank": 9, "product": "CodSpeed", "domain": "codspeed.io", "outcome_not_a_product": false, "wins": 3, "share_percent": 1 }, { "rank": 10, "product": "Tinybench", "domain": "github.com", "outcome_not_a_product": false, "wins": 3, "share_percent": 1 }, { "rank": 11, "product": "Vitest Bench", "domain": "vitest.dev", "outcome_not_a_product": false, "wins": 2, "share_percent": 1 }, { "rank": 12, "product": "perfgate", "domain": "github.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 13, "product": "Benchstat", "domain": "github.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 14, "product": "Gungraun", "domain": "github.com", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 }, { "rank": 15, "product": "Bencher", "domain": "bencher.dev", "outcome_not_a_product": false, "wins": 1, "share_percent": 0 } ], "by_agent": [ { "agent": "Cursor", "model": "Grok 4.6", "runs": 72, "most_chosen": { "product": "JMH", "wins": 6 }, "second": { "product": "pytest-benchmark", "wins": 4 } }, { "agent": "Claude Code", "model": "Claude Opus 5", "runs": 72, "most_chosen": { "product": "Autocannon", "wins": 5 }, "second": { "product": "JMH", "wins": 5 } }, { "agent": "Codex", "model": "GPT-5.6 Sol", "runs": 72, "most_chosen": { "product": "JMH", "wins": 11 }, "second": { "product": "hyperfine", "wins": 7 } } ], "by_persona": [ { "persona": "Senior engineer", "runs": 144, "most_chosen": { "product": "pytest-benchmark", "wins": 15 }, "second": { "product": "Autocannon", "wins": 14 } }, { "persona": "Enterprise team", "runs": 60, "most_chosen": { "product": "JMH", "wins": 22 }, "second": { "product": "PHPBench", "wins": 12 } }, { "persona": "Junior developer", "runs": 12, "most_chosen": null, "second": null } ] } ] }