{
  "schema_version": "cloud-deploy-benchmark.v1",
  "generated_at": "2026-09-04T23:20:03.238Z",
  "run_date": "4 September 2026",
  "protocol_version": "deploy-e2e/E1",
  "total_runs": 90,
  "provider_count": 5,
  "project_count": 3,
  "agent_count": 2,
  "repeats": 3,
  "judge": {
    "model": "gemini-3.7-flash",
    "version": 1
  },
  "project_audit": {
    "model": "gemini-3.7-flash",
    "version": "deploy-e2e-fidelity.v2",
    "method": "Post-run audit of saved source changes, deployment commands, tests and handoff. Not a second live functional test. Added after marker-only false positives were found.",
    "completed_projects": 17
  },
  "providers": [
    {
      "id": "railway",
      "name": "Railway",
      "runs": 18,
      "successes": 18,
      "success_rate": 1,
      "success_rate_ci95": [
        0.8241154494176252,
        1
      ],
      "mean_score": 96.6,
      "median_score": 96,
      "median_duration_ms": 182000,
      "median_tool_actions": 24,
      "cleanup_rate": 1,
      "components": {
        "production_proof": 40,
        "setup_auth": 18.6,
        "cli_configuration": 14.1,
        "error_recovery": 13.9,
        "handoff_cleanup": 10
      },
      "failure_stages": [],
      "verdict": "Every run produced a verified public URL. 7/18 also passed the saved-evidence project audit. Setup and authentication were usually direct.",
      "completed_projects": 7,
      "completion_rate": 0.3888888888888889,
      "fidelity_counts": {
        "faithful": 7,
        "partial": 11,
        "placeholder": 0,
        "no_deployment": 0,
        "unknown": 0
      },
      "by_project": [
        {
          "name": "Vite + Express invoice app",
          "runs": 6,
          "marker_verified": 6,
          "completed": 1
        },
        {
          "name": "Nuxt server app",
          "runs": 6,
          "marker_verified": 6,
          "completed": 0
        },
        {
          "name": "Flask service",
          "runs": 6,
          "marker_verified": 6,
          "completed": 6
        }
      ]
    },
    {
      "id": "vercel",
      "name": "Vercel",
      "runs": 18,
      "successes": 18,
      "success_rate": 1,
      "success_rate_ci95": [
        0.8241154494176252,
        1
      ],
      "mean_score": 91.6,
      "median_score": 92,
      "median_duration_ms": 205500,
      "median_tool_actions": 24,
      "cleanup_rate": 1,
      "components": {
        "production_proof": 40,
        "setup_auth": 15.3,
        "cli_configuration": 13,
        "error_recovery": 13.6,
        "handoff_cleanup": 9.7
      },
      "failure_stages": [],
      "verdict": "Every run produced a verified public URL. 5/18 also passed the saved-evidence project audit. Setup and authentication were usually direct.",
      "completed_projects": 5,
      "completion_rate": 0.2777777777777778,
      "fidelity_counts": {
        "faithful": 5,
        "partial": 12,
        "placeholder": 1,
        "no_deployment": 0,
        "unknown": 0
      },
      "by_project": [
        {
          "name": "Vite + Express invoice app",
          "runs": 6,
          "marker_verified": 6,
          "completed": 0
        },
        {
          "name": "Nuxt server app",
          "runs": 6,
          "marker_verified": 6,
          "completed": 0
        },
        {
          "name": "Flask service",
          "runs": 6,
          "marker_verified": 6,
          "completed": 5
        }
      ]
    },
    {
      "id": "render",
      "name": "Render",
      "runs": 18,
      "successes": 16,
      "success_rate": 0.8888888888888888,
      "success_rate_ci95": [
        0.6719975513339578,
        0.9689811315464172
      ],
      "mean_score": 80.3,
      "median_score": 86,
      "median_duration_ms": 666500,
      "median_tool_actions": 51,
      "cleanup_rate": 0.8333333333333334,
      "components": {
        "production_proof": 35.6,
        "setup_auth": 14.3,
        "cli_configuration": 9.9,
        "error_recovery": 12.2,
        "handoff_cleanup": 8.3
      },
      "failure_stages": [
        {
          "stage": "deploy",
          "count": 1
        },
        {
          "stage": "service_creation",
          "count": 1
        }
      ],
      "verdict": "16 of 18 runs produced a verified public URL. 5/18 also passed the saved-evidence project audit. Setup and authentication were usually direct.",
      "completed_projects": 5,
      "completion_rate": 0.2777777777777778,
      "fidelity_counts": {
        "faithful": 5,
        "partial": 11,
        "placeholder": 0,
        "no_deployment": 2,
        "unknown": 0
      },
      "by_project": [
        {
          "name": "Vite + Express invoice app",
          "runs": 6,
          "marker_verified": 6,
          "completed": 0
        },
        {
          "name": "Nuxt server app",
          "runs": 6,
          "marker_verified": 5,
          "completed": 0
        },
        {
          "name": "Flask service",
          "runs": 6,
          "marker_verified": 5,
          "completed": 5
        }
      ]
    },
    {
      "id": "cloudflare",
      "name": "Cloudflare",
      "runs": 18,
      "successes": 18,
      "success_rate": 1,
      "success_rate_ci95": [
        0.8241154494176252,
        1
      ],
      "mean_score": 96.7,
      "median_score": 97,
      "median_duration_ms": 163000,
      "median_tool_actions": 23,
      "cleanup_rate": 1,
      "components": {
        "production_proof": 40,
        "setup_auth": 18.7,
        "cli_configuration": 14.1,
        "error_recovery": 14.1,
        "handoff_cleanup": 9.9
      },
      "failure_stages": [],
      "verdict": "Every run produced a verified public URL. 0/18 also passed the saved-evidence project audit. Setup and authentication were usually direct.",
      "completed_projects": 0,
      "completion_rate": 0,
      "fidelity_counts": {
        "faithful": 0,
        "partial": 17,
        "placeholder": 1,
        "no_deployment": 0,
        "unknown": 0
      },
      "by_project": [
        {
          "name": "Vite + Express invoice app",
          "runs": 6,
          "marker_verified": 6,
          "completed": 0
        },
        {
          "name": "Nuxt server app",
          "runs": 6,
          "marker_verified": 6,
          "completed": 0
        },
        {
          "name": "Flask service",
          "runs": 6,
          "marker_verified": 6,
          "completed": 0
        }
      ]
    },
    {
      "id": "netlify",
      "name": "Netlify",
      "runs": 18,
      "successes": 18,
      "success_rate": 1,
      "success_rate_ci95": [
        0.8241154494176252,
        1
      ],
      "mean_score": 91.9,
      "median_score": 92,
      "median_duration_ms": 263000,
      "median_tool_actions": 29.5,
      "cleanup_rate": 1,
      "components": {
        "production_proof": 40,
        "setup_auth": 15.3,
        "cli_configuration": 13.2,
        "error_recovery": 13.7,
        "handoff_cleanup": 9.8
      },
      "failure_stages": [],
      "verdict": "Every run produced a verified public URL. 0/18 also passed the saved-evidence project audit. Setup and authentication were usually direct.",
      "completed_projects": 0,
      "completion_rate": 0,
      "fidelity_counts": {
        "faithful": 0,
        "partial": 12,
        "placeholder": 6,
        "no_deployment": 0,
        "unknown": 0
      },
      "by_project": [
        {
          "name": "Vite + Express invoice app",
          "runs": 6,
          "marker_verified": 6,
          "completed": 0
        },
        {
          "name": "Nuxt server app",
          "runs": 6,
          "marker_verified": 6,
          "completed": 0
        },
        {
          "name": "Flask service",
          "runs": 6,
          "marker_verified": 6,
          "completed": 0
        }
      ]
    }
  ],
  "projects": [
    {
      "id": "vite-invoice-tracker",
      "name": "Vite + Express invoice app",
      "note": "React client, Express API and Postgres · 1/30 passed project audit",
      "mean_score": 94
    },
    {
      "id": "nuxt-fieldservice",
      "name": "Nuxt server app",
      "note": "Server-rendered app, sessions and Postgres · 0/30 passed project audit",
      "mean_score": 90.5
    },
    {
      "id": "flask-shiftplanner",
      "name": "Flask service",
      "note": "Python web service and process binding · 16/30 passed project audit",
      "mean_score": 89.9
    }
  ],
  "agents": [
    {
      "id": "codex/gpt-5.6-sol",
      "name": "Codex · GPT-5.6",
      "note": "8/45 passed project audit",
      "mean_score": 92.1
    },
    {
      "id": "claude-code/claude-opus-5",
      "name": "Claude Code · Opus 5",
      "note": "9/45 passed project audit",
      "mean_score": 90.8
    }
  ],
  "findings": [
    {
      "title": "Railway led the project audit",
      "body": "7 of 18 attempts had a verified URL and saved evidence of the complete app. The original URL-workflow score was 96.6/100. Small samples do not establish a general winner."
    },
    {
      "title": "A working URL can hide a broken app",
      "body": "88/90 URLs passed the marker check. Only 17/90 also passed the saved-evidence project audit. Static placeholders, missing databases and incomplete ports did not count as completed projects."
    },
    {
      "title": "The project type mattered",
      "body": "Vite + Express invoice app averaged 94. Flask service averaged 89.9. Runtime and build assumptions changed the work."
    }
  ]
}