{
  "title": "Million-request citation-first RAG benchmark 2026",
  "evaluatorVersion": "pran-rag-scale/1.0.0",
  "executedAt": "2026-08-24",
  "project": "navyaai-consultation",
  "finalRegion": "us-east1",
  "niche": "Industrial maintenance and field-service troubleshooting",
  "scope": "Hybrid retrieval and extractive cited responses; LLM generation excluded",
  "corpus": {
    "version": "synthetic-industrial-maintenance-v1",
    "chunks": 20000,
    "embeddingDimensions": 64,
    "seed": 20260824,
    "synthetic": true,
    "seedDurationSeconds": 21.168509756
  },
  "architecture": {
    "edge": "Global external Application Load Balancer",
    "api": "Cloud Run, Go 1.24, 2 vCPU, 1 GiB, concurrency 80, min 1, max 10",
    "database": "Cloud SQL for PostgreSQL 16 Enterprise, db-custom-2-7680, 10 GiB SSD, zonal benchmark instance",
    "retrieval": "Exact fault-code lane plus reciprocal-rank fusion of PostgreSQL full-text and pgvector HNSW candidates",
    "cache": "Per-instance bounded cache, 10,000 entries, 15-minute TTL",
    "security": "Global edge, application bearer token, Secret Manager, dedicated service account, Cloud SQL connector",
    "imageDigest": "sha256:cd59a27a37a526392a36ecfe2a47e9e67c39c1ea0c6ce309b4466b6d33eabb18"
  },
  "qualityRuns": [
    {
      "id": "hybrid-rrf-v1-failed",
      "region": "us-central1",
      "queries": 500,
      "recallAt1": 0.198,
      "recallAt3": 0.338,
      "latencyMs": { "mean": 5.738, "p50": 5.477, "p90": 6.371, "p95": 6.89, "p99": 9.649, "p999": 13.019, "max": 36.562 },
      "passed": false,
      "failure": "Reciprocal-rank fusion diluted the exact domain fault-code signal."
    },
    {
      "id": "exact-code-plus-hybrid-final",
      "region": "us-east1",
      "queries": 500,
      "recallAt1": 1.0,
      "recallAt3": 1.0,
      "latencyMs": { "mean": 3.649, "p50": 3.533, "p90": 4.17, "p95": 4.447, "p99": 5.39, "p999": 8.622, "max": 14.439 },
      "passed": true
    }
  ],
  "loadRuns": [
    {
      "runId": "edge-probe-before-propagation-failed",
      "mode": "hot",
      "targetRequests": 100,
      "attemptedRequests": 100,
      "successful": 0,
      "errors": 100,
      "errorRate": 1.0,
      "durationSeconds": 10.002687007,
      "achievedRps": 9.997313714806712,
      "configuredRps": 10,
      "cacheHits": 0,
      "cacheMisses": 0,
      "latencyMs": { "mean": 1.884, "p50": 1.77, "p90": 2.15, "p95": 2.296, "p99": 4.445, "p999": 4.445, "max": 7.372 },
      "passed": false,
      "failure": "Global forwarding rule had not propagated; connections were reset before reaching Cloud Run."
    },
    {
      "runId": "edge-probe-100",
      "mode": "hot",
      "targetRequests": 100,
      "attemptedRequests": 100,
      "successful": 100,
      "errors": 0,
      "errorRate": 0,
      "durationSeconds": 10.017859698,
      "achievedRps": 9.982172142015958,
      "configuredRps": 10,
      "cacheHits": 1,
      "cacheMisses": 99,
      "cacheHitRate": 0.01,
      "responseBytes": 288974,
      "latencyMs": { "mean": 16.588, "p50": 15.94, "p90": 20.356, "p95": 23.264, "p99": 29.07, "p999": 29.07, "max": 32.432 },
      "passed": true
    },
    {
      "runId": "hot-10k-1000rps",
      "mode": "hot",
      "targetRequests": 10000,
      "attemptedRequests": 10000,
      "successful": 10000,
      "errors": 0,
      "errorRate": 0,
      "durationSeconds": 10.256396829,
      "achievedRps": 975.0012764448586,
      "configuredRps": 1000,
      "cacheHits": 9711,
      "cacheMisses": 289,
      "cacheHitRate": 0.9711,
      "responseBytes": 28850916,
      "latencyMs": { "mean": 30.284, "p50": 9.393, "p90": 14.433, "p95": 221.51, "p99": 523.712, "p999": 762.272, "max": 875.145 },
      "passed": false,
      "failure": "Scale-out cache warmup created tail latency despite zero errors."
    },
    {
      "runId": "hot-10k-1000rps-repeat",
      "mode": "hot",
      "targetRequests": 10000,
      "attemptedRequests": 10000,
      "successful": 10000,
      "errors": 0,
      "errorRate": 0,
      "durationSeconds": 10.264475798,
      "achievedRps": 974.233871928313,
      "configuredRps": 1000,
      "cacheHits": 10000,
      "cacheMisses": 0,
      "cacheHitRate": 1.0,
      "responseBytes": 28848466,
      "latencyMs": { "mean": 9.474, "p50": 9.12, "p90": 11.596, "p95": 13.186, "p99": 16.277, "p999": 28.627, "max": 96.695 },
      "passed": true
    },
    {
      "runId": "cold-10k-200rps",
      "mode": "cold",
      "targetRequests": 10000,
      "attemptedRequests": 10000,
      "successful": 10000,
      "errors": 0,
      "errorRate": 0,
      "durationSeconds": 50.019774571,
      "achievedRps": 199.9209329863255,
      "configuredRps": 200,
      "cacheHits": 0,
      "cacheMisses": 10000,
      "cacheHitRate": 0,
      "responseBytes": 29197715,
      "latencyMs": { "mean": 16.947, "p50": 15.127, "p90": 21.034, "p95": 24.963, "p99": 53.028, "p999": 88.102, "max": 254.47 },
      "passed": true
    },
    {
      "runId": "mixed-100k-1000rps-client-timeout-failed",
      "mode": "mixed",
      "targetRequests": 100000,
      "attemptedRequests": 100000,
      "successful": 99891,
      "errors": 109,
      "errorRate": 0.00109,
      "durationSeconds": 103.130592487,
      "achievedRps": 969.644385710335,
      "configuredRps": 1000,
      "cacheHits": 94541,
      "cacheMisses": 5350,
      "cacheHitRate": 0.9464416213672904,
      "responseBytes": 288352850,
      "latencyMs": { "mean": 17.55, "p50": 9.545, "p90": 14.402, "p95": 17.747, "p99": 38.053, "p999": 3000.856, "max": 3025.441 },
      "passed": false,
      "failure": "Load generator response-header timeout was 3 seconds while the service timeout was 5 seconds; Cloud Run recorded zero application errors."
    },
    {
      "runId": "mixed-100k-1000rps-clientfix",
      "mode": "mixed",
      "targetRequests": 100000,
      "attemptedRequests": 100000,
      "successful": 100000,
      "errors": 0,
      "errorRate": 0,
      "durationSeconds": 102.454016261,
      "achievedRps": 976.0476323861386,
      "configuredRps": 1000,
      "cacheHits": 95000,
      "cacheMisses": 5000,
      "cacheHitRate": 0.95,
      "responseBytes": 288664806,
      "latencyMs": { "mean": 10.356, "p50": 9.811, "p90": 13.649, "p95": 15.142, "p99": 19.403, "p999": 32.312, "max": 220.221 },
      "statusCodes": { "200": 100000 },
      "errorTypes": {},
      "passed": true
    },
    {
      "runId": "mixed-1m-1000rps",
      "mode": "mixed",
      "targetRequests": 1000000,
      "attemptedRequests": 1000000,
      "successful": 1000000,
      "errors": 0,
      "errorRate": 0,
      "durationSeconds": 1023.76162366,
      "achievedRps": 976.7898863262221,
      "configuredRps": 1000,
      "workers": 200,
      "cacheHits": 949697,
      "cacheMisses": 50303,
      "cacheHitRate": 0.949697,
      "responseBytes": 2886661768,
      "latencyMs": { "mean": 9.335, "p50": 8.695, "p90": 11.923, "p95": 13.448, "p99": 18.069, "p999": 39.701, "max": 816.281 },
      "histogram": { "<=5ms": 685, "<=10ms": 845601, "<=20ms": 147915, "<=30ms": 3770, "<=50ms": 1517, "<=75ms": 331, "<=100ms": 90, "<=150ms": 57, "<=250ms": 26, "<=500ms": 7, "<=1000ms": 1 },
      "statusCodes": { "200": 1000000 },
      "errorTypes": {},
      "passed": true
    }
  ],
  "monitoring": {
    "millionRun": {
      "requestCount": 1000000,
      "cpuAllocationVcpuSeconds": 4234.784491184029,
      "memoryAllocationGibSeconds": 2247.557643730896,
      "billableInstanceSeconds": 2168.3728015295046,
      "maxActiveInstances": 5,
      "meanActiveInstances": 2.263157894736842,
      "cloudSqlCpuUtilizationMean": 0.1118474068780811,
      "cloudSqlCpuUtilizationMax": 0.13368098824788249,
      "cloudSqlMemoryUtilizationMean": 0.4707800652260452,
      "cloudSqlMemoryUtilizationMax": 0.47439979546505712
    },
    "fullExperiment": {
      "requestCount": 1230104,
      "cpuAllocationVcpuSeconds": 5482.926514981043,
      "memoryAllocationGibSeconds": 3984.496761635776
    }
  },
  "pricing": {
    "currency": "USD",
    "snapshotDate": "2026-08-24",
    "freeTierExcluded": true,
    "rates": {
      "cloudRunActiveVcpuSecond": 0.000024,
      "cloudRunIdleVcpuSecond": 0.0000025,
      "cloudRunGibSecond": 0.0000025,
      "cloudRunPerMillionRequests": 0.4,
      "cloudSqlEnterpriseVcpuHour": 0.0413,
      "cloudSqlEnterpriseGibHour": 0.007,
      "cloudSqlHaVcpuHour": 0.0826,
      "cloudSqlHaGibHour": 0.014,
      "cloudSqlSsdGibMonth": 0.17,
      "globalForwardingRuleHour": 0.025,
      "loadBalancerProcessedGib": 0.008,
      "northAmericaInternetEgressGib": 0.12
    },
    "measuredMillionMarginal": {
      "cloudRunCpu": 0.1016348277884167,
      "cloudRunMemory": 0.00561889410932724,
      "cloudRunRequests": 0.4,
      "loadBalancerResponseProcessing": 0.021507306158542634,
      "internetEgressUpperBound": 0.32260959237813946,
      "totalUpperBound": 0.8513706204344261
    },
    "monthlyProductionBaseline": {
      "cloudSqlRegionalHa": 200.646,
      "globalForwardingRule": 18.25,
      "oneIdleMinCloudRunInstance": 19.71,
      "total": 238.606
    },
    "monthlyScenarios": [
      { "millionRequests": 1, "total": 239.45737062043446, "costPerMillion": 239.45737062043446 },
      { "millionRequests": 5, "total": 242.86285310217215, "costPerMillion": 48.57257062043443 },
      { "millionRequests": 50, "total": 281.17453102172135, "costPerMillion": 5.623490620434427 }
    ],
    "sources": [
      "https://cloud.google.com/run/pricing",
      "https://cloud.google.com/sql/pricing",
      "https://cloud.google.com/load-balancing/pricing",
      "https://cloud.google.com/vpc/pricing"
    ]
  },
  "limitations": [
    "The corpus and gold queries are synthetic; 100% source recovery does not imply real maintenance answer quality or safety.",
    "The result measures a retrieval and extractive citation layer, not uncached LLM generation. Model tokens and latency are excluded.",
    "The temporary database was zonal. The production cost model uses regional HA pricing, but HA failover was not tested.",
    "The per-instance cache produced a failed scale-out warmup run; a shared cache or deliberate prewarming is required for a strict burst SLO.",
    "Internet egress is an upper-bound list-price estimate; actual routing, free tiers, discounts and response compression change billed cost.",
    "The million-request run used a same-region Cloud Run job through an external load balancer, not geographically distributed end users."
  ]
}