{
  "schema": "spark-v12-matched-t4-regression-v1",
  "snapshot_date": "2026-10-08",
  "scope": "Isolated T4 comparison of the previous v1.1 text-serving model and the selected v1.2 release artifact. Not public API end-to-end measurements.",
  "source_assessment_sha256": "2c382a0155083a58fead3c5b26828b7f096e7c1e4300b491f0043b53a6352488",
  "versions": {
    "old": "v1.1 text-serving baseline",
    "candidate": "v1.2 release"
  },
  "main_metrics": {
    "snli": {
      "n": 150,
      "correct": {
        "old": 93,
        "candidate": 124
      },
      "accuracy_delta": 0.20666666666666667,
      "correct_delta": 31
    },
    "boolq": {
      "n": 100,
      "correct": {
        "old": 89,
        "candidate": 90
      },
      "accuracy_delta": 0.01,
      "correct_delta": 1
    },
    "pubmedqa": {
      "n": 300,
      "correct": {
        "old": 258,
        "candidate": 262
      },
      "accuracy_delta": 0.013333333333333334,
      "correct_delta": 4
    },
    "banking32": {
      "n": 32,
      "correct": {
        "old": 19,
        "candidate": 21
      },
      "accuracy_delta": 0.0625,
      "correct_delta": 2
    },
    "legacy_ax": {
      "n": 16,
      "correct": {
        "old": 12,
        "candidate": 12
      },
      "accuracy_delta": 0.0,
      "correct_delta": 0
    },
    "legacy_snake": {
      "n": 16,
      "correct": {
        "old": 11,
        "candidate": 12
      },
      "accuracy_delta": 0.0625,
      "correct_delta": 1
    },
    "legacy_tetris": {
      "n": 16,
      "correct": {
        "old": 3,
        "candidate": 2
      },
      "accuracy_delta": -0.0625,
      "correct_delta": -1
    },
    "legacy_pokemon": {
      "n": 16,
      "correct": {
        "old": 3,
        "candidate": 3
      },
      "accuracy_delta": 0.0,
      "correct_delta": 0
    },
    "legacy_jevbench_original": {
      "n": 16,
      "correct": {
        "old": 16,
        "candidate": 16
      },
      "accuracy_delta": 0.0,
      "correct_delta": 0
    }
  },
  "mixed_other_report_only": {
    "sst2": {
      "n": 64,
      "correct": {
        "old": 59,
        "candidate": 58
      }
    },
    "snli": {
      "n": 32,
      "correct": {
        "old": 19,
        "candidate": 27
      }
    },
    "boolq": {
      "n": 32,
      "correct": {
        "old": 28,
        "candidate": 29
      }
    }
  },
  "latency": {
    "scope": "VM-local llama HTTP completion; not public platform API end-to-end",
    "quantile": "nearest-rank ceil(.95*n)",
    "completion_count_per_version": 897,
    "p95_seconds": {
      "old": 0.7467721090000623,
      "candidate": 0.7350517649999802
    }
  },
  "help100": {
    "n": 100,
    "quality_scored": false,
    "report_only": true
  },
  "engineering_gate_pass": true,
  "quantization_parity_pass": null,
  "notes": [
    "SNLI 150, BoolQ 100, PubMedQA 300 and Banking 32 are distinct frozen regression slices. Accuracy denominators are preserved; no overall score is inferred.",
    "Five legacy slices have 16 cases each. The one-case Tetris decline is retained rather than omitted.",
    "Latency covers all 897 VM-local llama HTTP completions per version. It excludes the public platform API, Internet transit and browser/camera rendering.",
    "This PubMedQA 300 regression is different from the 300 items×5 repeats offline peer protocol.",
    "Engineering release gates passed; they do not establish SOTA, quantization parity to the offline research model, or new visual quality."
  ]
}