{
  "schema": "spark-v12-offline-peer-comparison-v1",
  "snapshot_date": "2026-10-08",
  "scope": "Selected v1.2 offline research snapshot and peer systems. Not released production T4 results.",
  "source_summary_sha256": "0b54cdf255bb36096d1f4da41a74fa6ffd6f67652cfd50833d160f8204a627c9",
  "source_report_sha256": "f177c5652a10026623b9b966de7f67c38877150864ad8167bdc460f902d2b7fa",
  "protocol": {
    "items_per_suite": 300,
    "actual_repeats_per_item": 5,
    "planned_decisions_per_suite": 1500,
    "repeats_are_independent_items": false,
    "suites": [
      "banking77",
      "pubmedqa",
      "helpsteer2"
    ],
    "banking77_option_count": 77,
    "public_API_max_choices": 8
  },
  "rows": [
    {
      "model": "Trio-Spark v1.2 (offline)",
      "measurement_path": "offline research",
      "suites": {
        "banking77": {
          "planned_n": 1500,
          "valid_n": 1500,
          "probability_valid_n": 1500,
          "strict_accuracy": 0.8046666666666666,
          "decision_score": 71.00877145751203,
          "nll": 0.6896352714673245,
          "nll_infinite": false,
          "ece15": 0.03457812782128652,
          "expected_score_mae_valid_probabilities": null,
          "ordered_rps_valid_probabilities": null
        },
        "pubmedqa": {
          "planned_n": 1500,
          "valid_n": 1500,
          "probability_valid_n": 1500,
          "strict_accuracy": 0.8833333333333333,
          "decision_score": 63.764164041549186,
          "nll": 0.28137247471724974,
          "nll_infinite": false,
          "ece15": 0.030891924699147543,
          "expected_score_mae_valid_probabilities": null,
          "ordered_rps_valid_probabilities": null
        },
        "helpsteer2": {
          "planned_n": 1500,
          "valid_n": 1390,
          "probability_valid_n": 1390,
          "strict_accuracy": 0.3933333333333333,
          "decision_score": -8.591169527139364,
          "nll": 1.8049473825495108,
          "nll_infinite": false,
          "ece15": 0.276542846676257,
          "expected_score_mae_valid_probabilities": 0.9228024612758883,
          "ordered_rps_valid_probabilities": 0.1744951860618851
        }
      }
    },
    {
      "model": "Jev",
      "measurement_path": "native hosted API",
      "suites": {
        "banking77": {
          "planned_n": 1500,
          "valid_n": 1500,
          "probability_valid_n": 1425,
          "strict_accuracy": 0.8,
          "decision_score": 67.90006281169275,
          "nll": null,
          "nll_infinite": true,
          "ece15": 0.0947438596491228,
          "expected_score_mae_valid_probabilities": null,
          "ordered_rps_valid_probabilities": null
        },
        "pubmedqa": {
          "planned_n": 1500,
          "valid_n": 1500,
          "probability_valid_n": 1500,
          "strict_accuracy": 0.9133333333333333,
          "decision_score": 69.09711375212224,
          "nll": 0.26171419033328375,
          "nll_infinite": false,
          "ece15": 0.057440000000000005,
          "expected_score_mae_valid_probabilities": null,
          "ordered_rps_valid_probabilities": null
        },
        "helpsteer2": {
          "planned_n": 1500,
          "valid_n": 1500,
          "probability_valid_n": 1476,
          "strict_accuracy": 0.41533333333333333,
          "decision_score": 8.954388792123291,
          "nll": null,
          "nll_infinite": true,
          "ece15": 0.1895460704607046,
          "expected_score_mae_valid_probabilities": 0.8576355013550134,
          "ordered_rps_valid_probabilities": 0.1497072662601626
        }
      }
    },
    {
      "model": "Clef-Flash",
      "measurement_path": "native hosted API",
      "suites": {
        "banking77": {
          "planned_n": 1500,
          "valid_n": 1500,
          "probability_valid_n": 1500,
          "strict_accuracy": 0.9466666666666667,
          "decision_score": 90.8952820386382,
          "nll": 0.23443128449026054,
          "nll_infinite": false,
          "ece15": 0.06214433333333333,
          "expected_score_mae_valid_probabilities": null,
          "ordered_rps_valid_probabilities": null
        },
        "pubmedqa": {
          "planned_n": 1500,
          "valid_n": 1499,
          "probability_valid_n": 1499,
          "strict_accuracy": 0.906,
          "decision_score": null,
          "nll": 0.2518177889555153,
          "nll_infinite": false,
          "ece15": 0.045351034022681785,
          "expected_score_mae_valid_probabilities": null,
          "ordered_rps_valid_probabilities": null
        },
        "helpsteer2": {
          "planned_n": 1500,
          "valid_n": 1492,
          "probability_valid_n": 1492,
          "strict_accuracy": 0.42733333333333334,
          "decision_score": null,
          "nll": 1.334004791162812,
          "nll_infinite": false,
          "ece15": 0.10656394101876676,
          "expected_score_mae_valid_probabilities": 0.8859770107238606,
          "ordered_rps_valid_probabilities": 0.14621165168063002
        }
      }
    },
    {
      "model": "Laya",
      "measurement_path": "official local SDK",
      "suites": {
        "banking77": {
          "planned_n": 1500,
          "valid_n": 1500,
          "probability_valid_n": 1500,
          "strict_accuracy": 0.38666666666666666,
          "decision_score": -12.957214575406773,
          "nll": null,
          "nll_infinite": true,
          "ece15": 0.5360658666666667,
          "expected_score_mae_valid_probabilities": null,
          "ordered_rps_valid_probabilities": null
        },
        "pubmedqa": {
          "planned_n": 1500,
          "valid_n": 1500,
          "probability_valid_n": 1500,
          "strict_accuracy": 0.5933333333333334,
          "decision_score": -31.165707342954185,
          "nll": 0.9201982599264572,
          "nll_infinite": false,
          "ece15": 0.246203,
          "expected_score_mae_valid_probabilities": null,
          "ordered_rps_valid_probabilities": null
        },
        "helpsteer2": {
          "planned_n": 1500,
          "valid_n": 1500,
          "probability_valid_n": 1500,
          "strict_accuracy": 0.35333333333333333,
          "decision_score": 2.0004205465597336,
          "nll": 1.3891242201613294,
          "nll_infinite": false,
          "ece15": 0.08030799999999999,
          "expected_score_mae_valid_probabilities": 0.9953763333333333,
          "ordered_rps_valid_probabilities": 0.1606673068416667
        }
      }
    },
    {
      "model": "Clef",
      "measurement_path": "native hosted API",
      "suites": {
        "banking77": {
          "planned_n": 1500,
          "valid_n": 1499,
          "probability_valid_n": 1499,
          "strict_accuracy": 0.946,
          "decision_score": null,
          "nll": 0.23292756269591058,
          "nll_infinite": false,
          "ece15": 0.061211941294196125,
          "expected_score_mae_valid_probabilities": null,
          "ordered_rps_valid_probabilities": null
        },
        "pubmedqa": {
          "planned_n": 1500,
          "valid_n": 1500,
          "probability_valid_n": 1500,
          "strict_accuracy": 0.8866666666666667,
          "decision_score": 66.4054146859083,
          "nll": 0.26828865577466415,
          "nll_infinite": false,
          "ece15": 0.04028866666666666,
          "expected_score_mae_valid_probabilities": null,
          "ordered_rps_valid_probabilities": null
        },
        "helpsteer2": {
          "planned_n": 1500,
          "valid_n": 1499,
          "probability_valid_n": 1499,
          "strict_accuracy": 0.41333333333333333,
          "decision_score": null,
          "nll": 1.3611027292872584,
          "nll_infinite": false,
          "ece15": 0.06208485657104737,
          "expected_score_mae_valid_probabilities": 0.9251791861240828,
          "ordered_rps_valid_probabilities": 0.1453596760873916
        }
      }
    }
  ],
  "unmeasured": [
    "v1.1 and v1.2 serving models on the 77-option Banking77 research protocol",
    "d1"
  ],
  "notes": [
    "Accuracy uses all 1500 planned decisions per suite. Five repeats of 300 items do not create 1500 independent examples.",
    "Valid-response and probability-valid counts are separate. NLL/ECE/MAE/RPS use probability-valid rows; missing metrics are null, never zero.",
    "Infinite NLL from a wire probability zero is marked nll_infinite, not represented as a finite score.",
    "v1.2 research HelpSteer2 has 1390 valid responses/1500 planned, including 110 fixed context refusals in the original planned-denominator protocol.",
    "Decision Score preserves the original protocol; null means incomplete evidence. No official aggregate or overall superiority claim.",
    "Banking77 uses 77 research options and does not expand the public 2\u20138 choice API.",
    "This repeated-item PubMedQA protocol is distinct from the matched 300 case T 4 regression. Do not merge their numbers."
  ],
  "released_followup": {
    "data_file": "serving-followup-2026-10-08.json",
    "scope": "Separate matched T4 release-artifact study covers the original 300 PubMedQA and HelpSteer2 items with five repeats per item. Offline model rows in this file are unchanged."
  }
}