{
  "schema": "spark-serving-same-input-comparison-v1",
  "snapshot_date": "2026-10-08",
  "scope": "Released v1.1 and v1.2 artifacts evaluated on the same T4 using their serving configurations. Original PubMedQA and HelpSteer2 item/order/repeat views also used by the offline peer comparison. Not a public API latency test or an official leaderboard reproduction.",
  "provenance": {
    "assessment_ref": "phase2-results.json",
    "assessment_sha256": "42d6baf6af4f65ecb5f59dd758e85eb57ef37393491232d968b2e70638713301",
    "archive_ref": "phase2-terminal-results",
    "archive_sha256": "0f2b0af3ede4eb6918be413f56757e48bc3ffd0829aaadd8100b609a84819094"
  },
  "protocol": {
    "items_per_suite": 300,
    "actual_repeats_per_item": 5,
    "planned_decisions_per_suite_per_model": 1500,
    "repeats_are_independent_items": false,
    "same_original_item_order_and_shuffle_views_as_offline_peers": true,
    "total_records": 6000,
    "actual_forwards": 5780,
    "context_refusals": 220,
    "no_retries": true,
    "suites": [
      "pubmedqa",
      "helpsteer2"
    ]
  },
  "rows": [
    {
      "model": "Trio-Spark v1.1 (released / T4)",
      "measurement_path": "Released serving configuration / local T4",
      "suites": {
        "pubmedqa": {
          "planned_n": 1500,
          "valid_n": 1500,
          "probability_valid_n": 1500,
          "correct": 1290,
          "strict_accuracy": 0.86,
          "macro_f1": 0.8454670329670331,
          "decision_score": 42.61921931775458,
          "decision_score_ci95": [
            37.32403714188152,
            47.01797126229726
          ],
          "full_denominator_loss": 0.2703782385747404,
          "empirical_prior_loss": 0.47120000000000006,
          "context_refusal_n": 0,
          "nll": 0.44722183857985554,
          "nll_infinite": false,
          "nll_clipped_1e12": 0.44722183857985554,
          "brier_sum": 0.27037823857474047,
          "ece15": 0.180594230110335,
          "ece10": 0.1749366165772181,
          "expected_score_mae_valid_probabilities": null,
          "ordered_rps_valid_probabilities": null,
          "argmax_mae_valid_score": null,
          "native_scalar_mae": null,
          "native_scalar_valid_n": 0,
          "errors": {},
          "repeat_flip_rate": 0.0,
          "repeat_flip_items": 300,
          "order_flip_rate": null,
          "order_flip_items": 0,
          "bootstrap": {
            "draws": 2000,
            "seed": 17,
            "rng": "numpy.default_rng PCG64; not claimed official harness RNG",
            "unit": "item, retaining five actual repeats"
          },
          "local_t4_completion_latency_seconds": {
            "n": 1500,
            "p50": 0.46583798799997567,
            "p95": 0.6304955460000201,
            "p99": 0.7287019070001861,
            "quantile": "nearest_rank"
          }
        },
        "helpsteer2": {
          "planned_n": 1500,
          "valid_n": 1390,
          "probability_valid_n": 1390,
          "correct": 575,
          "strict_accuracy": 0.38333333333333336,
          "macro_f1": 0.271754497738789,
          "decision_score": -2.162949681168036,
          "decision_score_ci95": [
            -12.637827323359343,
            5.359739756586361
          ],
          "full_denominator_loss": 0.16749331814256166,
          "empirical_prior_loss": 0.16394722222222222,
          "context_refusal_n": 110,
          "nll": 1.408854882626892,
          "nll_infinite": false,
          "nll_clipped_1e12": 1.408854882626892,
          "brier_sum": 0.7117449935072714,
          "ece15": 0.07116565421706975,
          "ece10": 0.07116565421706973,
          "expected_score_mae_valid_probabilities": 1.0900910707457876,
          "ordered_rps_valid_probabilities": 0.1631222857653543,
          "argmax_mae_valid_score": 1.0,
          "native_scalar_mae": null,
          "native_scalar_valid_n": 0,
          "errors": {
            "ContextLengthExceeded": 110
          },
          "repeat_flip_rate": 0.0,
          "repeat_flip_items": 278,
          "order_flip_rate": null,
          "order_flip_items": 0,
          "bootstrap": {
            "draws": 2000,
            "seed": 17,
            "rng": "numpy.default_rng PCG64; not claimed official harness RNG",
            "unit": "item, retaining five actual repeats"
          },
          "local_t4_completion_latency_seconds": {
            "n": 1390,
            "p50": 0.6011956449999616,
            "p95": 1.0180162380002002,
            "p99": 1.0698611850000361,
            "quantile": "nearest_rank"
          }
        }
      }
    },
    {
      "model": "Trio-Spark v1.2 (released / T4)",
      "measurement_path": "Released serving configuration / local T4",
      "suites": {
        "pubmedqa": {
          "planned_n": 1500,
          "valid_n": 1500,
          "probability_valid_n": 1500,
          "correct": 1310,
          "strict_accuracy": 0.8733333333333333,
          "macro_f1": 0.8625578703703703,
          "decision_score": 61.77581389991018,
          "decision_score_ci95": [
            51.22982997061289,
            71.10959484949016
          ],
          "full_denominator_loss": 0.18011236490362317,
          "empirical_prior_loss": 0.47120000000000006,
          "context_refusal_n": 0,
          "nll": 0.2936284344928907,
          "nll_infinite": false,
          "nll_clipped_1e12": 0.2936284344928907,
          "brier_sum": 0.18011236490362315,
          "ece15": 0.04746295953838707,
          "ece10": 0.03894181797267888,
          "expected_score_mae_valid_probabilities": null,
          "ordered_rps_valid_probabilities": null,
          "argmax_mae_valid_score": null,
          "native_scalar_mae": null,
          "native_scalar_valid_n": 0,
          "errors": {},
          "repeat_flip_rate": 0.0,
          "repeat_flip_items": 300,
          "order_flip_rate": null,
          "order_flip_items": 0,
          "bootstrap": {
            "draws": 2000,
            "seed": 17,
            "rng": "numpy.default_rng PCG64; not claimed official harness RNG",
            "unit": "item, retaining five actual repeats"
          },
          "local_t4_completion_latency_seconds": {
            "n": 1500,
            "p50": 0.46777860999964105,
            "p95": 0.6331967930000246,
            "p99": 0.7293191110002226,
            "quantile": "nearest_rank"
          }
        },
        "helpsteer2": {
          "planned_n": 1500,
          "valid_n": 1390,
          "probability_valid_n": 1390,
          "correct": 595,
          "strict_accuracy": 0.39666666666666667,
          "macro_f1": 0.2618473963707987,
          "decision_score": -6.369577955912664,
          "decision_score_ci95": [
            -19.406055669522505,
            6.106287586247955
          ],
          "full_denominator_loss": 0.17438996834822007,
          "empirical_prior_loss": 0.16394722222222222,
          "context_refusal_n": 110,
          "nll": 1.7134557864261744,
          "nll_infinite": false,
          "nll_clipped_1e12": 1.7134557864261744,
          "brier_sum": 0.7936277092212808,
          "ece15": 0.25074776520905734,
          "ece10": 0.24942325290637946,
          "expected_score_mae_valid_probabilities": 0.9158209175938152,
          "ordered_rps_valid_probabilities": 0.17056471404484178,
          "argmax_mae_valid_score": 1.0179856115107915,
          "native_scalar_mae": null,
          "native_scalar_valid_n": 0,
          "errors": {
            "ContextLengthExceeded": 110
          },
          "repeat_flip_rate": 0.0,
          "repeat_flip_items": 278,
          "order_flip_rate": null,
          "order_flip_items": 0,
          "bootstrap": {
            "draws": 2000,
            "seed": 17,
            "rng": "numpy.default_rng PCG64; not claimed official harness RNG",
            "unit": "item, retaining five actual repeats"
          },
          "local_t4_completion_latency_seconds": {
            "n": 1390,
            "p50": 0.6075539540001955,
            "p95": 1.0178570729999592,
            "p99": 1.0745580439997866,
            "quantile": "nearest_rank"
          }
        }
      }
    }
  ],
  "paired_decision_score_difference_ci95": {
    "pubmedqa": {
      "direction": "candidate minus old",
      "ci95": [
        12.619648057893498,
        25.236046124088055
      ],
      "paired_item_cluster_draws": 2000
    },
    "helpsteer2": {
      "direction": "candidate minus old",
      "ci95": [
        -14.148103927469089,
        6.078385813804706
      ],
      "paired_item_cluster_draws": 2000
    }
  },
  "notes": [
    "Each suite retains all1500 decisions permodel in strict accuracy and DecisionScore. HelpSteer2 has1390 valid probabilities and110 fixed context refusals permodel; originaluniform-five fallback is used only for those refusals in DecisionScore.",
    "NLL, ECE15, Brier, expected-scoreMAE and conditionalRPS use probability-valid counts. PubMedQA has1500 valid probabilities permodel; HelpSteer2 has1390.",
    "This compares complete frozen serving configurations; it does not isolate model changes from calibration changes.",
    "Same benchmark items and shuffled views do not imply identical provider wire format or internal tokenization.",
    "Banking77 serving measurements and d1 measurements remain unavailable.",
    "The v1.2 serving PubMedQA ECE15 is below Jev, but above Clef and Clef-Flash; no overall lowest-ECE or comprehensiveSOTA claim is supported.",
    "Latency fields measure local T4 completion, not end-to-end public API latency."
  ]
}