{"schema_version":"mcp-selection-lab.empirical-selection-reliability.v1","capability":"EMPIRICAL_SELECTION_RELIABILITY","status":"MEASURED_HISTORICAL_EVIDENCE","interpretation":"This is measured historical selection reliability from frozen benchmarks.","predictive_claim":false,"scanner_predictions_included":false,"aggregation_policy":{"schema_version":"mcp-selection-lab.empirical-selection-reliability.aggregation-policy.v1","capability":"EMPIRICAL_SELECTION_RELIABILITY","policy_id":"EMPIRICAL_SELECTION_RELIABILITY_AGGREGATION_V1","status":"FROZEN_IMPLEMENTATION_POLICY","interpretation":"This is measured historical selection reliability from frozen benchmarks.","eligible_evidence":{"include_only":"immutable completed benchmark evidence with a bound measurement identity","excluded":["abandoned drafts","uncommitted experiments","rejected gold cases","scanner predictions","provider infrastructure failures as semantic misses"],"included_tracks":["selection-evidence-expansion-v1","external-gold-v1","external-gold-expansion-v2-batch1","external-gold-expansion-v2-batch2","external-gold-expansion-v2-batch3","external-gold-expansion-v2-batch4","external-gold-expansion-v2-batch5","external-gold-expansion-v2-batch6"]},"subject_identity":{"merge_key":"exact canonical subject identity within the same provenance and identity mode","cross_provenance_aliasing":false,"version_bound_to_current_live_merge":false,"mixed_identity_status":"MIXED"},"aggregation":{"batch_independence":"each immutable benchmark/batch remains a separate batch_metrics record","pooling":"NONE","combined_headline_allowed":false,"combined_accuracy_allowed":false,"summary_fields":["total_measured_cases","number_of_batches","strict_accuracy_range_across_batches","latest_strict_accuracy","lowest_observed_strict_accuracy"],"provider_failures":"count separately; exclude from semantic miss count and strict semantic denominator","misses":"retain adjudicated observed semantic misses and case identities; do not infer predictive categories"},"reliability_status_rules":{"NO_MEASUREMENT_EVIDENCE":"zero eligible frozen cases","LIMITED_MEASUREMENT_EVIDENCE":"one eligible independent completed batch with fewer than 24 frozen cases","MEASURED":"one eligible independent completed batch with at least 24 frozen cases","MULTI_BATCH_MEASURED":"at least two eligible independent completed batches"},"public_claim_boundaries":{"predictive_claim":false,"future_failure_claim":false,"transport_health_claim":false,"profile_trust_grade_claim":false,"availability_claim":false,"current_live_status_claim":false,"benchmark_promotion_claim":false,"automatic_subject_ranking":false},"frozen_at":"2026-09-10T00:00:00Z"},"separation":{"transport_runtime_health":"SEPARATE_PROVIDER_VALID_AND_INFRASTRUCTURE_FIELDS","profile_trust_grade":"SEPARATE","availability":"NOT_EVALUATED","current_live_status":"NOT_CLAIMED","benchmark_promotion_level":"REPORTED_AS_SOURCE_EVIDENCE_ONLY","automatic_subject_ranking":false},"batch_records":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","metrics":{"frozen_case_count":240,"provider_valid_cases":240,"infrastructure_failures":0,"strict_correct":240,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.9842460800584412,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"240/240"},"measured_at":"2026-09-10T13:23:12.077611Z","latest_measured_at":"2026-09-10T13:23:12.077611Z","subject_metrics":[{"canonical_subject_id":"cloudflare-docs-mcp","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}},{"canonical_subject_id":"context7","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}},{"canonical_subject_id":"deepwiki-mcp","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}},{"canonical_subject_id":"dex-paprika-mcp-source-snapshot","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}},{"canonical_subject_id":"exa-mcp","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}},{"canonical_subject_id":"gitmcp-docs","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}},{"canonical_subject_id":"metropolitan-museum-mcp-source-snapshot","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}},{"canonical_subject_id":"national-parks-mcp-source-snapshot","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}},{"canonical_subject_id":"nixos-mcp-source-snapshot","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}},{"canonical_subject_id":"uk-premises-licence-mcp","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}}],"semantic_misses":[],"evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"},"source_files":[{"path":"validation/selection-evidence-expansion-v1/measurement-artifact-v1.json","sha256":"b18516e83f30484f70165e3360504c0228512631058894bb2d39f12a137d6974"},{"path":"validation/selection-evidence-expansion-v1/case-freeze-v1.json","sha256":"d55b04998bc670a68e944e4b4534e6d65c840520ac93c721b3700c12c4f1c0a7"},{"path":"validation/selection-evidence-expansion-v1/model-lock-v1.json","sha256":"adf4abc13e2313e4d2cc08b27ab96fed43c9bae93e0f23d29ba0d8cd6eb3e02c"},{"path":"validation/selection-evidence-expansion-v1/immutable-measurement-receipt-v1.json","sha256":"bc0a61269913e0207132ad6135d89667ce6ccb00bdd01a25e31e9d1129b9c5d3"},{"path":"validation/selection-evidence-expansion-v1/promotion-evaluation-receipt-v1.json","sha256":"1b4495a16c34723f01aa01d696dd27987fea913bdbd93b39cc7942b72bf05e4d"}],"subject_identity_metadata":{"context7":{"candidate_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","canonical_id":"context7","case_count":24,"identity_mode":"LIVE_TOOLS_LIST","name":"Context7","schema_path":"validation/selection-evidence-expansion-v1/subjects/context7/schema-v1.json","tool_count":2},"exa-mcp":{"candidate_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","canonical_id":"exa-mcp","case_count":24,"identity_mode":"LIVE_TOOLS_LIST","name":"Exa MCP","schema_path":"validation/selection-evidence-expansion-v1/subjects/exa-mcp/schema-v1.json","tool_count":2},"cloudflare-docs-mcp":{"candidate_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","canonical_id":"cloudflare-docs-mcp","case_count":24,"identity_mode":"LIVE_TOOLS_LIST","name":"Cloudflare Docs MCP","schema_path":"validation/selection-evidence-expansion-v1/subjects/cloudflare-docs-mcp/schema-v1.json","tool_count":2},"deepwiki-mcp":{"candidate_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","canonical_id":"deepwiki-mcp","case_count":24,"identity_mode":"LIVE_TOOLS_LIST","name":"DeepWiki MCP","schema_path":"validation/selection-evidence-expansion-v1/subjects/deepwiki-mcp/schema-v1.json","tool_count":3},"gitmcp-docs":{"candidate_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","canonical_id":"gitmcp-docs","case_count":24,"identity_mode":"LIVE_TOOLS_LIST","name":"GitMCP Docs","schema_path":"validation/selection-evidence-expansion-v1/subjects/gitmcp-docs/schema-v1.json","tool_count":5},"uk-premises-licence-mcp":{"candidate_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","canonical_id":"uk-premises-licence-mcp","case_count":24,"identity_mode":"LIVE_TOOLS_LIST","name":"UK Premises Licence MCP","schema_path":"validation/selection-evidence-expansion-v1/subjects/uk-premises-licence-mcp/schema-v1.json","tool_count":8},"nixos-mcp-source-snapshot":{"candidate_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","canonical_id":"nixos-mcp-source-snapshot","case_count":24,"identity_mode":"VERSION_BOUND_SOURCE_SNAPSHOT","name":"NixOS MCP (source snapshot)","schema_path":"validation/selection-evidence-expansion-v1/subjects/nixos-mcp-source-snapshot/schema-v1.json","tool_count":18},"dex-paprika-mcp-source-snapshot":{"candidate_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","canonical_id":"dex-paprika-mcp-source-snapshot","case_count":24,"identity_mode":"VERSION_BOUND_SOURCE_SNAPSHOT","name":"DEX Paprika MCP (source snapshot)","schema_path":"validation/selection-evidence-expansion-v1/subjects/dex-paprika-mcp-source-snapshot/schema-v1.json","tool_count":11},"national-parks-mcp-source-snapshot":{"candidate_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","canonical_id":"national-parks-mcp-source-snapshot","case_count":24,"identity_mode":"VERSION_BOUND_SOURCE_SNAPSHOT","name":"National Parks MCP (source snapshot)","schema_path":"validation/selection-evidence-expansion-v1/subjects/national-parks-mcp-source-snapshot/schema-v1.json","tool_count":6},"metropolitan-museum-mcp-source-snapshot":{"candidate_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","canonical_id":"metropolitan-museum-mcp-source-snapshot","case_count":24,"identity_mode":"VERSION_BOUND_SOURCE_SNAPSHOT","name":"Metropolitan Museum MCP (source snapshot)","schema_path":"validation/selection-evidence-expansion-v1/subjects/metropolitan-museum-mcp-source-snapshot/schema-v1.json","tool_count":3}}},{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":18,"strict_accuracy":0.75,"wilson_95":{"lower":0.551006,"upper":0.880006},"semantic_coverage":1.0,"observed_semantic_miss_count":6,"strict_score":"18/24"},"measured_at":"2026-09-09T22:51:26.259023Z","latest_measured_at":"2026-09-09T22:51:26.259023Z","subject_metrics":[{"canonical_subject_id":"external-gold-v1:Context7","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2"}},{"canonical_subject_id":"external-gold-v1:DEX Paprika","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":1,"strict_accuracy":0.5,"wilson_95":{"lower":0.094531,"upper":0.905469},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"1/2"}},{"canonical_subject_id":"external-gold-v1:Google Maps","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":0,"strict_accuracy":0.0,"wilson_95":{"lower":0.0,"upper":0.65762},"semantic_coverage":1.0,"observed_semantic_miss_count":2,"strict_score":"0/2"}},{"canonical_subject_id":"external-gold-v1:Huge Icons","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":1,"strict_accuracy":0.5,"wilson_95":{"lower":0.094531,"upper":0.905469},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"1/2"}},{"canonical_subject_id":"external-gold-v1:Medical Calculator","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2"}},{"canonical_subject_id":"external-gold-v1:Metropolitan Museum","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2"}},{"canonical_subject_id":"external-gold-v1:National Parks","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2"}},{"canonical_subject_id":"external-gold-v1:NixOS","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":1,"strict_accuracy":0.5,"wilson_95":{"lower":0.094531,"upper":0.905469},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"1/2"}},{"canonical_subject_id":"external-gold-v1:OKX Exchange","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":1,"strict_accuracy":0.5,"wilson_95":{"lower":0.094531,"upper":0.905469},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"1/2"}},{"canonical_subject_id":"external-gold-v1:Reddit","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2"}},{"canonical_subject_id":"external-gold-v1:Scientific Computing","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2"}},{"canonical_subject_id":"external-gold-v1:Wikipedia","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2"}}],"semantic_misses":[{"case_id":"dex_paprika_001","subject_id":"external-gold-v1:DEX Paprika","expected_tool":"getStats","selected_tool":"getNetworks","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"GOLD_SCHEMA_CONTRADICTION","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json"},{"case_id":"google_maps_000","subject_id":"external-gold-v1:Google Maps","expected_tool":"maps_geocode","selected_tool":"search_nearby","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"NON_UNIQUE_FIRST_TOOL","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json"},{"case_id":"google_maps_001","subject_id":"external-gold-v1:Google Maps","expected_tool":"maps_geocode","selected_tool":"search_nearby","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"NON_UNIQUE_FIRST_TOOL","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json"},{"case_id":"huge_icons_000","subject_id":"external-gold-v1:Huge Icons","expected_tool":"list_icons","selected_tool":"search_icons","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"NON_UNIQUE_FIRST_TOOL","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json"},{"case_id":"nixos_001","subject_id":"external-gold-v1:NixOS","expected_tool":"nixos_search","selected_tool":"nixos_info","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"GENUINE_SEQUENCE_ERROR","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json"},{"case_id":"okx_exchange_000","subject_id":"external-gold-v1:OKX Exchange","expected_tool":"get_price","selected_tool":"get_candlesticks","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"PROCEDURE_ORDER_DISAGREEMENT","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json"}],"evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"},"source_files":[{"path":"validation/independent-holdout-v1/external-gold-v1/real-model-measurement-v2.manifest.json","sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd"},{"path":"validation/independent-holdout-v1/external-gold-v1/real-model-measurement-v2.sha256","sha256":"2ab0e9a4904d027d5ee863309ecda835c322a7b68b053b422d6650b4303b5969"},{"path":"validation/independent-holdout-v1/external-gold-v1/real-model-lock-v2.json","sha256":"b5d5209172c6b65e6b13daa348a1992355370760d5d5cadda9620cd2455eece5"},{"path":"validation/independent-holdout-v1/external-gold-v1/final-verification-receipt.json","sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"},{"path":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json","sha256":"bb64f7d7a59cdbbb5ccda4ea1deb7feb52691fb1aff63b951d93964e2df49a74"},{"path":"validation/independent-holdout-v1/external-gold-v1/real-model-measurement-v2.parts/000.b64","sha256":"9366cfa5bb43fb4c0a6d2ae70e8a147362c493a4fd19ae23763d9eedce5f9271"},{"path":"validation/independent-holdout-v1/external-gold-v1/real-model-measurement-v2.parts/001.b64","sha256":"59876d5d611548e62d815f11a2945a3a40b9cb2308f89c823af9539c9905cfae"},{"path":"validation/independent-holdout-v1/external-gold-v1/real-model-measurement-v2.parts/002.b64","sha256":"cd654ae8d99361878efad7be33036daced6367681522724a90e878d55a6ef689"},{"path":"validation/independent-holdout-v1/external-gold-v1/real-model-measurement-v2.parts/003.b64","sha256":"b02ee1a9db9b5264aeac7279caee4e36c25c04bd87d0005f779907ddae0c3019"},{"path":"validation/independent-holdout-v1/external-gold-v1/real-model-measurement-v2.parts/004.b64","sha256":"158694907e71c928fc760972d24f09fae717643776fd07f5deecf5727a359a80"}]},{"batch_id":"external-gold-expansion-v2-batch1","benchmark_id":"external-gold-expansion-v2-batch1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":23,"strict_accuracy":0.9583333333333334,"wilson_95":{"lower":0.7975819352134406,"upper":0.9926065438344942},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"23/24"},"measured_at":"2026-09-10T15:38:03.968297Z","latest_measured_at":"2026-09-10T15:38:03.968297Z","subject_metrics":[{"canonical_subject_id":"datagouv-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.6096657120978346,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"deepwiki-mcp","metrics":{"frozen_case_count":4,"provider_valid_cases":4,"infrastructure_failures":0,"strict_correct":3,"strict_accuracy":0.75,"wilson_95":{"lower":0.30064184258240184,"upper":0.9544127391902995},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"3/4"}},{"canonical_subject_id":"exa-mcp","metrics":{"frozen_case_count":4,"provider_valid_cases":4,"infrastructure_failures":0,"strict_correct":4,"strict_accuracy":1.0,"wilson_95":{"lower":0.5101091635454027,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"4/4"}},{"canonical_subject_id":"microsoft-learn","metrics":{"frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5"}},{"canonical_subject_id":"microsoft-release-communications","metrics":{"frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5"}}],"semantic_misses":[{"case_id":"deepwiki-004","subject_id":"deepwiki-mcp","expected_tool":"ask_question","selected_tool":"read_wiki_structure","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"SEMANTIC_MISS","source_adjudication_artifact":"validation/external-gold-expansion-v2/measurement-v2-batch1/semantic-miss-adjudication-v2-batch1.json"}],"evidence_identity":{"freeze_input_commit":"c614ae51750d12ce113b5e3d51429f52e529f08c","final_freeze_receipt_commit":"b691170","measurement_commit":"b9f358b1a65aba348fa1c0e625eac6da1f31b651","policy_evaluation_commit":"3e560e5767f181831c65569b893ca789cd3bdff0","case_freeze_sha256":"17777d2c3983cd444353b8d69dcc98836a1909ee6fde6f5c8cf18cf9b059427e","model_lock_sha256":"c4330165deb3d8587e224a2ca1b3d886db60801ca436710a0edace55525923d2","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ab085cedf13eef5724f4e160b0f49f8402d62bc5b10203de0f8d81e41f2a017e","measurement_artifact_sha256":"01243422aa946296f235810aaee6f065ec21d9e78a01017ee8161e5e61a8b5e8","policy_evaluation_sha256":"4e710a9c5205660cc07603d2fd7538aa063eda4c2574c29308f9f255e4706f82"},"subject_identity_metadata":{"datagouv-mcp":{"subject_order":1,"canonical_subject_id":"datagouv-mcp","provider_repository":"data.gouv.fr; https://github.com/datagouv/datagouv-mcp","mcp_url":"https://mcp.data.gouv.fr/mcp","identity_mode":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_SNAPSHOT","schema_acquisition":"Local Docker initialize/session handshake followed by tools/list only; no tools/call.","schema_sha256":"fa394d9f6dd207dd3f15ac012a75a0029bdc456e2c1614a7db79c3cd2046dc1d","tools_list_raw_sha256":"bc8d083019728eab35ae94c628f6506d17104c759bfb559f9e494e30a87d0a07","tool_count":10,"tool_names":["get_dataservice_info","get_dataservice_openapi_spec","get_dataset_info","get_metrics","get_resource_info","list_dataset_resources","query_resource_data","search_dataservices","search_datasets","search_organizations"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots/datagouv-mcp/schema-v2.json","schema_snapshot_sha256":"50617b0c4cfb612d275d131eaa63bbe1d778a27f4ed923ef46daf260cbed29d4","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"},"microsoft-learn":{"subject_order":2,"canonical_subject_id":"microsoft-learn","provider_repository":"MicrosoftDocs; https://github.com/MicrosoftDocs/mcp","mcp_url":"https://learn.microsoft.com/api/mcp","identity_mode":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_SNAPSHOT","schema_acquisition":"Local Docker initialize/session handshake followed by tools/list only; no tools/call.","schema_sha256":"9729e31773dce5de4c9a08080af54658b13ff9cf96a078159fe234ef86581b7f","tools_list_raw_sha256":"380a2bbd84cafc59c9c7507baef0cedde7b40220dfd74b6cb8db601c473e73e0","tool_count":3,"tool_names":["microsoft_code_sample_search","microsoft_docs_fetch","microsoft_docs_search"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots/microsoft-learn/schema-v2.json","schema_snapshot_sha256":"dda9b6cfc3a8a93a2f5109e0772456016e586fb8bb40cfea42a607bedac46a1a","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"},"microsoft-release-communications":{"subject_order":3,"canonical_subject_id":"microsoft-release-communications","provider_repository":"MicrosoftDocs; https://github.com/MicrosoftDocs/microsoft-365-docs","mcp_url":"https://www.microsoft.com/releasecommunications/mcp","identity_mode":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_SNAPSHOT","schema_acquisition":"Local Docker initialize/session handshake followed by tools/list only; no tools/call.","schema_sha256":"f934d7f2ed46be9a4fc338c68259683b701c58cf0839e6cf2e2769dd02aa8b97","tools_list_raw_sha256":"d6de197436f503f59595ce18073b4471b4b008b48dbeb1bfbda2c0d8c3d8692b","tool_count":4,"tool_names":["get_azure_update_by_id","get_m365_roadmap_by_id","get_recent_azure_updates","get_recent_m365_roadmaps"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots/microsoft-release-communications/schema-v2.json","schema_snapshot_sha256":"94ed8a0076091d64801956b0215cab38725da7c33aa42f5d4c023c9394ad87b4","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"},"exa-mcp":{"subject_order":4,"canonical_subject_id":"exa-mcp","provider_repository":"Exa Labs; https://github.com/exa-labs/exa-mcp-server","mcp_url":"https://mcp.exa.ai/mcp","identity_mode":"CURRENT_LIVE_TOOLS_LIST_INHERITED_V1_AND_FROZEN_BY_COPY","schema_acquisition":"Exact immutable Expansion V1 tools/list schema snapshot copied into the additive V2 freeze directory; no V2 contact made.","schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","tool_count":2,"tool_names":["web_fetch_exa","web_search_exa"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots/exa-mcp/schema-v2.json","schema_snapshot_sha256":"d98db3a5825b1d97d2611369e8bfa3c63bfff19a834d14a686bd099c9a13e252","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"},"deepwiki-mcp":{"subject_order":5,"canonical_subject_id":"deepwiki-mcp","provider_repository":"Cognition/Devin; https://github.com/CognitionAI/deepwiki","mcp_url":"https://mcp.deepwiki.com/mcp","identity_mode":"CURRENT_LIVE_TOOLS_LIST_INHERITED_V1_AND_FROZEN_BY_COPY","schema_acquisition":"Exact immutable Expansion V1 tools/list schema snapshot copied into the additive V2 freeze directory; no V2 contact made.","schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","tool_count":3,"tool_names":["ask_question","read_wiki_contents","read_wiki_structure"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots/deepwiki-mcp/schema-v2.json","schema_snapshot_sha256":"bb02856d9514fd8f03237739275812ceba29916159607e92a5e29380876df479","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"}},"source_files":[{"path":"validation/external-gold-expansion-v2/measurement-v2-batch1/measurement-artifact-v2-batch1.json","sha256":"01243422aa946296f235810aaee6f065ec21d9e78a01017ee8161e5e61a8b5e8"},{"path":"validation/external-gold-expansion-v2/case-freeze-v2-batch1.json","sha256":"17777d2c3983cd444353b8d69dcc98836a1909ee6fde6f5c8cf18cf9b059427e"},{"path":"validation/external-gold-expansion-v2/final-subject-set-v2-batch1.json","sha256":"183b1f1222626d63185eb386a198e90c99d68be75829f2ce780a5b4f906112ce"},{"path":"validation/external-gold-expansion-v2/measurement-v2-batch1/semantic-miss-adjudication-v2-batch1.json","sha256":"bc596d74a02c59066e4db5b8b05f371ff96be118132b17e357ea2962f5ef7df5"},{"path":"validation/external-gold-expansion-v2/measurement-v2-batch1/raw-predictions-v2-batch1.jsonl","sha256":"ab085cedf13eef5724f4e160b0f49f8402d62bc5b10203de0f8d81e41f2a017e"}]},{"batch_id":"external-gold-expansion-v2-batch2","benchmark_id":"external-gold-expansion-v2-batch2","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","metrics":{"frozen_case_count":26,"provider_valid_cases":26,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":0.9230769230769231,"wilson_95":{"lower":0.7585844622837095,"upper":0.9786449115277238},"semantic_coverage":1.0,"observed_semantic_miss_count":2,"strict_score":"24/26"},"measured_at":"2026-09-10T16:49:53.346446Z","latest_measured_at":"2026-09-10T16:49:53.346446Z","subject_metrics":[{"canonical_subject_id":"datagouv-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":0.8333333333333334,"wilson_95":{"lower":0.43649717781352987,"upper":0.9699466302516935},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"5/6"}},{"canonical_subject_id":"deepwiki-mcp","metrics":{"frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5"}},{"canonical_subject_id":"gitmcp-docs","metrics":{"frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":4,"strict_accuracy":0.8,"wilson_95":{"lower":0.37553462976252533,"upper":0.9637758913675698},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"4/5"}},{"canonical_subject_id":"microsoft-learn","metrics":{"frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5"}},{"canonical_subject_id":"microsoft-release-communications","metrics":{"frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5"}}],"semantic_misses":[{"case_id":"datagouv-hard-004","subject_id":"datagouv-mcp","expected_tool":"list_dataset_resources","selected_tool":"get_dataset_info","boundary_family":"DATASET_VS_RESOURCE_LIST","adjudication":"SEMANTIC_MISS_GOLD_REMAINS_VALID","source_adjudication_artifact":"validation/external-gold-expansion-v2/measurement-v2-batch2/semantic-miss-adjudication-v2-batch2.json"},{"case_id":"gitmcp-hard-005","subject_id":"gitmcp-docs","expected_tool":"search_generic_documentation","selected_tool":"fetch_generic_documentation","boundary_family":"DOCUMENTED_BEHAVIOR_VS_IMPLEMENTATION","adjudication":"SEMANTIC_MISS_GOLD_REMAINS_VALID","source_adjudication_artifact":"validation/external-gold-expansion-v2/measurement-v2-batch2/semantic-miss-adjudication-v2-batch2.json"}],"evidence_identity":{"freeze_input_commit":"61f6511c41562c24019cd441b37bc6ee07283951","final_freeze_receipt_commit":"d3c55d2f96a4e45bf2577225ea16929877f2ded1","measurement_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","policy_evaluation_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","case_freeze_sha256":"ce21fc085bd9a0185c9e37d0d795221f286397cbc477ee941e876be24f7ba1b4","model_lock_sha256":"153647f94eed0ee3d3069229545846eef864fd43fa821f2d311dc3fb5d265ba9","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"8c4364ab598241ddbe4adb16453ba2e12b0917a1d1852caedb07884d1a54570b","measurement_artifact_sha256":"dd13aec8a90b2b5490c934c9ad9650c37733c65f18d824e2ae6066202176ced9","policy_evaluation_sha256":"736f154a5d5c3b8f0d77cd58fa0644ca97ad94c7a6b46af98d394c1dc1578f9a"},"subject_identity_metadata":{"datagouv-mcp":{"subject_order":1,"canonical_subject_id":"datagouv-mcp","provider_repository":"data.gouv.fr; https://github.com/datagouv/datagouv-mcp","mcp_url":"https://mcp.data.gouv.fr/mcp","identity_mode":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT","schema_acquisition":"Existing immutable V2 tools/list snapshot reused; no remote contact and no tools/call in Batch 2 preparation.","schema_sha256":"fa394d9f6dd207dd3f15ac012a75a0029bdc456e2c1614a7db79c3cd2046dc1d","tools_list_raw_sha256":"bc8d083019728eab35ae94c628f6506d17104c759bfb559f9e494e30a87d0a07","tool_count":10,"tool_names":["get_dataservice_info","get_dataservice_openapi_spec","get_dataset_info","get_metrics","get_resource_info","list_dataset_resources","query_resource_data","search_dataservices","search_datasets","search_organizations"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch2/datagouv-mcp/schema-v2-batch2.json","schema_snapshot_sha256":"50617b0c4cfb612d275d131eaa63bbe1d778a27f4ed923ef46daf260cbed29d4","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"},"gitmcp-docs":{"subject_order":2,"canonical_subject_id":"gitmcp-docs","provider_repository":"Ido Salomon; https://github.com/idosal/git-mcp","mcp_url":"https://gitmcp.io/docs","identity_mode":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT","schema_acquisition":"Existing stable corpus tools/list metadata snapshot reused; no remote contact and no tools/call in Batch 2 preparation.","schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","tools_list_raw_sha256":null,"tool_count":5,"tool_names":["fetch_generic_documentation","fetch_generic_url_content","match_common_libs_owner_repo_mapping","search_generic_code","search_generic_documentation"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch2/gitmcp-docs/schema-v2-batch2.json","schema_snapshot_sha256":"b90eb7608858850eff2594ab88943ab2f7be83630c7447dc1f34461a726c429b","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"},"microsoft-release-communications":{"subject_order":3,"canonical_subject_id":"microsoft-release-communications","provider_repository":"Microsoft; https://github.com/MicrosoftDocs/microsoft-365-docs","mcp_url":"https://www.microsoft.com/releasecommunications/mcp","identity_mode":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT","schema_acquisition":"Existing immutable V2 tools/list snapshot reused; no remote contact and no tools/call in Batch 2 preparation.","schema_sha256":"f934d7f2ed46be9a4fc338c68259683b701c58cf0839e6cf2e2769dd02aa8b97","tools_list_raw_sha256":"d6de197436f503f59595ce18073b4471b4b008b48dbeb1bfbda2c0d8c3d8692b","tool_count":4,"tool_names":["get_azure_update_by_id","get_m365_roadmap_by_id","get_recent_azure_updates","get_recent_m365_roadmaps"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch2/microsoft-release-communications/schema-v2-batch2.json","schema_snapshot_sha256":"94ed8a0076091d64801956b0215cab38725da7c33aa42f5d4c023c9394ad87b4","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"},"microsoft-learn":{"subject_order":4,"canonical_subject_id":"microsoft-learn","provider_repository":"MicrosoftDocs; https://github.com/MicrosoftDocs/mcp","mcp_url":"https://learn.microsoft.com/api/mcp","identity_mode":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT","schema_acquisition":"Existing immutable V2 tools/list snapshot reused; no remote contact and no tools/call in Batch 2 preparation.","schema_sha256":"9729e31773dce5de4c9a08080af54658b13ff9cf96a078159fe234ef86581b7f","tools_list_raw_sha256":"380a2bbd84cafc59c9c7507baef0cedde7b40220dfd74b6cb8db601c473e73e0","tool_count":3,"tool_names":["microsoft_code_sample_search","microsoft_docs_fetch","microsoft_docs_search"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch2/microsoft-learn/schema-v2-batch2.json","schema_snapshot_sha256":"dda9b6cfc3a8a93a2f5109e0772456016e586fb8bb40cfea42a607bedac46a1a","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"},"deepwiki-mcp":{"subject_order":5,"canonical_subject_id":"deepwiki-mcp","provider_repository":"Cognition/Devin; https://github.com/CognitionAI/deepwiki","mcp_url":"https://mcp.deepwiki.com/mcp","identity_mode":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT","schema_acquisition":"Existing immutable V2 tools/list snapshot reused; no remote contact and no tools/call in Batch 2 preparation.","schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","tools_list_raw_sha256":null,"tool_count":3,"tool_names":["ask_question","read_wiki_contents","read_wiki_structure"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch2/deepwiki-mcp/schema-v2-batch2.json","schema_snapshot_sha256":"bb02856d9514fd8f03237739275812ceba29916159607e92a5e29380876df479","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"}},"source_files":[{"path":"validation/external-gold-expansion-v2/measurement-v2-batch2/measurement-artifact-v2-batch2.json","sha256":"dd13aec8a90b2b5490c934c9ad9650c37733c65f18d824e2ae6066202176ced9"},{"path":"validation/external-gold-expansion-v2/case-freeze-v2-batch2.json","sha256":"ce21fc085bd9a0185c9e37d0d795221f286397cbc477ee941e876be24f7ba1b4"},{"path":"validation/external-gold-expansion-v2/final-subject-set-v2-batch2.json","sha256":"2713da3262e799306433f95555cdc19d14e5d54d25a976f9ed2279f07481ee19"},{"path":"validation/external-gold-expansion-v2/measurement-v2-batch2/semantic-miss-adjudication-v2-batch2.json","sha256":"064e119f4d197b5129051360f7c23ef9d7520a2074b77dc4a70daa474494db59"},{"path":"validation/external-gold-expansion-v2/measurement-v2-batch2/raw-predictions-v2-batch2.jsonl","sha256":"8c4364ab598241ddbe4adb16453ba2e12b0917a1d1852caedb07884d1a54570b"}]},{"batch_id":"external-gold-expansion-v2-batch3","benchmark_id":"external-gold-expansion-v2-batch3","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","metrics":{"frozen_case_count":36,"provider_valid_cases":36,"infrastructure_failures":0,"strict_correct":34,"strict_accuracy":0.944444,"wilson_95":{"lower":0.818553,"upper":0.98463},"semantic_coverage":1.0,"observed_semantic_miss_count":2,"strict_score":"34/36"},"measured_at":"2026-09-10T17:45:38.245198Z","latest_measured_at":"2026-09-10T17:46:44.384994Z","subject_metrics":[{"canonical_subject_id":"cloudflare-docs-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"datagouv-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"deepwiki-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"exa-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"gitmcp-docs","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":4,"strict_accuracy":0.666667,"wilson_95":{"lower":0.299993,"upper":0.903229},"semantic_coverage":1.0,"observed_semantic_miss_count":2,"strict_score":"4/6"}},{"canonical_subject_id":"microsoft-release-communications","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}}],"semantic_misses":[{"case_id":"b3-gm-005","subject_id":"gitmcp-docs","expected_tool":"fetch_generic_url_content","selected_tool":"fetch_generic_documentation","boundary_family":"DOCUMENTATION_VS_REFERENCED_URL","adjudication":"GOLD_REMAINS_VALID; semantic miss preserved in strict score. The public source supports the requested neighboring operation, and the selected tool is a plausible but incorrect neighboring retrieval mode.","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch3/measurement-v2-batch3/semantic-miss-adjudication-v2-batch3.json"},{"case_id":"b3-gm-006","subject_id":"gitmcp-docs","expected_tool":"search_generic_documentation","selected_tool":"fetch_generic_documentation","boundary_family":"DOCUMENTED_BEHAVIOR_VS_IMPLEMENTATION","adjudication":"GOLD_REMAINS_VALID; semantic miss preserved in strict score. The public source supports the requested neighboring operation, and the selected tool is a plausible but incorrect neighboring retrieval mode.","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch3/measurement-v2-batch3/semantic-miss-adjudication-v2-batch3.json"}],"evidence_identity":{"freeze_input_commit":"8450ed889756784a12ca4ebb8d84974d73b31778","final_freeze_receipt_commit":"a309bda7e8dd2867ccc7894dc7d42efbfc48395e","measurement_commit":"b81cc816158f7c0fbe7f8ac0a6fcf2b3f38f4c86","immutable_measurement_receipt_commit":"9d8290160849a5124d1896fc318d53470585ef41","case_freeze_sha256":"c1595743f48fe8ae6b3f70f2ea401731052b3bd7614297feabaa8172ce190ae5","model_lock_sha256":"48b5ed9c98f16cc2b3b76bc0da2e7ec8072ecd89ba1914683181b1c20074a3a8","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"c5cfe765f6b72489220cc73f689acb1c1c2c9284ffd8d093de0e597b645f6fbf","measurement_artifact_sha256":"7dcab9d4b71e7de59094c07cbca830ebf674b601ac6de024db0d3e677e7c0067","policy_evaluation_sha256":"3ea5b62044b4da6e92812831ebb00205970849accffe763a270248a24a0adef4"},"subject_identity_metadata":{"datagouv-mcp":{"subject_order":1,"canonical_subject_id":"datagouv-mcp","risk_group":"HIGH","provider_repository":"data.gouv.fr; https://github.com/datagouv/datagouv-mcp","mcp_url":"https://mcp.data.gouv.fr/mcp","source_schema":"schema-snapshots-batch2/datagouv-mcp/schema-v2-batch2.json","source_identity":"datagouv-readme-5a37d6f-2026-09-10","source_url":"https://github.com/datagouv/datagouv-mcp/blob/5a37d6f0f1da5504fd12918883498c26d6c28320/README.md","source_version":"repository commit 5a37d6f0f1da5504fd12918883498c26d6c28320; checked 2026-09-10","selection_rationale":"Largest qualified tool set with repeated catalog, metadata, resource, and query neighbors; useful high-risk boundary spread.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/datagouv-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"50617b0c4cfb612d275d131eaa63bbe1d778a27f4ed923ef46daf260cbed29d4","tool_count":10,"tool_names":["search_datasets","search_organizations","search_dataservices","get_dataservice_info","get_dataservice_openapi_spec","query_resource_data","get_dataset_info","list_dataset_resources","get_resource_info","get_metrics"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"},"gitmcp-docs":{"subject_order":2,"canonical_subject_id":"gitmcp-docs","risk_group":"HIGH","provider_repository":"Ido Salomon; https://github.com/idosal/git-mcp","mcp_url":"https://gitmcp.io/docs","source_schema":"schema-snapshots-batch2/gitmcp-docs/schema-v2-batch2.json","source_identity":"gitmcp-readme-c487a29-2026-05-08","source_url":"https://github.com/idosal/git-mcp/blob/c487a29895dcfcb5b672247e646426a56e2051c1/README.md","source_version":"repository commit c487a29895dcfcb5b672247e646426a56e2051c1; checked 2026-05-08","selection_rationale":"Multiple neighboring retrieval modes, including generic fetch, documentation search, code search, and library resolution, with null schemas.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/gitmcp-docs/schema-v2-batch3.json","schema_snapshot_sha256":"b90eb7608858850eff2594ab88943ab2f7be83630c7447dc1f34461a726c429b","tool_count":5,"tool_names":["match_common_libs_owner_repo_mapping","fetch_generic_documentation","search_generic_documentation","search_generic_code","fetch_generic_url_content"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"},"deepwiki-mcp":{"subject_order":3,"canonical_subject_id":"deepwiki-mcp","risk_group":"HIGH","provider_repository":"Cognition/Devin; https://github.com/CognitionAI/deepwiki","mcp_url":"https://mcp.deepwiki.com/mcp","source_schema":"schema-snapshots-batch2/deepwiki-mcp/schema-v2-batch2.json","source_identity":"deepwiki-aws-guide-2026-09-10","source_url":"https://docs.devin.ai/work-with-devin/deepwiki-mcp","source_version":"public guide and repository evidence captured 2026-09-10; repository commit 10f554ce266db71f3679a3c37eb140490384a81b","selection_rationale":"Structure, contents, and synthesized-answer tools share repository identity and exercise the observed question/structure boundary.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/deepwiki-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"bb02856d9514fd8f03237739275812ceba29916159607e92a5e29380876df479","tool_count":3,"tool_names":["ask_question","read_wiki_contents","read_wiki_structure"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"},"cloudflare-docs-mcp":{"subject_order":4,"canonical_subject_id":"cloudflare-docs-mcp","risk_group":"MEDIUM","provider_repository":"Cloudflare managed Documentation server; https://developers.cloudflare.com/agents/model-context-protocol/cloudflare/servers-for-cloudflare/","mcp_url":"https://docs.mcp.cloudflare.com/mcp","source_schema":"../selection-evidence-expansion-v1/subjects/cloudflare-docs-mcp/schema-v1.json","source_identity":"cloudflare-docs-official-2026-09-10","source_url":"https://developers.cloudflare.com/agents/model-context-protocol/cloudflare/servers-for-cloudflare/","source_version":"official pages last updated 2026-06-03 and 2026-07-28; checked 2026-09-10","selection_rationale":"A dedicated migration guide sits beside broad documentation search, providing a medium-risk control with explicit scope distinction.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/cloudflare-docs-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"5a02f1abdfa9ac3467c065d85fce654cc0debd79defebd3d4ed6ca198e1fad9c","tool_count":2,"tool_names":["search_cloudflare_documentation","migrate_pages_to_workers_guide"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"},"exa-mcp":{"subject_order":5,"canonical_subject_id":"exa-mcp","risk_group":"LOW","provider_repository":"Exa Labs; https://github.com/exa-labs/exa-mcp-server","mcp_url":"https://mcp.exa.ai/mcp","source_schema":"../selection-evidence-expansion-v1/subjects/exa-mcp/schema-v1.json","source_identity":"exa-official-reference-2026-09-10","source_url":"https://exa.ai/docs/reference/exa-mcp","source_version":"official reference crawled 2026-09-10; documentation index reports https://exa.ai/docs/llms.txt","selection_rationale":"Search and known-URL fetch are related retrieval operations but required input shapes provide comparatively explicit control separation.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/exa-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"d98db3a5825b1d97d2611369e8bfa3c63bfff19a834d14a686bd099c9a13e252","tool_count":2,"tool_names":["web_search_exa","web_fetch_exa"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"},"microsoft-release-communications":{"subject_order":6,"canonical_subject_id":"microsoft-release-communications","risk_group":"LOW","provider_repository":"Microsoft; https://github.com/MicrosoftDocs/microsoft-365-docs","mcp_url":"https://www.microsoft.com/releasecommunications/mcp","source_schema":"schema-snapshots-batch2/microsoft-release-communications/schema-v2-batch2.json","source_identity":"mrc-official-examples-2026-09-10","source_url":"https://learn.microsoft.com/en-us/microsoft-365/admin/manage/mrc-mcp?view=o365-worldwide","source_version":"documentation last updated 2026-08-25; source commit 03b138b88a789e34d5930f0a814d30ced515db4d","selection_rationale":"Azure/M365 and collection/specific-item pairs form explicit low-risk controls with distinct required identifiers or product families.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/microsoft-release-communications/schema-v2-batch3.json","schema_snapshot_sha256":"94ed8a0076091d64801956b0215cab38725da7c33aa42f5d4c023c9394ad87b4","tool_count":4,"tool_names":["get_m365_roadmap_by_id","get_azure_update_by_id","get_recent_m365_roadmaps","get_recent_azure_updates"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"},"microsoft-learn":{"subject_order":7,"canonical_subject_id":"microsoft-learn","risk_group":"RESERVE","provider_repository":"MicrosoftDocs; https://github.com/MicrosoftDocs/mcp","mcp_url":"https://learn.microsoft.com/api/mcp","source_schema":"schema-snapshots-batch2/microsoft-learn/schema-v2-batch2.json","source_identity":"microsoft-learn-official-2026-09-10","source_url":"https://learn.microsoft.com/en-us/training/support/mcp-developer-reference","source_version":"official page last updated 2025-12-03; source commit f8ffde185dfd232dbf5d187c22ce299eadc3d583","selection_rationale":"Qualified reserve with meaningful docs/code/fetch boundaries, held out to preserve a deterministic reserve and avoid overconcentrating the batch.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/microsoft-learn/schema-v2-batch3.json","schema_snapshot_sha256":"dda9b6cfc3a8a93a2f5109e0772456016e586fb8bb40cfea42a607bedac46a1a","tool_count":3,"tool_names":["microsoft_docs_search","microsoft_code_sample_search","microsoft_docs_fetch"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"}},"source_files":[{"path":"validation/external-gold-expansion-v2/batch3/measurement-v2-batch3/measurement-artifact-v2-batch3.json","sha256":"7dcab9d4b71e7de59094c07cbca830ebf674b601ac6de024db0d3e677e7c0067"},{"path":"validation/external-gold-expansion-v2/batch3/case-freeze-v2-batch3.json","sha256":"c1595743f48fe8ae6b3f70f2ea401731052b3bd7614297feabaa8172ce190ae5"},{"path":"validation/external-gold-expansion-v2/batch3/final-subject-set-v2-batch3.json","sha256":"2c84b0b96247869f90835e46c7b3922d7da664561e041bb3d1f43961afbd40e3"},{"path":"validation/external-gold-expansion-v2/batch3/measurement-v2-batch3/semantic-miss-adjudication-v2-batch3.json","sha256":"09e12b8cb3a996e51807bbe0a470cba0aec86a99a30d15674b192424e3e18308"},{"path":"validation/external-gold-expansion-v2/batch3/measurement-v2-batch3/raw-predictions-v2-batch3.jsonl","sha256":"c5cfe765f6b72489220cc73f689acb1c1c2c9284ffd8d093de0e597b645f6fbf"}]},{"batch_id":"external-gold-expansion-v2-batch4","benchmark_id":"external-gold-expansion-v2-batch4","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","metrics":{"frozen_case_count":48,"provider_valid_cases":48,"infrastructure_failures":0,"strict_correct":46,"strict_accuracy":0.958333,"wilson_95":{"lower":0.860243,"upper":0.988498,"lower_percent":86.0243,"upper_percent":98.8498},"semantic_coverage":1.0,"observed_semantic_miss_count":2,"strict_score":"46/48"},"measured_at":"2026-09-10T18:47:01.455700Z","latest_measured_at":"2026-09-10T18:48:10.224038Z","subject_metrics":[{"canonical_subject_id":"cloudflare-docs-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"datagouv-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"deepwiki-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":0.833333,"wilson_95":{"lower":0.436497,"upper":0.969947,"lower_percent":43.6497,"upper_percent":96.9947},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"5/6"}},{"canonical_subject_id":"exa-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"microsoft-learn","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"microsoft-release-communications","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"national-parks-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"nixos-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":0.833333,"wilson_95":{"lower":0.436497,"upper":0.969947,"lower_percent":43.6497,"upper_percent":96.9947},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"5/6"}}],"semantic_misses":[{"case_id":"b4-dw-005","subject_id":"deepwiki-mcp","expected_tool":"read_wiki_contents","selected_tool":"read_wiki_structure","boundary_family":"CONTENTS_VS_QUESTION","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch4/measurement-v2-batch4/semantic-miss-adjudication-v2-batch4.json"},{"case_id":"b4-nx-001","subject_id":"nixos-mcp","expected_tool":"home_manager_search","selected_tool":"home_manager_info","boundary_family":"HOME_MANAGER_VS_DARWIN_SEARCH","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch4/measurement-v2-batch4/semantic-miss-adjudication-v2-batch4.json"}],"evidence_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"},"subject_identity_metadata":{"nixos-mcp":{"canonical_subject_id":"nixos-mcp","provider_repository":"Accenture/mcp-bench NixOS server; source snapshot in the frozen corpus","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned MCP-Bench source; no remote contact in Batch 4.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/nixos-tools-list.json","source_identity":"nix-official-manuals-2026-09-10","source_url":"https://nix.dev/manual/nix/2.28/command-ref/new-cli/nix3-flake","source_version":"Nix 2.28.8 reference manual and current Home Manager/nix-darwin manuals accessed 2026-09-10","selection_rationale":"The frozen tool set contains neighboring NixOS, Home Manager, nix-darwin, and flake/version operations with cross-platform and granularity distinctions.","schema_snapshot_sha256":"b905b8aed0530aa0078dd080ef1e3e9de0dec005cf06a3c9e74f3910ac131185","tool_count":18,"tool_names":["nixos_search","nixos_info","nixos_channels","nixos_stats","home_manager_search","home_manager_info","home_manager_stats","home_manager_list_options","home_manager_options_by_prefix","darwin_search","darwin_info","darwin_stats","darwin_list_options","darwin_options_by_prefix","nixos_flakes_stats","nixos_flakes_search","nixhub_package_versions","nixhub_find_version"],"subject_order":1,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"national-parks-mcp":{"canonical_subject_id":"national-parks-mcp","provider_repository":"Accenture/mcp-bench national parks server; source snapshot in the frozen corpus","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned MCP-Bench source; no remote contact in Batch 4.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/national-parks-tools-list.json","source_identity":"nps-api-docs-2017-08-15","source_url":"https://www.nps.gov/subjects/developer/api-documentation.htm","source_version":"NPS API documentation last updated 2017-08-15; accessed 2026-09-10","selection_rationale":"Six public-lands operations provide independent park search, detail, alert, visitor-center, campground, and event boundaries.","schema_snapshot_sha256":"4ed812f3622705efbcba74689502fa0de6f2bd80bb5249384a4ca53a2b950de9","tool_count":6,"tool_names":["findParks","getParkDetails","getAlerts","getVisitorCenters","getCampgrounds","getEvents"],"subject_order":2,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"datagouv-mcp":{"subject_order":3,"canonical_subject_id":"datagouv-mcp","risk_group":"HIGH","provider_repository":"data.gouv.fr; https://github.com/datagouv/datagouv-mcp","mcp_url":"https://mcp.data.gouv.fr/mcp","source_schema":"schema-snapshots-batch2/datagouv-mcp/schema-v2-batch2.json","source_identity":"datagouv-readme-5a37d6f-2026-09-10","source_url":"https://github.com/datagouv/datagouv-mcp/blob/5a37d6f0f1da5504fd12918883498c26d6c28320/README.md","source_version":"repository commit 5a37d6f0f1da5504fd12918883498c26d6c28320; checked 2026-09-10","selection_rationale":"Largest qualified tool set with repeated catalog, metadata, resource, and query neighbors; useful high-risk boundary spread.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/datagouv-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"50617b0c4cfb612d275d131eaa63bbe1d778a27f4ed923ef46daf260cbed29d4","tool_count":10,"tool_names":["search_datasets","search_organizations","search_dataservices","get_dataservice_info","get_dataservice_openapi_spec","query_resource_data","get_dataset_info","list_dataset_resources","get_resource_info","get_metrics"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"deepwiki-mcp":{"subject_order":4,"canonical_subject_id":"deepwiki-mcp","risk_group":"HIGH","provider_repository":"Cognition/Devin; https://github.com/CognitionAI/deepwiki","mcp_url":"https://mcp.deepwiki.com/mcp","source_schema":"schema-snapshots-batch2/deepwiki-mcp/schema-v2-batch2.json","source_identity":"deepwiki-aws-guide-2026-09-10","source_url":"https://docs.devin.ai/work-with-devin/deepwiki-mcp","source_version":"public guide and repository evidence captured 2026-09-10; repository commit 10f554ce266db71f3679a3c37eb140490384a81b","selection_rationale":"Structure, contents, and synthesized-answer tools share repository identity and exercise the observed question/structure boundary.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/deepwiki-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"bb02856d9514fd8f03237739275812ceba29916159607e92a5e29380876df479","tool_count":3,"tool_names":["ask_question","read_wiki_contents","read_wiki_structure"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"microsoft-release-communications":{"subject_order":5,"canonical_subject_id":"microsoft-release-communications","risk_group":"LOW","provider_repository":"Microsoft; https://github.com/MicrosoftDocs/microsoft-365-docs","mcp_url":"https://www.microsoft.com/releasecommunications/mcp","source_schema":"schema-snapshots-batch2/microsoft-release-communications/schema-v2-batch2.json","source_identity":"mrc-official-examples-2026-09-10","source_url":"https://learn.microsoft.com/en-us/microsoft-365/admin/manage/mrc-mcp?view=o365-worldwide","source_version":"documentation last updated 2026-08-25; source commit 03b138b88a789e34d5930f0a814d30ced515db4d","selection_rationale":"Azure/M365 and collection/specific-item pairs form explicit low-risk controls with distinct required identifiers or product families.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/microsoft-release-communications/schema-v2-batch3.json","schema_snapshot_sha256":"94ed8a0076091d64801956b0215cab38725da7c33aa42f5d4c023c9394ad87b4","tool_count":4,"tool_names":["get_m365_roadmap_by_id","get_azure_update_by_id","get_recent_m365_roadmaps","get_recent_azure_updates"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"cloudflare-docs-mcp":{"subject_order":6,"canonical_subject_id":"cloudflare-docs-mcp","risk_group":"MEDIUM","provider_repository":"Cloudflare managed Documentation server; https://developers.cloudflare.com/agents/model-context-protocol/cloudflare/servers-for-cloudflare/","mcp_url":"https://docs.mcp.cloudflare.com/mcp","source_schema":"../selection-evidence-expansion-v1/subjects/cloudflare-docs-mcp/schema-v1.json","source_identity":"cloudflare-docs-official-2026-09-10","source_url":"https://developers.cloudflare.com/agents/model-context-protocol/cloudflare/servers-for-cloudflare/","source_version":"official pages last updated 2026-06-03 and 2026-07-28; checked 2026-09-10","selection_rationale":"A dedicated migration guide sits beside broad documentation search, providing a medium-risk control with explicit scope distinction.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/cloudflare-docs-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"5a02f1abdfa9ac3467c065d85fce654cc0debd79defebd3d4ed6ca198e1fad9c","tool_count":2,"tool_names":["search_cloudflare_documentation","migrate_pages_to_workers_guide"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"microsoft-learn":{"subject_order":7,"canonical_subject_id":"microsoft-learn","risk_group":"RESERVE","provider_repository":"MicrosoftDocs; https://github.com/MicrosoftDocs/mcp","mcp_url":"https://learn.microsoft.com/api/mcp","source_schema":"schema-snapshots-batch2/microsoft-learn/schema-v2-batch2.json","source_identity":"microsoft-learn-official-2026-09-10","source_url":"https://learn.microsoft.com/en-us/training/support/mcp-developer-reference","source_version":"official page last updated 2025-12-03; source commit f8ffde185dfd232dbf5d187c22ce299eadc3d583","selection_rationale":"Qualified reserve with meaningful docs/code/fetch boundaries, held out to preserve a deterministic reserve and avoid overconcentrating the batch.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/microsoft-learn/schema-v2-batch3.json","schema_snapshot_sha256":"dda9b6cfc3a8a93a2f5109e0772456016e586fb8bb40cfea42a607bedac46a1a","tool_count":3,"tool_names":["microsoft_docs_search","microsoft_code_sample_search","microsoft_docs_fetch"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"exa-mcp":{"subject_order":8,"canonical_subject_id":"exa-mcp","risk_group":"LOW","provider_repository":"Exa Labs; https://github.com/exa-labs/exa-mcp-server","mcp_url":"https://mcp.exa.ai/mcp","source_schema":"../selection-evidence-expansion-v1/subjects/exa-mcp/schema-v1.json","source_identity":"exa-official-reference-2026-09-10","source_url":"https://exa.ai/docs/reference/exa-mcp","source_version":"official reference crawled 2026-09-10; documentation index reports https://exa.ai/docs/llms.txt","selection_rationale":"Search and known-URL fetch are related retrieval operations but required input shapes provide comparatively explicit control separation.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/exa-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"d98db3a5825b1d97d2611369e8bfa3c63bfff19a834d14a686bd099c9a13e252","tool_count":2,"tool_names":["web_search_exa","web_fetch_exa"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"}},"source_files":[{"path":"validation/external-gold-expansion-v2/batch4/measurement-v2-batch4/measurement-artifact-v2-batch4.json","sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9"},{"path":"validation/external-gold-expansion-v2/batch4/case-freeze-v2-batch4.json","sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d"},{"path":"validation/external-gold-expansion-v2/batch4/final-subject-set-v2-batch4.json","sha256":"3dcfc1b819a14c568e012b1c72f80969624d04d092c44398bcf08d62cf7e5497"},{"path":"validation/external-gold-expansion-v2/batch4/measurement-v2-batch4/semantic-miss-adjudication-v2-batch4.json","sha256":"f12dafa8590b6f44938de43a9f070f83d0c304d868d97cb7131a6dae9e58bcdf"},{"path":"validation/external-gold-expansion-v2/batch4/measurement-v2-batch4/raw-predictions-v2-batch4.jsonl","sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89"}]},{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","metrics":{"frozen_case_count":77,"provider_valid_cases":77,"infrastructure_failures":0,"strict_correct":74,"strict_accuracy":0.961039,"wilson_95":{"lower":0.8916,"upper":0.986662,"lower_percent":89.16,"upper_percent":98.6662},"semantic_coverage":1.0,"observed_semantic_miss_count":3,"strict_score":"74/77"},"measured_at":"2026-09-10T19:13:49.842728Z","latest_measured_at":"2026-09-10T19:15:38.318428Z","subject_metrics":[{"canonical_subject_id":"cloudflare-docs-mcp","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}},{"canonical_subject_id":"datagouv-mcp","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}},{"canonical_subject_id":"deepwiki-mcp","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":4,"strict_accuracy":0.571429,"wilson_95":{"lower":0.250458,"upper":0.84178,"lower_percent":25.0458,"upper_percent":84.178},"semantic_coverage":1.0,"observed_semantic_miss_count":3,"strict_score":"4/7"}},{"canonical_subject_id":"exa-mcp","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}},{"canonical_subject_id":"google-maps-mcp","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}},{"canonical_subject_id":"metropolitan-museum-mcp","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}},{"canonical_subject_id":"microsoft-learn","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}},{"canonical_subject_id":"microsoft-release-communications","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}},{"canonical_subject_id":"national-parks-mcp","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}},{"canonical_subject_id":"nixos-mcp","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}},{"canonical_subject_id":"wikipedia-mcp-reference","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}}],"semantic_misses":[{"case_id":"b5-dw-003","subject_id":"deepwiki-mcp","expected_tool":"read_wiki_contents","selected_tool":"read_wiki_structure","boundary_family":"CONTENTS_VS_QUESTION","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch5/measurement-v2-batch5/semantic-miss-adjudication-v2-batch5.json"},{"case_id":"b5-dw-004","subject_id":"deepwiki-mcp","expected_tool":"ask_question","selected_tool":"read_wiki_contents","boundary_family":"QUESTION_VS_CONTENTS","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch5/measurement-v2-batch5/semantic-miss-adjudication-v2-batch5.json"},{"case_id":"b5-dw-006","subject_id":"deepwiki-mcp","expected_tool":"read_wiki_contents","selected_tool":"read_wiki_structure","boundary_family":"CONTENTS_VS_STRUCTURE","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch5/measurement-v2-batch5/semantic-miss-adjudication-v2-batch5.json"}],"evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"},"subject_identity_metadata":{"datagouv-mcp":{"subject_order":1,"canonical_subject_id":"datagouv-mcp","risk_group":"HIGH","provider_repository":"data.gouv.fr; https://github.com/datagouv/datagouv-mcp","mcp_url":"https://mcp.data.gouv.fr/mcp","source_schema":"schema-snapshots-batch2/datagouv-mcp/schema-v2-batch2.json","source_identity":"datagouv-readme-5a37d6f-2026-09-10","source_url":"https://github.com/datagouv/datagouv-mcp/blob/5a37d6f0f1da5504fd12918883498c26d6c28320/README.md","source_version":"repository commit 5a37d6f0f1da5504fd12918883498c26d6c28320; checked 2026-09-10","selection_rationale":"Largest qualified tool set with repeated catalog, metadata, resource, and query neighbors; useful high-risk boundary spread.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/datagouv-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"50617b0c4cfb612d275d131eaa63bbe1d778a27f4ed923ef46daf260cbed29d4","tool_count":10,"tool_names":["search_datasets","search_organizations","search_dataservices","get_dataservice_info","get_dataservice_openapi_spec","query_resource_data","get_dataset_info","list_dataset_resources","get_resource_info","get_metrics"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"deepwiki-mcp":{"subject_order":2,"canonical_subject_id":"deepwiki-mcp","risk_group":"HIGH","provider_repository":"Cognition/Devin; https://github.com/CognitionAI/deepwiki","mcp_url":"https://mcp.deepwiki.com/mcp","source_schema":"schema-snapshots-batch2/deepwiki-mcp/schema-v2-batch2.json","source_identity":"deepwiki-aws-guide-2026-09-10","source_url":"https://docs.devin.ai/work-with-devin/deepwiki-mcp","source_version":"public guide and repository evidence captured 2026-09-10; repository commit 10f554ce266db71f3679a3c37eb140490384a81b","selection_rationale":"Structure, contents, and synthesized-answer tools share repository identity and exercise the observed question/structure boundary.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/deepwiki-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"bb02856d9514fd8f03237739275812ceba29916159607e92a5e29380876df479","tool_count":3,"tool_names":["ask_question","read_wiki_contents","read_wiki_structure"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"microsoft-release-communications":{"subject_order":3,"canonical_subject_id":"microsoft-release-communications","risk_group":"LOW","provider_repository":"Microsoft; https://github.com/MicrosoftDocs/microsoft-365-docs","mcp_url":"https://www.microsoft.com/releasecommunications/mcp","source_schema":"schema-snapshots-batch2/microsoft-release-communications/schema-v2-batch2.json","source_identity":"mrc-official-examples-2026-09-10","source_url":"https://learn.microsoft.com/en-us/microsoft-365/admin/manage/mrc-mcp?view=o365-worldwide","source_version":"documentation last updated 2026-08-25; source commit 03b138b88a789e34d5930f0a814d30ced515db4d","selection_rationale":"Azure/M365 and collection/specific-item pairs form explicit low-risk controls with distinct required identifiers or product families.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/microsoft-release-communications/schema-v2-batch3.json","schema_snapshot_sha256":"94ed8a0076091d64801956b0215cab38725da7c33aa42f5d4c023c9394ad87b4","tool_count":4,"tool_names":["get_m365_roadmap_by_id","get_azure_update_by_id","get_recent_m365_roadmaps","get_recent_azure_updates"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"microsoft-learn":{"subject_order":4,"canonical_subject_id":"microsoft-learn","risk_group":"RESERVE","provider_repository":"MicrosoftDocs; https://github.com/MicrosoftDocs/mcp","mcp_url":"https://learn.microsoft.com/api/mcp","source_schema":"schema-snapshots-batch2/microsoft-learn/schema-v2-batch2.json","source_identity":"microsoft-learn-official-2026-09-10","source_url":"https://learn.microsoft.com/en-us/training/support/mcp-developer-reference","source_version":"official page last updated 2025-12-03; source commit f8ffde185dfd232dbf5d187c22ce299eadc3d583","selection_rationale":"Qualified reserve with meaningful docs/code/fetch boundaries, held out to preserve a deterministic reserve and avoid overconcentrating the batch.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/microsoft-learn/schema-v2-batch3.json","schema_snapshot_sha256":"dda9b6cfc3a8a93a2f5109e0772456016e586fb8bb40cfea42a607bedac46a1a","tool_count":3,"tool_names":["microsoft_docs_search","microsoft_code_sample_search","microsoft_docs_fetch"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"cloudflare-docs-mcp":{"subject_order":5,"canonical_subject_id":"cloudflare-docs-mcp","risk_group":"MEDIUM","provider_repository":"Cloudflare managed Documentation server; https://developers.cloudflare.com/agents/model-context-protocol/cloudflare/servers-for-cloudflare/","mcp_url":"https://docs.mcp.cloudflare.com/mcp","source_schema":"../selection-evidence-expansion-v1/subjects/cloudflare-docs-mcp/schema-v1.json","source_identity":"cloudflare-docs-official-2026-09-10","source_url":"https://developers.cloudflare.com/agents/model-context-protocol/cloudflare/servers-for-cloudflare/","source_version":"official pages last updated 2026-06-03 and 2026-07-28; checked 2026-09-10","selection_rationale":"A dedicated migration guide sits beside broad documentation search, providing a medium-risk control with explicit scope distinction.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/cloudflare-docs-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"5a02f1abdfa9ac3467c065d85fce654cc0debd79defebd3d4ed6ca198e1fad9c","tool_count":2,"tool_names":["search_cloudflare_documentation","migrate_pages_to_workers_guide"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"exa-mcp":{"subject_order":6,"canonical_subject_id":"exa-mcp","risk_group":"LOW","provider_repository":"Exa Labs; https://github.com/exa-labs/exa-mcp-server","mcp_url":"https://mcp.exa.ai/mcp","source_schema":"../selection-evidence-expansion-v1/subjects/exa-mcp/schema-v1.json","source_identity":"exa-official-reference-2026-09-10","source_url":"https://exa.ai/docs/reference/exa-mcp","source_version":"official reference crawled 2026-09-10; documentation index reports https://exa.ai/docs/llms.txt","selection_rationale":"Search and known-URL fetch are related retrieval operations but required input shapes provide comparatively explicit control separation.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/exa-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"d98db3a5825b1d97d2611369e8bfa3c63bfff19a834d14a686bd099c9a13e252","tool_count":2,"tool_names":["web_search_exa","web_fetch_exa"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"nixos-mcp":{"canonical_subject_id":"nixos-mcp","provider_repository":"Accenture/mcp-bench NixOS server; source snapshot in the frozen corpus","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned MCP-Bench source; no remote contact in Batch 4.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/nixos-tools-list.json","source_identity":"nix-official-manuals-2026-09-10","source_url":"https://nix.dev/manual/nix/2.28/command-ref/new-cli/nix3-flake","source_version":"Nix 2.28.8 reference manual and current Home Manager/nix-darwin manuals accessed 2026-09-10","selection_rationale":"The frozen tool set contains neighboring NixOS, Home Manager, nix-darwin, and flake/version operations with cross-platform and granularity distinctions.","schema_snapshot_sha256":"b905b8aed0530aa0078dd080ef1e3e9de0dec005cf06a3c9e74f3910ac131185","tool_count":18,"tool_names":["nixos_search","nixos_info","nixos_channels","nixos_stats","home_manager_search","home_manager_info","home_manager_stats","home_manager_list_options","home_manager_options_by_prefix","darwin_search","darwin_info","darwin_stats","darwin_list_options","darwin_options_by_prefix","nixos_flakes_stats","nixos_flakes_search","nixhub_package_versions","nixhub_find_version"],"subject_order":7,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"national-parks-mcp":{"canonical_subject_id":"national-parks-mcp","provider_repository":"Accenture/mcp-bench national parks server; source snapshot in the frozen corpus","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned MCP-Bench source; no remote contact in Batch 4.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/national-parks-tools-list.json","source_identity":"nps-api-docs-2017-08-15","source_url":"https://www.nps.gov/subjects/developer/api-documentation.htm","source_version":"NPS API documentation last updated 2017-08-15; accessed 2026-09-10","selection_rationale":"Six public-lands operations provide independent park search, detail, alert, visitor-center, campground, and event boundaries.","schema_snapshot_sha256":"4ed812f3622705efbcba74689502fa0de6f2bd80bb5249384a4ca53a2b950de9","tool_count":6,"tool_names":["findParks","getParkDetails","getAlerts","getVisitorCenters","getCampgrounds","getEvents"],"subject_order":8,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"wikipedia-mcp-reference":{"canonical_subject_id":"wikipedia-mcp-reference","provider_repository":"stable corpus / wikipedia MCP reconstruction","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned stable corpus; no remote contact in Batch 5.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/wikipedia-tools-list.json","schema_snapshot_sha256":"ba0abc3b84718f0f3b73ece26ae93f12513ef37027516dd75d19cce837cdcf65","tool_count":9,"tool_names":["search_wikipedia","get_article","get_summary","summarize_article_for_query","summarize_article_section","extract_key_facts","get_related_topics","get_sections","get_links"],"source_identity":"mediawiki-api-official-2026-09-10","source_url":"https://www.mediawiki.org/wiki/API:Search","source_version":"MediaWiki API documentation accessed 2026-09-10","subject_order":9,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"google-maps-mcp":{"canonical_subject_id":"google-maps-mcp","provider_repository":"stable corpus / Google Maps MCP reconstruction","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned stable corpus; no remote contact in Batch 5.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/phase-3/google-maps-tools-list.json","schema_snapshot_sha256":"fca473a69e8a5732bd84ed01049b9ba1bd2e7e706c073e643040145cca92a3a1","tool_count":7,"tool_names":["search_nearby","get_place_details","maps_geocode","maps_reverse_geocode","maps_distance_matrix","maps_directions","maps_elevation"],"source_identity":"google-maps-platform-official-2026-09-10","source_url":"https://developers.google.com/maps/documentation","source_version":"Google Maps Platform documentation accessed 2026-09-10","subject_order":10,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"metropolitan-museum-mcp":{"canonical_subject_id":"metropolitan-museum-mcp","provider_repository":"stable corpus / Metropolitan Museum MCP reconstruction","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned stable corpus; no remote contact in Batch 5.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/phase-3/metropolitan-museum-tools-list.json","schema_snapshot_sha256":"0ef880de13ed23c83110318eff4dd339756a3d8f4c8384c820476c54a5cfe2b2","tool_count":3,"tool_names":["list-departments","search-museum-objects","get-museum-object"],"source_identity":"met-collection-api-official-2026-09-10","source_url":"https://metmuseum.github.io/","source_version":"Met Collection API documentation accessed 2026-09-10","subject_order":11,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"}},"source_files":[{"path":"validation/external-gold-expansion-v2/batch5/measurement-v2-batch5/measurement-artifact-v2-batch5.json","sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb"},{"path":"validation/external-gold-expansion-v2/batch5/case-freeze-v2-batch5.json","sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc"},{"path":"validation/external-gold-expansion-v2/batch5/final-subject-set-v2-batch5.json","sha256":"2881b264136944f2df33463cc8f3f4618e3bf200ce3d4660be4358464e9ad151"},{"path":"validation/external-gold-expansion-v2/batch5/measurement-v2-batch5/semantic-miss-adjudication-v2-batch5.json","sha256":"32646e9f45f4931c98a6ca5a646761cc6b9595a7b941e6956605b25e72142f6f"},{"path":"validation/external-gold-expansion-v2/batch5/measurement-v2-batch5/raw-predictions-v2-batch5.jsonl","sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68"}]},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","metrics":{"frozen_case_count":80,"provider_valid_cases":80,"infrastructure_failures":0,"strict_correct":80,"strict_accuracy":1.0,"wilson_95":{"lower":0.954182,"upper":1.0,"lower_percent":95.4182,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"80/80"},"measured_at":"2026-09-11T00:07:39.419700Z","latest_measured_at":"2026-09-11T00:10:04.575675Z","subject_metrics":[{"canonical_subject_id":"cloudflare-docs-mcp","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}},{"canonical_subject_id":"datagouv-mcp","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}},{"canonical_subject_id":"exa-mcp","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}},{"canonical_subject_id":"google-maps-mcp","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}},{"canonical_subject_id":"metropolitan-museum-mcp","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}},{"canonical_subject_id":"microsoft-learn","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}},{"canonical_subject_id":"microsoft-release-communications","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}},{"canonical_subject_id":"national-parks-mcp","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}},{"canonical_subject_id":"nixos-mcp","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}},{"canonical_subject_id":"wikipedia-mcp-reference","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}}],"semantic_misses":[],"evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"},"subject_identity_metadata":{"datagouv-mcp":{"subject_order":1,"canonical_subject_id":"datagouv-mcp","risk_group":"HIGH","provider_repository":"data.gouv.fr; https://github.com/datagouv/datagouv-mcp","mcp_url":"https://mcp.data.gouv.fr/mcp","source_schema":"schema-snapshots-batch2/datagouv-mcp/schema-v2-batch2.json","source_identity":"datagouv-readme-5a37d6f-2026-09-10","source_url":"https://github.com/datagouv/datagouv-mcp/blob/5a37d6f0f1da5504fd12918883498c26d6c28320/README.md","source_version":"repository commit 5a37d6f0f1da5504fd12918883498c26d6c28320; checked 2026-09-10","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/datagouv-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"50617b0c4cfb612d275d131eaa63bbe1d778a27f4ed923ef46daf260cbed29d4","tool_count":10,"tool_names":["search_datasets","search_organizations","search_dataservices","get_dataservice_info","get_dataservice_openapi_spec","query_resource_data","get_dataset_info","list_dataset_resources","get_resource_info","get_metrics"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED"},"microsoft-release-communications":{"subject_order":2,"canonical_subject_id":"microsoft-release-communications","risk_group":"LOW","provider_repository":"Microsoft; https://github.com/MicrosoftDocs/microsoft-365-docs","mcp_url":"https://www.microsoft.com/releasecommunications/mcp","source_schema":"schema-snapshots-batch2/microsoft-release-communications/schema-v2-batch2.json","source_identity":"mrc-official-examples-2026-09-10","source_url":"https://learn.microsoft.com/en-us/microsoft-365/admin/manage/mrc-mcp?view=o365-worldwide","source_version":"documentation last updated 2026-08-25; source commit 03b138b88a789e34d5930f0a814d30ced515db4d","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/microsoft-release-communications/schema-v2-batch3.json","schema_snapshot_sha256":"94ed8a0076091d64801956b0215cab38725da7c33aa42f5d4c023c9394ad87b4","tool_count":4,"tool_names":["get_m365_roadmap_by_id","get_azure_update_by_id","get_recent_m365_roadmaps","get_recent_azure_updates"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED"},"microsoft-learn":{"subject_order":3,"canonical_subject_id":"microsoft-learn","risk_group":"RESERVE","provider_repository":"MicrosoftDocs; https://github.com/MicrosoftDocs/mcp","mcp_url":"https://learn.microsoft.com/api/mcp","source_schema":"schema-snapshots-batch2/microsoft-learn/schema-v2-batch2.json","source_identity":"microsoft-learn-official-2026-09-10","source_url":"https://learn.microsoft.com/en-us/training/support/mcp-developer-reference","source_version":"official page last updated 2025-12-03; source commit f8ffde185dfd232dbf5d187c22ce299eadc3d583","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/microsoft-learn/schema-v2-batch3.json","schema_snapshot_sha256":"dda9b6cfc3a8a93a2f5109e0772456016e586fb8bb40cfea42a607bedac46a1a","tool_count":3,"tool_names":["microsoft_docs_search","microsoft_code_sample_search","microsoft_docs_fetch"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED"},"cloudflare-docs-mcp":{"subject_order":4,"canonical_subject_id":"cloudflare-docs-mcp","risk_group":"MEDIUM","provider_repository":"Cloudflare managed Documentation server; https://developers.cloudflare.com/agents/model-context-protocol/cloudflare/servers-for-cloudflare/","mcp_url":"https://docs.mcp.cloudflare.com/mcp","source_schema":"../selection-evidence-expansion-v1/subjects/cloudflare-docs-mcp/schema-v1.json","source_identity":"cloudflare-docs-official-2026-09-10","source_url":"https://developers.cloudflare.com/agents/model-context-protocol/cloudflare/servers-for-cloudflare/","source_version":"official pages last updated 2026-06-03 and 2026-07-28; checked 2026-09-10","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/cloudflare-docs-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"5a02f1abdfa9ac3467c065d85fce654cc0debd79defebd3d4ed6ca198e1fad9c","tool_count":2,"tool_names":["search_cloudflare_documentation","migrate_pages_to_workers_guide"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED"},"exa-mcp":{"subject_order":5,"canonical_subject_id":"exa-mcp","risk_group":"LOW","provider_repository":"Exa Labs; https://github.com/exa-labs/exa-mcp-server","mcp_url":"https://mcp.exa.ai/mcp","source_schema":"../selection-evidence-expansion-v1/subjects/exa-mcp/schema-v1.json","source_identity":"exa-official-reference-2026-09-10","source_url":"https://exa.ai/docs/reference/exa-mcp","source_version":"official reference crawled 2026-09-10; documentation index reports https://exa.ai/docs/llms.txt","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/exa-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"d98db3a5825b1d97d2611369e8bfa3c63bfff19a834d14a686bd099c9a13e252","tool_count":2,"tool_names":["web_search_exa","web_fetch_exa"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED"},"nixos-mcp":{"canonical_subject_id":"nixos-mcp","provider_repository":"Accenture/mcp-bench NixOS server; source snapshot in the frozen corpus","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned MCP-Bench source; no remote contact in Batch 4.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/nixos-tools-list.json","source_identity":"nix-official-manuals-2026-09-10","source_url":"https://nix.dev/manual/nix/2.28/command-ref/new-cli/nix3-flake","source_version":"Nix 2.28.8 reference manual and current Home Manager/nix-darwin manuals accessed 2026-09-10","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss.","schema_snapshot_sha256":"b905b8aed0530aa0078dd080ef1e3e9de0dec005cf06a3c9e74f3910ac131185","tool_count":18,"tool_names":["nixos_search","nixos_info","nixos_channels","nixos_stats","home_manager_search","home_manager_info","home_manager_stats","home_manager_list_options","home_manager_options_by_prefix","darwin_search","darwin_info","darwin_stats","darwin_list_options","darwin_options_by_prefix","nixos_flakes_stats","nixos_flakes_search","nixhub_package_versions","nixhub_find_version"],"subject_order":6,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED"},"national-parks-mcp":{"canonical_subject_id":"national-parks-mcp","provider_repository":"Accenture/mcp-bench national parks server; source snapshot in the frozen corpus","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned MCP-Bench source; no remote contact in Batch 4.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/national-parks-tools-list.json","source_identity":"nps-api-docs-2017-08-15","source_url":"https://www.nps.gov/subjects/developer/api-documentation.htm","source_version":"NPS API documentation last updated 2017-08-15; accessed 2026-09-10","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss.","schema_snapshot_sha256":"4ed812f3622705efbcba74689502fa0de6f2bd80bb5249384a4ca53a2b950de9","tool_count":6,"tool_names":["findParks","getParkDetails","getAlerts","getVisitorCenters","getCampgrounds","getEvents"],"subject_order":7,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED"},"wikipedia-mcp-reference":{"canonical_subject_id":"wikipedia-mcp-reference","provider_repository":"stable corpus / wikipedia MCP reconstruction","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned stable corpus; no remote contact in Batch 5.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/wikipedia-tools-list.json","schema_snapshot_sha256":"ba0abc3b84718f0f3b73ece26ae93f12513ef37027516dd75d19cce837cdcf65","tool_count":9,"tool_names":["search_wikipedia","get_article","get_summary","summarize_article_for_query","summarize_article_section","extract_key_facts","get_related_topics","get_sections","get_links"],"source_identity":"mediawiki-api-official-2026-09-10","source_url":"https://www.mediawiki.org/wiki/API:Search","source_version":"MediaWiki API documentation accessed 2026-09-10","subject_order":8,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss."},"google-maps-mcp":{"canonical_subject_id":"google-maps-mcp","provider_repository":"stable corpus / Google Maps MCP reconstruction","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned stable corpus; no remote contact in Batch 5.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/phase-3/google-maps-tools-list.json","schema_snapshot_sha256":"fca473a69e8a5732bd84ed01049b9ba1bd2e7e706c073e643040145cca92a3a1","tool_count":7,"tool_names":["search_nearby","get_place_details","maps_geocode","maps_reverse_geocode","maps_distance_matrix","maps_directions","maps_elevation"],"source_identity":"google-maps-platform-official-2026-09-10","source_url":"https://developers.google.com/maps/documentation","source_version":"Google Maps Platform documentation accessed 2026-09-10","subject_order":9,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss."},"metropolitan-museum-mcp":{"canonical_subject_id":"metropolitan-museum-mcp","provider_repository":"stable corpus / Metropolitan Museum MCP reconstruction","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned stable corpus; no remote contact in Batch 5.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/phase-3/metropolitan-museum-tools-list.json","schema_snapshot_sha256":"0ef880de13ed23c83110318eff4dd339756a3d8f4c8384c820476c54a5cfe2b2","tool_count":3,"tool_names":["list-departments","search-museum-objects","get-museum-object"],"source_identity":"met-collection-api-official-2026-09-10","source_url":"https://metmuseum.github.io/","source_version":"Met Collection API documentation accessed 2026-09-10","subject_order":10,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss."}},"source_files":[{"path":"validation/external-gold-expansion-v2/batch6/measurement-v2-batch6/measurement-artifact-v2-batch6.json","sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912"},{"path":"validation/external-gold-expansion-v2/batch6/case-freeze-v2-batch6.json","sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b"},{"path":"validation/external-gold-expansion-v2/batch6/final-subject-set-v2-batch6.json","sha256":"26355ec41096de990187e1a55d05c9478e4701754372bebfd5b07bb08acdcba6"},{"path":"validation/external-gold-expansion-v2/batch6/measurement-v2-batch6/semantic-miss-adjudication-v2-batch6.json","sha256":"354a0e94289c15bf8ad0cfaf2ba0e019fddc050979be47693515cb4f43fa77a9"},{"path":"validation/external-gold-expansion-v2/batch6/measurement-v2-batch6/raw-predictions-v2-batch6.jsonl","sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51"}]}],"subject_records":[{"canonical_subject_id":"cloudflare-docs-mcp","subject_name":"Cloudflare Docs MCP","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"MIXED","identity_variants":[{"canonical_subject_id":"cloudflare-docs-mcp","subject_name":"Cloudflare Docs MCP","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"LIVE_TOOLS_LIST"},{"canonical_subject_id":"cloudflare-docs-mcp","subject_name":"Cloudflare Docs","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"}],"number_of_independent_benchmark_batches":5,"total_measured_frozen_cases":51,"total_provider_valid_cases":51,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}},{"batch_id":"external-gold-expansion-v2-batch3","benchmark_id":"external-gold-expansion-v2-batch3","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T17:45:38.245198Z","evidence_identity":{"freeze_input_commit":"8450ed889756784a12ca4ebb8d84974d73b31778","final_freeze_receipt_commit":"a309bda7e8dd2867ccc7894dc7d42efbfc48395e","measurement_commit":"b81cc816158f7c0fbe7f8ac0a6fcf2b3f38f4c86","immutable_measurement_receipt_commit":"9d8290160849a5124d1896fc318d53470585ef41","case_freeze_sha256":"c1595743f48fe8ae6b3f70f2ea401731052b3bd7614297feabaa8172ce190ae5","model_lock_sha256":"48b5ed9c98f16cc2b3b76bc0da2e7ec8072ecd89ba1914683181b1c20074a3a8","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"c5cfe765f6b72489220cc73f689acb1c1c2c9284ffd8d093de0e597b645f6fbf","measurement_artifact_sha256":"7dcab9d4b71e7de59094c07cbca830ebf674b601ac6de024db0d3e677e7c0067","policy_evaluation_sha256":"3ea5b62044b4da6e92812831ebb00205970849accffe763a270248a24a0adef4"}},{"batch_id":"external-gold-expansion-v2-batch4","benchmark_id":"external-gold-expansion-v2-batch4","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T18:47:01.455700Z","evidence_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"}},{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S1","S2"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":51,"number_of_batches":5,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"context7","subject_name":"Context7","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_variants":[{"canonical_subject_id":"context7","subject_name":"Context7","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"LIVE_TOOLS_LIST"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":24,"total_provider_valid_cases":24,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S1"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MEASURED","summary":{"total_measured_cases":24,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"datagouv-mcp","subject_name":"data.gouv.fr","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"MIXED","identity_variants":[{"canonical_subject_id":"datagouv-mcp","subject_name":"data.gouv.fr","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_SNAPSHOT"},{"canonical_subject_id":"datagouv-mcp","subject_name":"data.gouv.fr","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT"},{"canonical_subject_id":"datagouv-mcp","subject_name":"data.gouv.fr","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"}],"number_of_independent_benchmark_batches":6,"total_measured_frozen_cases":39,"total_provider_valid_cases":39,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-expansion-v2-batch1","benchmark_id":"external-gold-expansion-v2-batch1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.6096657120978346,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T15:38:03.968297Z","evidence_identity":{"freeze_input_commit":"c614ae51750d12ce113b5e3d51429f52e529f08c","final_freeze_receipt_commit":"b691170","measurement_commit":"b9f358b1a65aba348fa1c0e625eac6da1f31b651","policy_evaluation_commit":"3e560e5767f181831c65569b893ca789cd3bdff0","case_freeze_sha256":"17777d2c3983cd444353b8d69dcc98836a1909ee6fde6f5c8cf18cf9b059427e","model_lock_sha256":"c4330165deb3d8587e224a2ca1b3d886db60801ca436710a0edace55525923d2","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ab085cedf13eef5724f4e160b0f49f8402d62bc5b10203de0f8d81e41f2a017e","measurement_artifact_sha256":"01243422aa946296f235810aaee6f065ec21d9e78a01017ee8161e5e61a8b5e8","policy_evaluation_sha256":"4e710a9c5205660cc07603d2fd7538aa063eda4c2574c29308f9f255e4706f82"}},{"batch_id":"external-gold-expansion-v2-batch2","benchmark_id":"external-gold-expansion-v2-batch2","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":0.8333333333333334,"wilson_95":{"lower":0.43649717781352987,"upper":0.9699466302516935},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"5/6","measured_at":"2026-09-10T16:49:53.346446Z","evidence_identity":{"freeze_input_commit":"61f6511c41562c24019cd441b37bc6ee07283951","final_freeze_receipt_commit":"d3c55d2f96a4e45bf2577225ea16929877f2ded1","measurement_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","policy_evaluation_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","case_freeze_sha256":"ce21fc085bd9a0185c9e37d0d795221f286397cbc477ee941e876be24f7ba1b4","model_lock_sha256":"153647f94eed0ee3d3069229545846eef864fd43fa821f2d311dc3fb5d265ba9","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"8c4364ab598241ddbe4adb16453ba2e12b0917a1d1852caedb07884d1a54570b","measurement_artifact_sha256":"dd13aec8a90b2b5490c934c9ad9650c37733c65f18d824e2ae6066202176ced9","policy_evaluation_sha256":"736f154a5d5c3b8f0d77cd58fa0644ca97ad94c7a6b46af98d394c1dc1578f9a"}},{"batch_id":"external-gold-expansion-v2-batch3","benchmark_id":"external-gold-expansion-v2-batch3","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T17:45:38.245198Z","evidence_identity":{"freeze_input_commit":"8450ed889756784a12ca4ebb8d84974d73b31778","final_freeze_receipt_commit":"a309bda7e8dd2867ccc7894dc7d42efbfc48395e","measurement_commit":"b81cc816158f7c0fbe7f8ac0a6fcf2b3f38f4c86","immutable_measurement_receipt_commit":"9d8290160849a5124d1896fc318d53470585ef41","case_freeze_sha256":"c1595743f48fe8ae6b3f70f2ea401731052b3bd7614297feabaa8172ce190ae5","model_lock_sha256":"48b5ed9c98f16cc2b3b76bc0da2e7ec8072ecd89ba1914683181b1c20074a3a8","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"c5cfe765f6b72489220cc73f689acb1c1c2c9284ffd8d093de0e597b645f6fbf","measurement_artifact_sha256":"7dcab9d4b71e7de59094c07cbca830ebf674b601ac6de024db0d3e677e7c0067","policy_evaluation_sha256":"3ea5b62044b4da6e92812831ebb00205970849accffe763a270248a24a0adef4"}},{"batch_id":"external-gold-expansion-v2-batch4","benchmark_id":"external-gold-expansion-v2-batch4","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T18:47:01.455700Z","evidence_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"}},{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":1,"observed_semantic_miss_case_ids":["datagouv-hard-004"],"observed_semantic_misses":[{"case_id":"datagouv-hard-004","subject_id":"datagouv-mcp","expected_tool":"list_dataset_resources","selected_tool":"get_dataset_info","boundary_family":"DATASET_VS_RESOURCE_LIST","adjudication":"SEMANTIC_MISS_GOLD_REMAINS_VALID","source_adjudication_artifact":"validation/external-gold-expansion-v2/measurement-v2-batch2/semantic-miss-adjudication-v2-batch2.json","benchmark_id":"external-gold-expansion-v2-batch2","batch_id":"external-gold-expansion-v2-batch2"}],"observed_miss_boundary_families":["DATASET_VS_RESOURCE_LIST"],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T15:38:03.968297Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":39,"number_of_batches":6,"strict_accuracy_range_across_batches":{"minimum":0.8333333333333334,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":0.8333333333333334}},{"canonical_subject_id":"deepwiki-mcp","subject_name":"DeepWiki MCP","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"MIXED","identity_variants":[{"canonical_subject_id":"deepwiki-mcp","subject_name":"DeepWiki MCP","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"LIVE_TOOLS_LIST"},{"canonical_subject_id":"deepwiki-mcp","subject_name":"DeepWiki","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_INHERITED_V1_AND_FROZEN_BY_COPY"},{"canonical_subject_id":"deepwiki-mcp","subject_name":"DeepWiki","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT"},{"canonical_subject_id":"deepwiki-mcp","subject_name":"DeepWiki","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"}],"number_of_independent_benchmark_batches":6,"total_measured_frozen_cases":52,"total_provider_valid_cases":52,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}},{"batch_id":"external-gold-expansion-v2-batch1","benchmark_id":"external-gold-expansion-v2-batch1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":4,"provider_valid_cases":4,"infrastructure_failures":0,"strict_correct":3,"strict_accuracy":0.75,"wilson_95":{"lower":0.30064184258240184,"upper":0.9544127391902995},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"3/4","measured_at":"2026-09-10T15:38:03.968297Z","evidence_identity":{"freeze_input_commit":"c614ae51750d12ce113b5e3d51429f52e529f08c","final_freeze_receipt_commit":"b691170","measurement_commit":"b9f358b1a65aba348fa1c0e625eac6da1f31b651","policy_evaluation_commit":"3e560e5767f181831c65569b893ca789cd3bdff0","case_freeze_sha256":"17777d2c3983cd444353b8d69dcc98836a1909ee6fde6f5c8cf18cf9b059427e","model_lock_sha256":"c4330165deb3d8587e224a2ca1b3d886db60801ca436710a0edace55525923d2","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ab085cedf13eef5724f4e160b0f49f8402d62bc5b10203de0f8d81e41f2a017e","measurement_artifact_sha256":"01243422aa946296f235810aaee6f065ec21d9e78a01017ee8161e5e61a8b5e8","policy_evaluation_sha256":"4e710a9c5205660cc07603d2fd7538aa063eda4c2574c29308f9f255e4706f82"}},{"batch_id":"external-gold-expansion-v2-batch2","benchmark_id":"external-gold-expansion-v2-batch2","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5","measured_at":"2026-09-10T16:49:53.346446Z","evidence_identity":{"freeze_input_commit":"61f6511c41562c24019cd441b37bc6ee07283951","final_freeze_receipt_commit":"d3c55d2f96a4e45bf2577225ea16929877f2ded1","measurement_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","policy_evaluation_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","case_freeze_sha256":"ce21fc085bd9a0185c9e37d0d795221f286397cbc477ee941e876be24f7ba1b4","model_lock_sha256":"153647f94eed0ee3d3069229545846eef864fd43fa821f2d311dc3fb5d265ba9","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"8c4364ab598241ddbe4adb16453ba2e12b0917a1d1852caedb07884d1a54570b","measurement_artifact_sha256":"dd13aec8a90b2b5490c934c9ad9650c37733c65f18d824e2ae6066202176ced9","policy_evaluation_sha256":"736f154a5d5c3b8f0d77cd58fa0644ca97ad94c7a6b46af98d394c1dc1578f9a"}},{"batch_id":"external-gold-expansion-v2-batch3","benchmark_id":"external-gold-expansion-v2-batch3","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T17:45:38.245198Z","evidence_identity":{"freeze_input_commit":"8450ed889756784a12ca4ebb8d84974d73b31778","final_freeze_receipt_commit":"a309bda7e8dd2867ccc7894dc7d42efbfc48395e","measurement_commit":"b81cc816158f7c0fbe7f8ac0a6fcf2b3f38f4c86","immutable_measurement_receipt_commit":"9d8290160849a5124d1896fc318d53470585ef41","case_freeze_sha256":"c1595743f48fe8ae6b3f70f2ea401731052b3bd7614297feabaa8172ce190ae5","model_lock_sha256":"48b5ed9c98f16cc2b3b76bc0da2e7ec8072ecd89ba1914683181b1c20074a3a8","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"c5cfe765f6b72489220cc73f689acb1c1c2c9284ffd8d093de0e597b645f6fbf","measurement_artifact_sha256":"7dcab9d4b71e7de59094c07cbca830ebf674b601ac6de024db0d3e677e7c0067","policy_evaluation_sha256":"3ea5b62044b4da6e92812831ebb00205970849accffe763a270248a24a0adef4"}},{"batch_id":"external-gold-expansion-v2-batch4","benchmark_id":"external-gold-expansion-v2-batch4","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":0.833333,"wilson_95":{"lower":0.436497,"upper":0.969947,"lower_percent":43.6497,"upper_percent":96.9947},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"5/6","measured_at":"2026-09-10T18:47:01.455700Z","evidence_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"}},{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":4,"strict_accuracy":0.571429,"wilson_95":{"lower":0.250458,"upper":0.84178,"lower_percent":25.0458,"upper_percent":84.178},"semantic_coverage":1.0,"observed_semantic_miss_count":3,"strict_score":"4/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}}],"observed_semantic_miss_count":5,"observed_semantic_miss_case_ids":["deepwiki-004","b4-dw-005","b5-dw-003","b5-dw-004","b5-dw-006"],"observed_semantic_misses":[{"case_id":"deepwiki-004","subject_id":"deepwiki-mcp","expected_tool":"ask_question","selected_tool":"read_wiki_structure","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"SEMANTIC_MISS","source_adjudication_artifact":"validation/external-gold-expansion-v2/measurement-v2-batch1/semantic-miss-adjudication-v2-batch1.json","benchmark_id":"external-gold-expansion-v2-batch1","batch_id":"external-gold-expansion-v2-batch1"},{"case_id":"b4-dw-005","subject_id":"deepwiki-mcp","expected_tool":"read_wiki_contents","selected_tool":"read_wiki_structure","boundary_family":"CONTENTS_VS_QUESTION","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch4/measurement-v2-batch4/semantic-miss-adjudication-v2-batch4.json","benchmark_id":"external-gold-expansion-v2-batch4","batch_id":"external-gold-expansion-v2-batch4"},{"case_id":"b5-dw-003","subject_id":"deepwiki-mcp","expected_tool":"read_wiki_contents","selected_tool":"read_wiki_structure","boundary_family":"CONTENTS_VS_QUESTION","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch5/measurement-v2-batch5/semantic-miss-adjudication-v2-batch5.json","benchmark_id":"external-gold-expansion-v2-batch5","batch_id":"external-gold-expansion-v2-batch5"},{"case_id":"b5-dw-004","subject_id":"deepwiki-mcp","expected_tool":"ask_question","selected_tool":"read_wiki_contents","boundary_family":"QUESTION_VS_CONTENTS","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch5/measurement-v2-batch5/semantic-miss-adjudication-v2-batch5.json","benchmark_id":"external-gold-expansion-v2-batch5","batch_id":"external-gold-expansion-v2-batch5"},{"case_id":"b5-dw-006","subject_id":"deepwiki-mcp","expected_tool":"read_wiki_contents","selected_tool":"read_wiki_structure","boundary_family":"CONTENTS_VS_STRUCTURE","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch5/measurement-v2-batch5/semantic-miss-adjudication-v2-batch5.json","benchmark_id":"external-gold-expansion-v2-batch5","batch_id":"external-gold-expansion-v2-batch5"}],"observed_miss_boundary_families":["CONTENTS_VS_QUESTION","CONTENTS_VS_STRUCTURE","NOT_RECORDED_IN_SOURCE_EVIDENCE","QUESTION_VS_CONTENTS"],"boundary_family_data_complete":false,"evidence_levels_represented":["S1","S2"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-10T19:13:49.842728Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-10T19:13:49.842728Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":52,"number_of_batches":6,"strict_accuracy_range_across_batches":{"minimum":0.571429,"maximum":1.0},"latest_strict_accuracy":0.571429,"lowest_observed_strict_accuracy":0.571429}},{"canonical_subject_id":"dex-paprika-mcp-source-snapshot","subject_name":"DEX Paprika MCP (source snapshot)","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"dex-paprika-mcp-source-snapshot","subject_name":"DEX Paprika MCP (source snapshot)","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":24,"total_provider_valid_cases":24,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S1"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MEASURED","summary":{"total_measured_cases":24,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"exa-mcp","subject_name":"Exa MCP","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"MIXED","identity_variants":[{"canonical_subject_id":"exa-mcp","subject_name":"Exa MCP","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"LIVE_TOOLS_LIST"},{"canonical_subject_id":"exa-mcp","subject_name":"Exa","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_INHERITED_V1_AND_FROZEN_BY_COPY"},{"canonical_subject_id":"exa-mcp","subject_name":"Exa","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"}],"number_of_independent_benchmark_batches":6,"total_measured_frozen_cases":55,"total_provider_valid_cases":55,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}},{"batch_id":"external-gold-expansion-v2-batch1","benchmark_id":"external-gold-expansion-v2-batch1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":4,"provider_valid_cases":4,"infrastructure_failures":0,"strict_correct":4,"strict_accuracy":1.0,"wilson_95":{"lower":0.5101091635454027,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"4/4","measured_at":"2026-09-10T15:38:03.968297Z","evidence_identity":{"freeze_input_commit":"c614ae51750d12ce113b5e3d51429f52e529f08c","final_freeze_receipt_commit":"b691170","measurement_commit":"b9f358b1a65aba348fa1c0e625eac6da1f31b651","policy_evaluation_commit":"3e560e5767f181831c65569b893ca789cd3bdff0","case_freeze_sha256":"17777d2c3983cd444353b8d69dcc98836a1909ee6fde6f5c8cf18cf9b059427e","model_lock_sha256":"c4330165deb3d8587e224a2ca1b3d886db60801ca436710a0edace55525923d2","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ab085cedf13eef5724f4e160b0f49f8402d62bc5b10203de0f8d81e41f2a017e","measurement_artifact_sha256":"01243422aa946296f235810aaee6f065ec21d9e78a01017ee8161e5e61a8b5e8","policy_evaluation_sha256":"4e710a9c5205660cc07603d2fd7538aa063eda4c2574c29308f9f255e4706f82"}},{"batch_id":"external-gold-expansion-v2-batch3","benchmark_id":"external-gold-expansion-v2-batch3","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T17:45:38.245198Z","evidence_identity":{"freeze_input_commit":"8450ed889756784a12ca4ebb8d84974d73b31778","final_freeze_receipt_commit":"a309bda7e8dd2867ccc7894dc7d42efbfc48395e","measurement_commit":"b81cc816158f7c0fbe7f8ac0a6fcf2b3f38f4c86","immutable_measurement_receipt_commit":"9d8290160849a5124d1896fc318d53470585ef41","case_freeze_sha256":"c1595743f48fe8ae6b3f70f2ea401731052b3bd7614297feabaa8172ce190ae5","model_lock_sha256":"48b5ed9c98f16cc2b3b76bc0da2e7ec8072ecd89ba1914683181b1c20074a3a8","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"c5cfe765f6b72489220cc73f689acb1c1c2c9284ffd8d093de0e597b645f6fbf","measurement_artifact_sha256":"7dcab9d4b71e7de59094c07cbca830ebf674b601ac6de024db0d3e677e7c0067","policy_evaluation_sha256":"3ea5b62044b4da6e92812831ebb00205970849accffe763a270248a24a0adef4"}},{"batch_id":"external-gold-expansion-v2-batch4","benchmark_id":"external-gold-expansion-v2-batch4","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T18:47:01.455700Z","evidence_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"}},{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S1","S2"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":55,"number_of_batches":6,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"external-gold-v1:Context7","subject_name":"Context7","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:Context7","subject_name":"Context7","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"external-gold-v1:DEX Paprika","subject_name":"DEX Paprika","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:DEX Paprika","subject_name":"DEX Paprika","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":1,"strict_accuracy":0.5,"wilson_95":{"lower":0.094531,"upper":0.905469},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"1/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":1,"observed_semantic_miss_case_ids":["dex_paprika_001"],"observed_semantic_misses":[{"case_id":"dex_paprika_001","subject_id":"external-gold-v1:DEX Paprika","expected_tool":"getStats","selected_tool":"getNetworks","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"GOLD_SCHEMA_CONTRADICTION","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json","benchmark_id":"external-gold-v1","batch_id":"external-gold-v1"}],"observed_miss_boundary_families":["NOT_RECORDED_IN_SOURCE_EVIDENCE"],"boundary_family_data_complete":false,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":0.5,"maximum":0.5},"latest_strict_accuracy":0.5,"lowest_observed_strict_accuracy":0.5}},{"canonical_subject_id":"external-gold-v1:Google Maps","subject_name":"Google Maps","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:Google Maps","subject_name":"Google Maps","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":0,"strict_accuracy":0.0,"wilson_95":{"lower":0.0,"upper":0.65762},"semantic_coverage":1.0,"observed_semantic_miss_count":2,"strict_score":"0/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":2,"observed_semantic_miss_case_ids":["google_maps_000","google_maps_001"],"observed_semantic_misses":[{"case_id":"google_maps_000","subject_id":"external-gold-v1:Google Maps","expected_tool":"maps_geocode","selected_tool":"search_nearby","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"NON_UNIQUE_FIRST_TOOL","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json","benchmark_id":"external-gold-v1","batch_id":"external-gold-v1"},{"case_id":"google_maps_001","subject_id":"external-gold-v1:Google Maps","expected_tool":"maps_geocode","selected_tool":"search_nearby","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"NON_UNIQUE_FIRST_TOOL","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json","benchmark_id":"external-gold-v1","batch_id":"external-gold-v1"}],"observed_miss_boundary_families":["NOT_RECORDED_IN_SOURCE_EVIDENCE"],"boundary_family_data_complete":false,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":0.0,"maximum":0.0},"latest_strict_accuracy":0.0,"lowest_observed_strict_accuracy":0.0}},{"canonical_subject_id":"external-gold-v1:Huge Icons","subject_name":"Huge Icons","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:Huge Icons","subject_name":"Huge Icons","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":1,"strict_accuracy":0.5,"wilson_95":{"lower":0.094531,"upper":0.905469},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"1/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":1,"observed_semantic_miss_case_ids":["huge_icons_000"],"observed_semantic_misses":[{"case_id":"huge_icons_000","subject_id":"external-gold-v1:Huge Icons","expected_tool":"list_icons","selected_tool":"search_icons","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"NON_UNIQUE_FIRST_TOOL","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json","benchmark_id":"external-gold-v1","batch_id":"external-gold-v1"}],"observed_miss_boundary_families":["NOT_RECORDED_IN_SOURCE_EVIDENCE"],"boundary_family_data_complete":false,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":0.5,"maximum":0.5},"latest_strict_accuracy":0.5,"lowest_observed_strict_accuracy":0.5}},{"canonical_subject_id":"external-gold-v1:Medical Calculator","subject_name":"Medical Calculator","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:Medical Calculator","subject_name":"Medical Calculator","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"external-gold-v1:Metropolitan Museum","subject_name":"Metropolitan Museum","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:Metropolitan Museum","subject_name":"Metropolitan Museum","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"external-gold-v1:National Parks","subject_name":"National Parks","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:National Parks","subject_name":"National Parks","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"external-gold-v1:NixOS","subject_name":"NixOS","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:NixOS","subject_name":"NixOS","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":1,"strict_accuracy":0.5,"wilson_95":{"lower":0.094531,"upper":0.905469},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"1/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":1,"observed_semantic_miss_case_ids":["nixos_001"],"observed_semantic_misses":[{"case_id":"nixos_001","subject_id":"external-gold-v1:NixOS","expected_tool":"nixos_search","selected_tool":"nixos_info","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"GENUINE_SEQUENCE_ERROR","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json","benchmark_id":"external-gold-v1","batch_id":"external-gold-v1"}],"observed_miss_boundary_families":["NOT_RECORDED_IN_SOURCE_EVIDENCE"],"boundary_family_data_complete":false,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":0.5,"maximum":0.5},"latest_strict_accuracy":0.5,"lowest_observed_strict_accuracy":0.5}},{"canonical_subject_id":"external-gold-v1:OKX Exchange","subject_name":"OKX Exchange","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:OKX Exchange","subject_name":"OKX Exchange","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":1,"strict_accuracy":0.5,"wilson_95":{"lower":0.094531,"upper":0.905469},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"1/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":1,"observed_semantic_miss_case_ids":["okx_exchange_000"],"observed_semantic_misses":[{"case_id":"okx_exchange_000","subject_id":"external-gold-v1:OKX Exchange","expected_tool":"get_price","selected_tool":"get_candlesticks","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"PROCEDURE_ORDER_DISAGREEMENT","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json","benchmark_id":"external-gold-v1","batch_id":"external-gold-v1"}],"observed_miss_boundary_families":["NOT_RECORDED_IN_SOURCE_EVIDENCE"],"boundary_family_data_complete":false,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":0.5,"maximum":0.5},"latest_strict_accuracy":0.5,"lowest_observed_strict_accuracy":0.5}},{"canonical_subject_id":"external-gold-v1:Reddit","subject_name":"Reddit","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:Reddit","subject_name":"Reddit","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"external-gold-v1:Scientific Computing","subject_name":"Scientific Computing","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:Scientific Computing","subject_name":"Scientific Computing","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"external-gold-v1:Wikipedia","subject_name":"Wikipedia","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:Wikipedia","subject_name":"Wikipedia","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"gitmcp-docs","subject_name":"GitMCP Docs","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"MIXED","identity_variants":[{"canonical_subject_id":"gitmcp-docs","subject_name":"GitMCP Docs","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"LIVE_TOOLS_LIST"},{"canonical_subject_id":"gitmcp-docs","subject_name":"GitMCP","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT"},{"canonical_subject_id":"gitmcp-docs","subject_name":"GitMCP","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"}],"number_of_independent_benchmark_batches":3,"total_measured_frozen_cases":35,"total_provider_valid_cases":35,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}},{"batch_id":"external-gold-expansion-v2-batch2","benchmark_id":"external-gold-expansion-v2-batch2","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":4,"strict_accuracy":0.8,"wilson_95":{"lower":0.37553462976252533,"upper":0.9637758913675698},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"4/5","measured_at":"2026-09-10T16:49:53.346446Z","evidence_identity":{"freeze_input_commit":"61f6511c41562c24019cd441b37bc6ee07283951","final_freeze_receipt_commit":"d3c55d2f96a4e45bf2577225ea16929877f2ded1","measurement_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","policy_evaluation_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","case_freeze_sha256":"ce21fc085bd9a0185c9e37d0d795221f286397cbc477ee941e876be24f7ba1b4","model_lock_sha256":"153647f94eed0ee3d3069229545846eef864fd43fa821f2d311dc3fb5d265ba9","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"8c4364ab598241ddbe4adb16453ba2e12b0917a1d1852caedb07884d1a54570b","measurement_artifact_sha256":"dd13aec8a90b2b5490c934c9ad9650c37733c65f18d824e2ae6066202176ced9","policy_evaluation_sha256":"736f154a5d5c3b8f0d77cd58fa0644ca97ad94c7a6b46af98d394c1dc1578f9a"}},{"batch_id":"external-gold-expansion-v2-batch3","benchmark_id":"external-gold-expansion-v2-batch3","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":4,"strict_accuracy":0.666667,"wilson_95":{"lower":0.299993,"upper":0.903229},"semantic_coverage":1.0,"observed_semantic_miss_count":2,"strict_score":"4/6","measured_at":"2026-09-10T17:45:38.245198Z","evidence_identity":{"freeze_input_commit":"8450ed889756784a12ca4ebb8d84974d73b31778","final_freeze_receipt_commit":"a309bda7e8dd2867ccc7894dc7d42efbfc48395e","measurement_commit":"b81cc816158f7c0fbe7f8ac0a6fcf2b3f38f4c86","immutable_measurement_receipt_commit":"9d8290160849a5124d1896fc318d53470585ef41","case_freeze_sha256":"c1595743f48fe8ae6b3f70f2ea401731052b3bd7614297feabaa8172ce190ae5","model_lock_sha256":"48b5ed9c98f16cc2b3b76bc0da2e7ec8072ecd89ba1914683181b1c20074a3a8","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"c5cfe765f6b72489220cc73f689acb1c1c2c9284ffd8d093de0e597b645f6fbf","measurement_artifact_sha256":"7dcab9d4b71e7de59094c07cbca830ebf674b601ac6de024db0d3e677e7c0067","policy_evaluation_sha256":"3ea5b62044b4da6e92812831ebb00205970849accffe763a270248a24a0adef4"}}],"observed_semantic_miss_count":3,"observed_semantic_miss_case_ids":["gitmcp-hard-005","b3-gm-005","b3-gm-006"],"observed_semantic_misses":[{"case_id":"gitmcp-hard-005","subject_id":"gitmcp-docs","expected_tool":"search_generic_documentation","selected_tool":"fetch_generic_documentation","boundary_family":"DOCUMENTED_BEHAVIOR_VS_IMPLEMENTATION","adjudication":"SEMANTIC_MISS_GOLD_REMAINS_VALID","source_adjudication_artifact":"validation/external-gold-expansion-v2/measurement-v2-batch2/semantic-miss-adjudication-v2-batch2.json","benchmark_id":"external-gold-expansion-v2-batch2","batch_id":"external-gold-expansion-v2-batch2"},{"case_id":"b3-gm-005","subject_id":"gitmcp-docs","expected_tool":"fetch_generic_url_content","selected_tool":"fetch_generic_documentation","boundary_family":"DOCUMENTATION_VS_REFERENCED_URL","adjudication":"GOLD_REMAINS_VALID; semantic miss preserved in strict score. The public source supports the requested neighboring operation, and the selected tool is a plausible but incorrect neighboring retrieval mode.","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch3/measurement-v2-batch3/semantic-miss-adjudication-v2-batch3.json","benchmark_id":"external-gold-expansion-v2-batch3","batch_id":"external-gold-expansion-v2-batch3"},{"case_id":"b3-gm-006","subject_id":"gitmcp-docs","expected_tool":"search_generic_documentation","selected_tool":"fetch_generic_documentation","boundary_family":"DOCUMENTED_BEHAVIOR_VS_IMPLEMENTATION","adjudication":"GOLD_REMAINS_VALID; semantic miss preserved in strict score. The public source supports the requested neighboring operation, and the selected tool is a plausible but incorrect neighboring retrieval mode.","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch3/measurement-v2-batch3/semantic-miss-adjudication-v2-batch3.json","benchmark_id":"external-gold-expansion-v2-batch3","batch_id":"external-gold-expansion-v2-batch3"}],"observed_miss_boundary_families":["DOCUMENTATION_VS_REFERENCED_URL","DOCUMENTED_BEHAVIOR_VS_IMPLEMENTATION"],"boundary_family_data_complete":true,"evidence_levels_represented":["S1","S2"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-10T17:45:38.245198Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-10T17:45:38.245198Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":35,"number_of_batches":3,"strict_accuracy_range_across_batches":{"minimum":0.666667,"maximum":1.0},"latest_strict_accuracy":0.666667,"lowest_observed_strict_accuracy":0.666667}},{"canonical_subject_id":"google-maps-mcp","subject_name":"Google Maps","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"google-maps-mcp","subject_name":"Google Maps","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":2,"total_measured_frozen_cases":15,"total_provider_valid_cases":15,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T19:13:49.842728Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":15,"number_of_batches":2,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"metropolitan-museum-mcp","subject_name":"Metropolitan Museum","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"metropolitan-museum-mcp","subject_name":"Metropolitan Museum","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":2,"total_measured_frozen_cases":15,"total_provider_valid_cases":15,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T19:13:49.842728Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":15,"number_of_batches":2,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"metropolitan-museum-mcp-source-snapshot","subject_name":"Metropolitan Museum MCP (source snapshot)","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"metropolitan-museum-mcp-source-snapshot","subject_name":"Metropolitan Museum MCP (source snapshot)","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":24,"total_provider_valid_cases":24,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S1"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MEASURED","summary":{"total_measured_cases":24,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"microsoft-learn","subject_name":"Microsoft Learn","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"MIXED","identity_variants":[{"canonical_subject_id":"microsoft-learn","subject_name":"Microsoft Learn","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_SNAPSHOT"},{"canonical_subject_id":"microsoft-learn","subject_name":"Microsoft Learn","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT"},{"canonical_subject_id":"microsoft-learn","subject_name":"Microsoft Learn","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"}],"number_of_independent_benchmark_batches":5,"total_measured_frozen_cases":31,"total_provider_valid_cases":31,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-expansion-v2-batch1","benchmark_id":"external-gold-expansion-v2-batch1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5","measured_at":"2026-09-10T15:38:03.968297Z","evidence_identity":{"freeze_input_commit":"c614ae51750d12ce113b5e3d51429f52e529f08c","final_freeze_receipt_commit":"b691170","measurement_commit":"b9f358b1a65aba348fa1c0e625eac6da1f31b651","policy_evaluation_commit":"3e560e5767f181831c65569b893ca789cd3bdff0","case_freeze_sha256":"17777d2c3983cd444353b8d69dcc98836a1909ee6fde6f5c8cf18cf9b059427e","model_lock_sha256":"c4330165deb3d8587e224a2ca1b3d886db60801ca436710a0edace55525923d2","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ab085cedf13eef5724f4e160b0f49f8402d62bc5b10203de0f8d81e41f2a017e","measurement_artifact_sha256":"01243422aa946296f235810aaee6f065ec21d9e78a01017ee8161e5e61a8b5e8","policy_evaluation_sha256":"4e710a9c5205660cc07603d2fd7538aa063eda4c2574c29308f9f255e4706f82"}},{"batch_id":"external-gold-expansion-v2-batch2","benchmark_id":"external-gold-expansion-v2-batch2","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5","measured_at":"2026-09-10T16:49:53.346446Z","evidence_identity":{"freeze_input_commit":"61f6511c41562c24019cd441b37bc6ee07283951","final_freeze_receipt_commit":"d3c55d2f96a4e45bf2577225ea16929877f2ded1","measurement_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","policy_evaluation_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","case_freeze_sha256":"ce21fc085bd9a0185c9e37d0d795221f286397cbc477ee941e876be24f7ba1b4","model_lock_sha256":"153647f94eed0ee3d3069229545846eef864fd43fa821f2d311dc3fb5d265ba9","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"8c4364ab598241ddbe4adb16453ba2e12b0917a1d1852caedb07884d1a54570b","measurement_artifact_sha256":"dd13aec8a90b2b5490c934c9ad9650c37733c65f18d824e2ae6066202176ced9","policy_evaluation_sha256":"736f154a5d5c3b8f0d77cd58fa0644ca97ad94c7a6b46af98d394c1dc1578f9a"}},{"batch_id":"external-gold-expansion-v2-batch4","benchmark_id":"external-gold-expansion-v2-batch4","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T18:47:01.455700Z","evidence_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"}},{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T15:38:03.968297Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":31,"number_of_batches":5,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"microsoft-release-communications","subject_name":"Microsoft Release Communications","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"MIXED","identity_variants":[{"canonical_subject_id":"microsoft-release-communications","subject_name":"Microsoft Release Communications","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_SNAPSHOT"},{"canonical_subject_id":"microsoft-release-communications","subject_name":"Microsoft Release Communications","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT"},{"canonical_subject_id":"microsoft-release-communications","subject_name":"Microsoft Release Communications","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"}],"number_of_independent_benchmark_batches":6,"total_measured_frozen_cases":37,"total_provider_valid_cases":37,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-expansion-v2-batch1","benchmark_id":"external-gold-expansion-v2-batch1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5","measured_at":"2026-09-10T15:38:03.968297Z","evidence_identity":{"freeze_input_commit":"c614ae51750d12ce113b5e3d51429f52e529f08c","final_freeze_receipt_commit":"b691170","measurement_commit":"b9f358b1a65aba348fa1c0e625eac6da1f31b651","policy_evaluation_commit":"3e560e5767f181831c65569b893ca789cd3bdff0","case_freeze_sha256":"17777d2c3983cd444353b8d69dcc98836a1909ee6fde6f5c8cf18cf9b059427e","model_lock_sha256":"c4330165deb3d8587e224a2ca1b3d886db60801ca436710a0edace55525923d2","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ab085cedf13eef5724f4e160b0f49f8402d62bc5b10203de0f8d81e41f2a017e","measurement_artifact_sha256":"01243422aa946296f235810aaee6f065ec21d9e78a01017ee8161e5e61a8b5e8","policy_evaluation_sha256":"4e710a9c5205660cc07603d2fd7538aa063eda4c2574c29308f9f255e4706f82"}},{"batch_id":"external-gold-expansion-v2-batch2","benchmark_id":"external-gold-expansion-v2-batch2","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5","measured_at":"2026-09-10T16:49:53.346446Z","evidence_identity":{"freeze_input_commit":"61f6511c41562c24019cd441b37bc6ee07283951","final_freeze_receipt_commit":"d3c55d2f96a4e45bf2577225ea16929877f2ded1","measurement_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","policy_evaluation_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","case_freeze_sha256":"ce21fc085bd9a0185c9e37d0d795221f286397cbc477ee941e876be24f7ba1b4","model_lock_sha256":"153647f94eed0ee3d3069229545846eef864fd43fa821f2d311dc3fb5d265ba9","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"8c4364ab598241ddbe4adb16453ba2e12b0917a1d1852caedb07884d1a54570b","measurement_artifact_sha256":"dd13aec8a90b2b5490c934c9ad9650c37733c65f18d824e2ae6066202176ced9","policy_evaluation_sha256":"736f154a5d5c3b8f0d77cd58fa0644ca97ad94c7a6b46af98d394c1dc1578f9a"}},{"batch_id":"external-gold-expansion-v2-batch3","benchmark_id":"external-gold-expansion-v2-batch3","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T17:45:38.245198Z","evidence_identity":{"freeze_input_commit":"8450ed889756784a12ca4ebb8d84974d73b31778","final_freeze_receipt_commit":"a309bda7e8dd2867ccc7894dc7d42efbfc48395e","measurement_commit":"b81cc816158f7c0fbe7f8ac0a6fcf2b3f38f4c86","immutable_measurement_receipt_commit":"9d8290160849a5124d1896fc318d53470585ef41","case_freeze_sha256":"c1595743f48fe8ae6b3f70f2ea401731052b3bd7614297feabaa8172ce190ae5","model_lock_sha256":"48b5ed9c98f16cc2b3b76bc0da2e7ec8072ecd89ba1914683181b1c20074a3a8","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"c5cfe765f6b72489220cc73f689acb1c1c2c9284ffd8d093de0e597b645f6fbf","measurement_artifact_sha256":"7dcab9d4b71e7de59094c07cbca830ebf674b601ac6de024db0d3e677e7c0067","policy_evaluation_sha256":"3ea5b62044b4da6e92812831ebb00205970849accffe763a270248a24a0adef4"}},{"batch_id":"external-gold-expansion-v2-batch4","benchmark_id":"external-gold-expansion-v2-batch4","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T18:47:01.455700Z","evidence_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"}},{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T15:38:03.968297Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":37,"number_of_batches":6,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"national-parks-mcp","subject_name":"National Parks","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"national-parks-mcp","subject_name":"National Parks","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":3,"total_measured_frozen_cases":21,"total_provider_valid_cases":21,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-expansion-v2-batch4","benchmark_id":"external-gold-expansion-v2-batch4","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T18:47:01.455700Z","evidence_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"}},{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T18:47:01.455700Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":21,"number_of_batches":3,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"national-parks-mcp-source-snapshot","subject_name":"National Parks MCP (source snapshot)","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"national-parks-mcp-source-snapshot","subject_name":"National Parks MCP (source snapshot)","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":24,"total_provider_valid_cases":24,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S1"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MEASURED","summary":{"total_measured_cases":24,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"nixos-mcp","subject_name":"NixOS","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"nixos-mcp","subject_name":"NixOS","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":3,"total_measured_frozen_cases":21,"total_provider_valid_cases":21,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-expansion-v2-batch4","benchmark_id":"external-gold-expansion-v2-batch4","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":0.833333,"wilson_95":{"lower":0.436497,"upper":0.969947,"lower_percent":43.6497,"upper_percent":96.9947},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"5/6","measured_at":"2026-09-10T18:47:01.455700Z","evidence_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"}},{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":1,"observed_semantic_miss_case_ids":["b4-nx-001"],"observed_semantic_misses":[{"case_id":"b4-nx-001","subject_id":"nixos-mcp","expected_tool":"home_manager_search","selected_tool":"home_manager_info","boundary_family":"HOME_MANAGER_VS_DARWIN_SEARCH","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch4/measurement-v2-batch4/semantic-miss-adjudication-v2-batch4.json","benchmark_id":"external-gold-expansion-v2-batch4","batch_id":"external-gold-expansion-v2-batch4"}],"observed_miss_boundary_families":["HOME_MANAGER_VS_DARWIN_SEARCH"],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T18:47:01.455700Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":21,"number_of_batches":3,"strict_accuracy_range_across_batches":{"minimum":0.833333,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":0.833333}},{"canonical_subject_id":"nixos-mcp-source-snapshot","subject_name":"NixOS MCP (source snapshot)","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"nixos-mcp-source-snapshot","subject_name":"NixOS MCP (source snapshot)","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":24,"total_provider_valid_cases":24,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S1"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MEASURED","summary":{"total_measured_cases":24,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"uk-premises-licence-mcp","subject_name":"UK Premises Licence MCP","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_variants":[{"canonical_subject_id":"uk-premises-licence-mcp","subject_name":"UK Premises Licence MCP","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"LIVE_TOOLS_LIST"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":24,"total_provider_valid_cases":24,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S1"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MEASURED","summary":{"total_measured_cases":24,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"wikipedia-mcp-reference","subject_name":"wikipedia-mcp-reference","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"wikipedia-mcp-reference","subject_name":"wikipedia-mcp-reference","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":2,"total_measured_frozen_cases":15,"total_provider_valid_cases":15,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T19:13:49.842728Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":15,"number_of_batches":2,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}}],"counts":{"subjects_with_empirical_reliability_records":30,"subjects_with_multi_batch_evidence":12,"subjects_with_single_batch_evidence":18,"subjects_with_s2_backed_evidence":24,"subjects_with_observed_semantic_misses":9,"observed_semantic_miss_count":16},"known_limitations":["Batch metrics are intentionally not pooled into a single accuracy or reliability score.","Observed misses describe frozen historical measurements and do not predict future failures.","Version-bound snapshots do not establish current-live status.","Boundary-family fields are unavailable for some older external evidence and are disclosed as such.","Provider infrastructure failures are reported separately and are not semantic misses."]}