{"schema_version":"mcp-selection-lab.agent-trust-lab.profile.v1","profile_type":"AGENT_TRUST_LAB_PUBLIC_PROFILE","selection_evidence":{"schema_version":"mcp-selection-lab.agent-trust-lab.selection-evidence-status.v1","track":"SELECTION_EVIDENCE_EXPANSION_V1","status":"MEASURED","badge":"selection-tested","benchmark_version":"selection-evidence-expansion-v1","measurement_status":"MEASURED","strict_accuracy":1.0,"case_count":240,"provider_valid_count":240,"provider_valid_semantic_case_count":240,"infrastructure_failure_count":0,"provider_infrastructure_failure_count":0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.9842460800584412,"method":"wilson_score"},"case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","frozen_measurement_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","qualification_sha256":"057d9f889d1347467b3f048e9f8d6520451779ff9dfd0e7e4d6f5c421f2f6f25","artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826"},"measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_status":"S1 selection-tested","promotion_level":"S1","promotion_badge":"selection-tested","promotion_policy_version":"SELECTION_EVIDENCE_PROMOTION_POLICY_V1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","promotion_policy_commit_sha":"da80c29500dfacd14460648b89bd4650f46a4b06","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","constructed_schema_derived_cases":true,"external_public_gold":false,"s2_external_selection_tested":false,"maximum_level":"S1","historical_measurement_promotion":{"badge":null,"policy_path":"SELECTION_EVIDENCE_PROMOTION_POLICY_V1.md","reason":"SELECTION_EVIDENCE_PROMOTION_POLICY_V1_NOT_PRESENT","s2_eligible":false,"status":"PROMOTION_BLOCKED"},"subjects":[{"canonical_id":"context7","subject_type":"LIVE_TOOLS_LIST","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","qualification_status":"FREEZE_READY","selection_evidence_status":"MEASURED","badge":"selection-tested","score":{"canonical_id":"context7","frozen_case_count":24,"infrastructure_failure_count":0,"infrastructure_outcome_counts":{},"predictions":[{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-01","correct":true,"expected_tool":"resolve-library-id","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:17:46.191337Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"134947708284fa143d324a548d9b65ed9ea29605e4b302b760c11a1a3e88b867","response_hash":"ded6628ea026385315ac2ec857e726c0bc60590beacebbc1742b3ed5f52b2eeb","selected_tool":"resolve-library-id","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":41,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1052,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1093}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-02","correct":true,"expected_tool":"query-docs","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:17:48.112257Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"b21adbda1480827b13f2031a7bb0848f8e244c791360e0b38e8a59384b616a85","response_hash":"447d9241328570276081c22dcf67372fdea1b0a0f7223ef1199939929147ec6a","selected_tool":"query-docs","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":28,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1054,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1082}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-03","correct":true,"expected_tool":"resolve-library-id","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:17:49.249569Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"962d957ccbd97c387a91fb8d0218c17a64d4b36827bd7d396b93705ccd540160","response_hash":"db55c2c54abd6ef906e9ca552026cb027dac7ddcccabfc6b96312c871880ea2a","selected_tool":"resolve-library-id","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":40,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1053,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1093}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-04","correct":true,"expected_tool":"query-docs","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:17:50.690676Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"f441b1abbdc622238fc46c9295de57f537befc0da2b8e4b7c15e9b2da90fcd3e","response_hash":"008a7bb3d78842e08a13509ec51fe3c7267df9b2dcb6885894b7865cfdaf33b4","selected_tool":"query-docs","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":24,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1055,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1079}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-05","correct":true,"expected_tool":"resolve-library-id","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:17:52.085007Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"8fa64564920327df3c71ce78f5f26b8ad03eb3627c7339bb087edc2a453b9e25","response_hash":"93345bccc51caaac7fcf773c3ef040f66f4ef2db92e8eb919aee246d87e30d57","selected_tool":"resolve-library-id","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":43,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1051,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1094}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-06","correct":true,"expected_tool":"query-docs","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:17:53.157142Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"bc86c76957cf1f0dff420af8c99cd0c6d7ab18f13b22c421a5d089d6757c3647","response_hash":"8f7debb08cbd966660858196a3187ed292e180bb543953e849b292be49644920","selected_tool":"query-docs","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":23,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1053,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1076}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-07","correct":true,"expected_tool":"resolve-library-id","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:17:54.545313Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"3dfc39a6d635d3a0373d6fd8438d2538a11517c1449199e80e6166f873372834","response_hash":"7a88193d6c4babf621d0531fe4fbd8c179f8b9266b1ece01a58e76a875a75e67","selected_tool":"resolve-library-id","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":50,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1051,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1101}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-08","correct":true,"expected_tool":"query-docs","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:17:56.313822Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"124e726520fe6058fa6b0052ac26ff89ea300d8f9fe93075b7de0d1c761da3af","response_hash":"4879a59cde548bdfb665a28e4e86805e9303265dc73e1530c61f1e4afee0a3b0","selected_tool":"query-docs","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":28,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1053,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1081}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-09","correct":true,"expected_tool":"resolve-library-id","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:17:57.439573Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"740638a38ec088747e508485d010982b92330ee15d8398c6d2a67932d15a5eeb","response_hash":"5611f77b3e4e2f34f95f593286d3d2d3f4dab6ee17006969a087b18de8345863","selected_tool":"resolve-library-id","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":41,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1051,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1092}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-10","correct":true,"expected_tool":"query-docs","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:17:59.163441Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"607fb340f7a5f7b2d419b3d97f69c9f899959793c8e320f832ddb98c04ad39ee","response_hash":"0c45e7c37748f7c34f570f5b512585a562ce6b0690be6e464af019ef84faa4cf","selected_tool":"query-docs","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":28,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1053,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1081}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-11","correct":true,"expected_tool":"resolve-library-id","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:00.177520Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"e992262648f039cc4da8ca3748272d3be47bdcc0c81aa5258b34159eb8377b91","response_hash":"494b23cf76af548994be94fa4fcf6deb9896046ea580e1f2cffaf6b6ffd46983","selected_tool":"resolve-library-id","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":40,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1050,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1090}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-12","correct":true,"expected_tool":"query-docs","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:01.655295Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"8cf3e4f08b6baffc16d9218e0e1dc48971c197ade58a1da16fe12f62d3d62644","response_hash":"efdd5f87f23face5ac44730f31bac9fff0aa03ac5f5619fdeb86a1bb33f33031","selected_tool":"query-docs","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":40,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1052,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1092}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-13","correct":true,"expected_tool":"resolve-library-id","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:02.976330Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"6f80e46eaf93a857dd435d8bdba8a4990e3b526a5f7765bcb69ddb5479d4e673","response_hash":"b3be8c4627132e1c748a046f6fadcbea35c98023cdf2c6cc5e4aab63f220d1a0","selected_tool":"resolve-library-id","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":40,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1052,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1092}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-14","correct":true,"expected_tool":"query-docs","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:04.803158Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"bbfe888d86d4aba1ed1f047b084b5080ab98b2134b8bcda87a64c24710526e0f","response_hash":"4af097d961a811a977f26e38f1f54b2752b0b9eac3fdea90c4cb879ca8471ffb","selected_tool":"query-docs","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":36,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1054,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1090}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-15","correct":true,"expected_tool":"resolve-library-id","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:07.785519Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"4b276c9e4ab596309289183d377c0bd58e99efa29ed0d77845b008ce91da9400","response_hash":"c17ee091f388bdd3c5ae1e774410b8f96b4b81ffc2a1999833d31e469b2ae20e","selected_tool":"resolve-library-id","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":51,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1051,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1102}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-16","correct":true,"expected_tool":"query-docs","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:09.074492Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"56c5518bc79e3a4769bbebf99e76eadd3d88c40df8f4e2e4e374d9605e412381","response_hash":"2979e1f9ea5a6778fbabe99fd183f4f6bf233863ad88b89c3c2880d2c05cc87c","selected_tool":"query-docs","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":39,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1053,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1092}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-17","correct":true,"expected_tool":"resolve-library-id","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:10.111537Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"f2e665e6ef1fd95bf459702bb9cd14c23a340fb9584c9786c555207bf9fe5881","response_hash":"407b0122866e6556c160c1bbfbde85ade6e780627c92d1b8c049b3f06095a992","selected_tool":"resolve-library-id","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":40,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1052,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1092}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-18","correct":true,"expected_tool":"query-docs","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:11.398590Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"260f00ef762f31255c51c9b449cce77b455de52e2737e0946449c5e3525aa142","response_hash":"1ee400be8ffdd7844e277efae957471220f61285d83ca0e1417aa345ca06e775","selected_tool":"query-docs","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":23,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1054,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1077}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-19","correct":true,"expected_tool":"resolve-library-id","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:12.565727Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"d0e511ffec7fc6be6534e588d7fb9f7c9a81806230ea16a516237a536fa995a8","response_hash":"930b12d5f3c703516a9fe4c8565a89f5c517760e2fddac19ce0a2aa64c6982c6","selected_tool":"resolve-library-id","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":44,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1050,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1094}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-20","correct":true,"expected_tool":"query-docs","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:14.258233Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"f716a95d40389a4d7329c07fe39efa62c2ca2a057fea29dafff61c643f1a1149","response_hash":"1e5d1145896732607846cd1f94a5cd7e59dd05eaba6255e2fa3cab0492c1dd7e","selected_tool":"query-docs","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":40,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1052,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1092}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-21","correct":true,"expected_tool":"resolve-library-id","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:16.132876Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"551c587013ab5681c2acf1178d1ffc29cf2137841d51782eac3872124c362020","response_hash":"b85469e9e6865f12fe17f85381387c760c665539ead0381d7a4564f8209d0139","selected_tool":"resolve-library-id","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":51,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1052,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1103}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-22","correct":true,"expected_tool":"query-docs","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:17.532159Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"ca736fe8eb8a8fcc67a1144bc7f8d2818e98ab87be03acf646e730b459d1f678","response_hash":"40ed987830e74d8f8d457829b1783d94bc84e96111bdbc4f4420e34556867f3e","selected_tool":"query-docs","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":36,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1054,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1090}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-23","correct":true,"expected_tool":"resolve-library-id","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:18.622705Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"4f6b22d60295295069327f0290aef7bc3f3e85a4a26051c60ca906546b110f48","response_hash":"cd35854ef3c7eb41c2821b5a6c2818adcc78ecfaecf74061e5c65e9ae92cf37e","selected_tool":"resolve-library-id","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":40,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1050,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1090}},{"attempt_count":1,"canonical_id":"context7","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"context7-24","correct":true,"expected_tool":"query-docs","frozen_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:19.950207Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"9112ec19095e0d0ffbb01e8b3a2aac076cf4f27b70344ec9266ecdf7890a6d8d","response_hash":"57c9e0d983d462a2cfd9c56d8e00d10805f10de5a5e3f6b59bf775e337b752e4","selected_tool":"query-docs","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":36,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1052,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":1088}}],"provider_valid_semantic_case_count":24,"semantic_coverage":1.0,"semantic_outcome_counts":{"SELECTED_VALID_TOOL":24},"strict_accuracy":1.0,"strict_correct_count":24,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"}},"promotion":{"adjusted_accuracy":null,"badge":"selection-tested","checks":{"S1":{"full_frozen_case_coverage":true,"immutable_identity_hashes":true,"min_case_count":true,"no_business_tool_execution":true,"no_post_result_mutation":true,"no_target_mcp_network_calls":true,"prediction_count_equals_case_count":true,"semantic_failures_not_retried":true,"valid_semantic_selection_count_equals_case_count":true,"zero_provider_infrastructure_failures":true},"S2":{"external_benchmark_source":false,"inherits_s1":true,"misses_adjudicated_when_present":true,"no_adjusted_accuracy":true,"strict_score_authoritative":true},"S3":{"fresh_within_30_days":false,"inherits_s2":false,"min_case_count":false,"min_distinct_frozen_measurements":false,"min_strict_accuracy":true,"min_wilson_95_lower":true,"same_runtime_schema_epoch":false},"S4":{"inherits_s3":false,"min_distinct_model_families_or_providers":false,"min_distinct_models":false,"min_total_case_count":false,"no_unresolved_gold_schema_contradiction_latest":true,"pooled_thresholds_met":false}},"level":"S1","maximum_level_for_this_case_source":"S1","policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","policy_version":"SELECTION_EVIDENCE_PROMOTION_POLICY_V1","reasons":[],"s2_eligible":false,"status":"S1 selection-tested","strict_accuracy_authoritative":true}},{"canonical_id":"exa-mcp","subject_type":"LIVE_TOOLS_LIST","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","qualification_status":"FREEZE_READY","selection_evidence_status":"MEASURED","badge":"selection-tested","score":{"canonical_id":"exa-mcp","frozen_case_count":24,"infrastructure_failure_count":0,"infrastructure_outcome_counts":{},"predictions":[{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-01","correct":true,"expected_tool":"web_search_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:21.316757Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"775ece3546c4266c67294b42f95ea68eb38f98173aef0a4242bb40f0f0cfab4d","response_hash":"3d31c62ce3819bc7c4f16874fcb2684a7079ec0e193ebb749be501586b822910","selected_tool":"web_search_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":32,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":475,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":507}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-02","correct":true,"expected_tool":"web_fetch_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:22.288339Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"08b6ac6b2cddd9f4162b1683bae0ab6c914acf942808ff21ace370ebcbdfae11","response_hash":"b93753bbc3e239ae87f6ffa90ebbad91fed2eb8bec0ed184ce0cc227752c52d6","selected_tool":"web_fetch_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":23,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":470,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":493}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-03","correct":true,"expected_tool":"web_search_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:23.956753Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"ff13f021cc9e8ce0d372fa8a5724febb1389ed360ce30bc53ba804c9f6a96613","response_hash":"783bd871c6370059ea4dc232c374b3856858c417cf5a9440381f27fcdcde7676","selected_tool":"web_search_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":37,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":476,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":513}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-04","correct":true,"expected_tool":"web_fetch_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:25.025214Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"da732510b4e1ba2fb73ba6831d4151e5ae1db699b68707ea1e91d93afab123d4","response_hash":"41a4854cccd667ad7ceab77d7c113df76d0813cb1d974bc667ffeea48233ee58","selected_tool":"web_fetch_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":23,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":471,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":494}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-05","correct":true,"expected_tool":"web_search_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:25.921020Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"4304078178e0d3f2446db22a9fc7f6c0412177190246db7c8148f20001ced661","response_hash":"41b96993687a1026c24776901b0d79837db7c96185fed727df895a29dfff8474","selected_tool":"web_search_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":32,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":474,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":506}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-06","correct":true,"expected_tool":"web_fetch_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:26.828860Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"5d7cc99827728b0fd698c83cffbaae20131270341e4372aa50529aa3a9c06fc4","response_hash":"b00560417c2341d28354443c295d6f6732d05be5943b1576887f926038f553c3","selected_tool":"web_fetch_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":23,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":469,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":492}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-07","correct":true,"expected_tool":"web_search_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:27.969776Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"b0d793cebee965954f41b383da484412abc21ab1fbabaa6b4734877cff290c60","response_hash":"95118c4d3c708beb084d6cc11a05fb1065b51898845be1167cca236f2d03bd1f","selected_tool":"web_search_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":32,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":474,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":506}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-08","correct":true,"expected_tool":"web_fetch_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:28.927041Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"2a06345846ceddc50f290a90670574cf328ac9339676216c2ed2efc85ad9e7d0","response_hash":"20f774195431b0335acc6335529685300a80a49b66c1edfdf43e229eebe6d7da","selected_tool":"web_fetch_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":23,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":469,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":492}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-09","correct":true,"expected_tool":"web_search_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:29.882197Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"3422cb6718c0edd3c6049ec93876690eedd46c5c726baf3b83c762d16f4f1cd5","response_hash":"4f053d6df35d716c5025d98d8f637eeb22a9ffb51ad31612228873ad4704a8dd","selected_tool":"web_search_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":32,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":474,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":506}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-10","correct":true,"expected_tool":"web_fetch_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:31.158013Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"5b7f85c42ece4c7be5aefd28d64230cdbbadfbd6e1e515db299a2f87aa2ec902","response_hash":"38a5f7b3b249ea64acfaeaf04fb87c8552e2afc12acb0a754c9b6fb87813a9a1","selected_tool":"web_fetch_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":23,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":469,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":492}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-11","correct":true,"expected_tool":"web_search_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:32.126904Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"ce8c881bdea37928443c100c5f4d8621d7d18b02f5cbfa91c2e91c40762a47d8","response_hash":"68520d1acfee8b29256774cb328d3748b10b1571a3be02c7c9c5328abf97d29a","selected_tool":"web_search_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":32,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":473,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":505}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-12","correct":true,"expected_tool":"web_fetch_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:33.158696Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"1a010fbbde223608d91af9e8d01aaae5498e558d80741fbce6fd1d571fa34e48","response_hash":"209ca6b5a93af2b05eb3539be5c5d31d9ac4b7e815d3cb5602384a3f6c7e2038","selected_tool":"web_fetch_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":23,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":468,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":491}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-13","correct":true,"expected_tool":"web_search_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:34.137477Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"4d4105e8b4e9b6ca756595d8f4d64690354522ab2870385011eea2749017ae6f","response_hash":"5047bd7273d92063ca4b8197978a0f3bb0c1410b60fb63c5c4f581dcf33d4d95","selected_tool":"web_search_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":32,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":475,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":507}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-14","correct":true,"expected_tool":"web_fetch_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:35.980081Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"5db6b4a51f87f64ac9c25c4fb655a89abd31cfdd22da03d31a63330289e229d9","response_hash":"0bf1c7868bf9a2c21adbde4eb8f1b225f7d458206362d27c8f65304804a6eca9","selected_tool":"web_fetch_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":23,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":470,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":493}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-15","correct":true,"expected_tool":"web_search_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:37.439677Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"32ad108c7cb4d7748d0c4a4bfc7e446270904a67ed3a31507210624c33a99e3b","response_hash":"367a70ec818492a05bb7c9408a937917efbdc85bb81416ed71c31a9506ee2111","selected_tool":"web_search_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":36,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":474,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":510}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-16","correct":true,"expected_tool":"web_fetch_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:38.318666Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"4caf6c8085f14d53aa9b6b7a756d3f0f24a85091662732d620d798e8167c1c8e","response_hash":"da2e5ffa75eec4734a37e4d2ae3a2856a7f1822a024c962545c99041a9d07bb8","selected_tool":"web_fetch_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":23,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":469,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":492}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-17","correct":true,"expected_tool":"web_search_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:39.608182Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"d75b13d33dbbc4b5a3e307e2a154cc40b85946a5671276c3b141273e72f4ab81","response_hash":"0cc755a3825d9fea59372b2fa0b1607bfa2e9af31c1853852386a7ce4c5025ef","selected_tool":"web_search_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":33,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":475,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":508}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-18","correct":true,"expected_tool":"web_fetch_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:40.598015Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"d458a32a8dcbe889ec62e963ce734d27035b55ba99186e769a12d76ac02da845","response_hash":"15a06a74cb510984fcf2a35f470fdf334ad0cc8995716acdc65b03bc9533fc4b","selected_tool":"web_fetch_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":23,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":470,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":493}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-19","correct":true,"expected_tool":"web_search_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:42.043399Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"75ece8bcd6830779867caab6ce01bc09e0e549beacecd910de54f56a7ef9d5e1","response_hash":"445dede3de252e44a2d2e2ee522aefabfa95028f1fc8f456ac14820065619130","selected_tool":"web_search_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":38,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":473,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":511}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-20","correct":true,"expected_tool":"web_fetch_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:43.296471Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"c5e844699fe046903b9cf4a0f65620b0c56ce3243799c2fdea1895ada1d6b498","response_hash":"ef53b415a8fe68c9fdfc1e55eba65ae55d9daf9da1760e85f1a53c533d734740","selected_tool":"web_fetch_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":23,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":468,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":491}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-21","correct":true,"expected_tool":"web_search_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:44.550886Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"1e48b6271b4ad026b97a524d7f6d730413d22c194d84c3241df46d1a3c2e99b8","response_hash":"056cc115b3a61c8d32148bdbf79396a14a51a2173522e74532f1df7ec89abef4","selected_tool":"web_search_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":32,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":475,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":507}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-22","correct":true,"expected_tool":"web_fetch_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:45.451142Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"30ea1569687665c4f5cb396aaf50b09b23c72faac923e0762a6b303d613c8280","response_hash":"04e202e98fb8ba2e6849b6ed84842a5230868f6749348744ebcbe7d86600dd67","selected_tool":"web_fetch_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":23,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":470,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":493}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-23","correct":true,"expected_tool":"web_search_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:46.361743Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"03eff835861346aa8cb8030fe6c22ea85b434c2aad88f373bf0f7cd36133ef4b","response_hash":"8a3cb276633bd8f740cf1649eb236f7b50bbc73dba8f5d57b3a54608f2e69867","selected_tool":"web_search_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":32,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":473,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":505}},{"attempt_count":1,"canonical_id":"exa-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"exa-mcp-24","correct":true,"expected_tool":"web_fetch_exa","frozen_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:47.451899Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"d8718ab3b0e032ab6559ca990fbfaf8e8ed37effc80e0c44eeae4773d0da3985","response_hash":"8719ab4026ec878f4b51db553da6da28ee509a04b00982e984440057b85ae3f6","selected_tool":"web_fetch_exa","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":23,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":468,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":491}}],"provider_valid_semantic_case_count":24,"semantic_coverage":1.0,"semantic_outcome_counts":{"SELECTED_VALID_TOOL":24},"strict_accuracy":1.0,"strict_correct_count":24,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"}},"promotion":{"adjusted_accuracy":null,"badge":"selection-tested","checks":{"S1":{"full_frozen_case_coverage":true,"immutable_identity_hashes":true,"min_case_count":true,"no_business_tool_execution":true,"no_post_result_mutation":true,"no_target_mcp_network_calls":true,"prediction_count_equals_case_count":true,"semantic_failures_not_retried":true,"valid_semantic_selection_count_equals_case_count":true,"zero_provider_infrastructure_failures":true},"S2":{"external_benchmark_source":false,"inherits_s1":true,"misses_adjudicated_when_present":true,"no_adjusted_accuracy":true,"strict_score_authoritative":true},"S3":{"fresh_within_30_days":false,"inherits_s2":false,"min_case_count":false,"min_distinct_frozen_measurements":false,"min_strict_accuracy":true,"min_wilson_95_lower":true,"same_runtime_schema_epoch":false},"S4":{"inherits_s3":false,"min_distinct_model_families_or_providers":false,"min_distinct_models":false,"min_total_case_count":false,"no_unresolved_gold_schema_contradiction_latest":true,"pooled_thresholds_met":false}},"level":"S1","maximum_level_for_this_case_source":"S1","policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","policy_version":"SELECTION_EVIDENCE_PROMOTION_POLICY_V1","reasons":[],"s2_eligible":false,"status":"S1 selection-tested","strict_accuracy_authoritative":true}},{"canonical_id":"cloudflare-docs-mcp","subject_type":"LIVE_TOOLS_LIST","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","qualification_status":"FREEZE_READY","selection_evidence_status":"MEASURED","badge":"selection-tested","score":{"canonical_id":"cloudflare-docs-mcp","frozen_case_count":24,"infrastructure_failure_count":0,"infrastructure_outcome_counts":{},"predictions":[{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-01","correct":true,"expected_tool":"search_cloudflare_documentation","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:48.672806Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"eb20910839b4f38a9c941bc96bec8513e7fdfcc787984646a5ffecfe6f641833","response_hash":"da6a4235803bc595c39b33b147fc0efa1a50c404fd073f32b1cab7b99ee833bc","selected_tool":"search_cloudflare_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":329,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":351}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-02","correct":true,"expected_tool":"migrate_pages_to_workers_guide","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:49.496155Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"b257c492f5ab9dfdda8f17248473ad06a5821d2114c73a61267a50e87823f563","response_hash":"ffa3f959855e9e62a9aa7991b35daba99ef5930a3e6854b4d855438c373660be","selected_tool":"migrate_pages_to_workers_guide","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":328,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":346}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-03","correct":true,"expected_tool":"search_cloudflare_documentation","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:50.604182Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"031df8403a04e11b2b2f3a658b2f29721f707af7bc82cdbcda7f7bdc18024530","response_hash":"0e161e98c1dfeb091d7dc2ec865c0ae92e4d7294bc36daba335a3067e828c67a","selected_tool":"search_cloudflare_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":24,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":330,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":354}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-04","correct":true,"expected_tool":"migrate_pages_to_workers_guide","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:51.572234Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"f13925efe1a7677be2e1dfeec8ef3d42ad407ed1dd5a4d61a461ba17c6f7952f","response_hash":"e0ef135727dee3a2a5a1be6605513c34e455171df59249cbf6e46bb85a237ed7","selected_tool":"migrate_pages_to_workers_guide","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":329,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":347}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-05","correct":true,"expected_tool":"search_cloudflare_documentation","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:52.656420Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"9437549f5408a21edb139c047dbbfd3fbe4f30af1ec0a29e31d2ab17298cafd7","response_hash":"393ea2e6e5673249c6368d618fdb6407b498e5391271170cd52d1684e1515935","selected_tool":"search_cloudflare_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":328,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":350}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-06","correct":true,"expected_tool":"migrate_pages_to_workers_guide","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:53.391536Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"76fc97106e65bd616673ec70cb33175cb2b5ff6222e11527fd54dc29591d4177","response_hash":"aefa49eeb1b890b7d60068da3985c9b58a61cb207a8b6958cf8c32f9bb778265","selected_tool":"migrate_pages_to_workers_guide","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":327,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":345}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-07","correct":true,"expected_tool":"search_cloudflare_documentation","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:54.199890Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"545d1af1d8ec9013d16b6403c3a5d0abb4f887ebbca60a8fa26f1e774955a45a","response_hash":"4992b735b73680793833d5c4ff39fcd2216fccf43ff662cb6fb80b7d8c8f914e","selected_tool":"search_cloudflare_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":24,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":328,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":352}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-08","correct":true,"expected_tool":"migrate_pages_to_workers_guide","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:54.978241Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"934b58f7d310f4c79866e658f484907efb64b608fd3e5959671502359fa95b82","response_hash":"b0ce8cf061f5ca3324265fc93e02d06fb32182ac2e45b372dac55f9b2011016e","selected_tool":"migrate_pages_to_workers_guide","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":327,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":345}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-09","correct":true,"expected_tool":"search_cloudflare_documentation","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:55.883666Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"8e26a3383149f07543e01b4d60e2235ca89b4b6451ad6bfa5fbf4f59bcc6e007","response_hash":"24b69b2e16e1596a5975bd1755f7af99e431aaa355cb75adabf4a0eb0bd15fe7","selected_tool":"search_cloudflare_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":24,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":328,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":352}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-10","correct":true,"expected_tool":"migrate_pages_to_workers_guide","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:56.636005Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"9a28458bfaa78bf78268b0c6e0a8f88ae2e438a93a84ca15562ea71c6f082217","response_hash":"87975076e771368a7f605fbe4158fcaede5d8b0081a316f9cc29139d736e5771","selected_tool":"migrate_pages_to_workers_guide","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":327,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":345}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-11","correct":true,"expected_tool":"search_cloudflare_documentation","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:57.631697Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"8565c01f24d96e54f0086863040f4691fe70017c8ccb45aaf6315cc657455dac","response_hash":"97f2f4d305ef68c0f0f4fd68152deb015353ddc4732c40436776276ff9e79782","selected_tool":"search_cloudflare_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":327,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":349}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-12","correct":true,"expected_tool":"migrate_pages_to_workers_guide","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:58.692966Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"838341e7336857c1ad2e93f214327c9df52432b26a0f247aa51e01722ccc95e6","response_hash":"a14c676add980619fe4c0c7049b04c0c241826b585f0b84638459220743b6b3f","selected_tool":"migrate_pages_to_workers_guide","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":326,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":344}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-13","correct":true,"expected_tool":"search_cloudflare_documentation","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:18:59.487981Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"2c3ceeb4a73f5515adae219b0db9a1762ce698075625179e5395ae43723be139","response_hash":"dec87a0d638b36e82e70f1415f215c2976e19f64b2c331e37798fcba578dae36","selected_tool":"search_cloudflare_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":24,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":329,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":353}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-14","correct":true,"expected_tool":"migrate_pages_to_workers_guide","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:00.387384Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"312c8d66b77eba7add5110266f0774dfd9c0e27aaf15132140342c5d47e496a7","response_hash":"cffb32845fe34817079a15be640e121b87070285ac37a951d8d4add298a608fa","selected_tool":"migrate_pages_to_workers_guide","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":328,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":346}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-15","correct":true,"expected_tool":"search_cloudflare_documentation","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:01.395409Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"e14ef4febe422d1300ec0e3c2f3b19b5673d79acc73de3680f7bbc9986b5f3e7","response_hash":"dbe5c76a5bbb8b7aec002fbe09cc830fa6336ccf20552ac0b8fbe0a40e26bc89","selected_tool":"search_cloudflare_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":328,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":350}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-16","correct":true,"expected_tool":"migrate_pages_to_workers_guide","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:02.301274Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"ed10b6ce22c229d503a0073440e080d0958a360aa064c8801cf9ba575df8eefb","response_hash":"c507468167bb75413ab5685635e9a8f8db109182e4156a16056f40a8433214b2","selected_tool":"migrate_pages_to_workers_guide","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":327,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":345}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-17","correct":true,"expected_tool":"search_cloudflare_documentation","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:03.158057Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"aa5a04963af57ff4c0f252ff13258f462142c38f5755ad15f349225d3cae8e2f","response_hash":"f417afb3af33caa4a8c4f181e0a69375b362a7e6d1e7bf1c515a339ad2fbd2dc","selected_tool":"search_cloudflare_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":329,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":351}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-18","correct":true,"expected_tool":"migrate_pages_to_workers_guide","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:04.195156Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"a0db821885b7d649c1ca877bbf86bd668c17183379c6d21443d1df1461b7d841","response_hash":"36e61034ae046433b9d84c22dd877eddace965934f682179dfa50a006f3ea0cb","selected_tool":"migrate_pages_to_workers_guide","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":328,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":346}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-19","correct":true,"expected_tool":"search_cloudflare_documentation","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:05.642188Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"cb80afa69cbdc0190a07b8d6ce2f8f435c33af26349d4565102617690755adb4","response_hash":"2576fb8e99f7f334a3c8882eb4119660ee52596165aee3d68f047d63c77f002c","selected_tool":"search_cloudflare_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":24,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":327,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":351}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-20","correct":true,"expected_tool":"migrate_pages_to_workers_guide","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:06.490847Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"624e82f01c3f3cd3b1c600c9105631f204480467f5c702cba0e86124a7ac9ed8","response_hash":"12f89258f9b4731a41eb8d05b3e79828aec3605fcc0cc9b4651887063eff56ba","selected_tool":"migrate_pages_to_workers_guide","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":326,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":344}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-21","correct":true,"expected_tool":"search_cloudflare_documentation","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:12.345606Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"fde0be8b2bc1abf4e6146ff6e1a3ace59cfa9516927c9b688be8f28e5659c2d4","response_hash":"934b8b033725a6b9d078b979e1b9f2d023bfd63f44a083bfab33ae3d95b40263","selected_tool":"search_cloudflare_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":24,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":329,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":353}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-22","correct":true,"expected_tool":"migrate_pages_to_workers_guide","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:13.524137Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"8df7fc58c57526d772ed7266e3743db83d0d6dff35f2373102819b51e737da3f","response_hash":"576d44c11b12ae89a4b4713e8d4b6434c0547b2f67d6c7430ca485a5fb00b9e2","selected_tool":"migrate_pages_to_workers_guide","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":328,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":346}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-23","correct":true,"expected_tool":"search_cloudflare_documentation","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:14.407959Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"c22d7bff23a76f321a221ec14aa5bec970c3beb0f0389ca6407c5e5c1210c05b","response_hash":"33c5411d2c7749b35873831af813e4f3b59c225c0a6f71cf2151dfd91420b3d1","selected_tool":"search_cloudflare_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":24,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":327,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":351}},{"attempt_count":1,"canonical_id":"cloudflare-docs-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"cloudflare-docs-mcp-24","correct":true,"expected_tool":"migrate_pages_to_workers_guide","frozen_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:15.309150Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"20e3752c66205eb81dc6d940ac28863ba11dc065c50878ce559d8481a7322ad5","response_hash":"997aa2ee981172887bc668f2882da1ab7a6f336750ccb355937bbd2c34fcdd6c","selected_tool":"migrate_pages_to_workers_guide","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":326,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":344}}],"provider_valid_semantic_case_count":24,"semantic_coverage":1.0,"semantic_outcome_counts":{"SELECTED_VALID_TOOL":24},"strict_accuracy":1.0,"strict_correct_count":24,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"}},"promotion":{"adjusted_accuracy":null,"badge":"selection-tested","checks":{"S1":{"full_frozen_case_coverage":true,"immutable_identity_hashes":true,"min_case_count":true,"no_business_tool_execution":true,"no_post_result_mutation":true,"no_target_mcp_network_calls":true,"prediction_count_equals_case_count":true,"semantic_failures_not_retried":true,"valid_semantic_selection_count_equals_case_count":true,"zero_provider_infrastructure_failures":true},"S2":{"external_benchmark_source":false,"inherits_s1":true,"misses_adjudicated_when_present":true,"no_adjusted_accuracy":true,"strict_score_authoritative":true},"S3":{"fresh_within_30_days":false,"inherits_s2":false,"min_case_count":false,"min_distinct_frozen_measurements":false,"min_strict_accuracy":true,"min_wilson_95_lower":true,"same_runtime_schema_epoch":false},"S4":{"inherits_s3":false,"min_distinct_model_families_or_providers":false,"min_distinct_models":false,"min_total_case_count":false,"no_unresolved_gold_schema_contradiction_latest":true,"pooled_thresholds_met":false}},"level":"S1","maximum_level_for_this_case_source":"S1","policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","policy_version":"SELECTION_EVIDENCE_PROMOTION_POLICY_V1","reasons":[],"s2_eligible":false,"status":"S1 selection-tested","strict_accuracy_authoritative":true}},{"canonical_id":"deepwiki-mcp","subject_type":"LIVE_TOOLS_LIST","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","qualification_status":"FREEZE_READY","selection_evidence_status":"MEASURED","badge":"selection-tested","score":{"canonical_id":"deepwiki-mcp","frozen_case_count":24,"infrastructure_failure_count":0,"infrastructure_outcome_counts":{},"predictions":[{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-01","correct":true,"expected_tool":"ask_question","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:16.746706Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"15f42db6267e15df2e058a19fe3404c38349779b5e1d959f04f2eb5549cfec9f","response_hash":"b7212621cefe06079ff13d6f441c06c51c66ddcfb88301ae9e1f712160a61aef","selected_tool":"ask_question","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":32,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":328,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":360}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-02","correct":true,"expected_tool":"read_wiki_contents","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:18.340304Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"3c1008c3ef5674fdcb2dc42ef8adef4bf9079c9e44db9af9627884c7bbc2dc6c","response_hash":"444cfb62a826515f978d14666754d1a99cee3ee747c91fa6d45616b41871d1b5","selected_tool":"read_wiki_contents","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":21,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":315,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":336}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-03","correct":true,"expected_tool":"read_wiki_structure","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:19.366583Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"5bad0a577401d18e75b077669caa4ebf10a2580dac1cdbad41db5a0b3dc87c14","response_hash":"7e6e7387e475bbcef64610e5412a4b10f0d49d9868d13e7c9508f55d48450c95","selected_tool":"read_wiki_structure","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":21,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":319,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":340}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-04","correct":true,"expected_tool":"ask_question","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:20.977487Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"3e6e209b781c52383e7dbae77d2c70e6418842a20bf1212c6beff2cb458d4c73","response_hash":"6844ab1ed74f4a49bfdc40a7324d6b26c1e9d69e1b1d70b9af2d2f7315c59958","selected_tool":"ask_question","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":39,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":329,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":368}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-05","correct":true,"expected_tool":"read_wiki_contents","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:22.663798Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"b46b3d442bbf62c3d82d5d1b10ae83a4656ee7b3ae2064190f31d200747b9aa8","response_hash":"31c282f772b5f0595faaf19f079d4216615bc1b538c3a9d1f7c8f09ce758364d","selected_tool":"read_wiki_contents","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":316,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":338}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-06","correct":true,"expected_tool":"read_wiki_structure","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:24.168555Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"c420884c8b7cd43376de8dabefee37142a255f2fbb022de23adba91eb9e61801","response_hash":"fbabc9c498304053f97ab6cbf04ebc4c86a2c45344158fe718361e113ae020f1","selected_tool":"read_wiki_structure","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":320,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":342}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-07","correct":true,"expected_tool":"ask_question","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:25.552734Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"ce2ab1628f7f38af67de1abe4294dfc4b1a6a313aa50d4c4118d8ebab3cd509f","response_hash":"d982c2afb43e9c38ce3feee1c9241990b81edfe77a8a2d83db4ef4b9615d4d6b","selected_tool":"ask_question","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":35,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":327,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":362}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-08","correct":true,"expected_tool":"read_wiki_contents","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:29.243507Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"1bf6f5fd668fbaeb00b5a098da0b3e58d5f20e2f26500b7d9c0eef176d94c68d","response_hash":"ee2f37571cee64cc5b3171044a263eff58983bfa810f8e2252b63737d51c1b78","selected_tool":"read_wiki_contents","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":21,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":314,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":335}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-09","correct":true,"expected_tool":"read_wiki_structure","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:30.814921Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"755049460a0d442fc819bd0527f4b0ada8d779e1b441a0571ab31923a7f9d83e","response_hash":"2dc25412134c7b2a92939dfd01b781cbabcc9328d49ea849af481466bf035270","selected_tool":"read_wiki_structure","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":21,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":318,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":339}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-10","correct":true,"expected_tool":"ask_question","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:32.403371Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"7afd25a925e555f0fa4143656f26e758f74e6b8e0aa97b92640205bb79138637","response_hash":"098092823fd76338644f55b3ad8321452fed93a3cad47d434684a24333c9c8d6","selected_tool":"ask_question","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":44,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":327,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":371}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-11","correct":true,"expected_tool":"read_wiki_contents","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:33.812576Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"3f53a883fd1a2c68fed3f7a78297bfba29a771aeda6e9748ba9a01bb4c9091ae","response_hash":"87e42245ac9a432f02584a5466211c7ea6387961086bc8cf0bbeb7926b4f8176","selected_tool":"read_wiki_contents","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":25,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":314,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":339}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-12","correct":true,"expected_tool":"read_wiki_structure","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:35.497331Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"d6ae8195bb55807875653e2e94e17b2f48d0626aad64a24f5cb1d6540cbb5981","response_hash":"e687878f11cd69ca87521a6ab4e0fd1919168d93ce8ab555cc7a2dcc6ff36541","selected_tool":"read_wiki_structure","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":318,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":340}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-13","correct":true,"expected_tool":"ask_question","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:37.199300Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"e4c3d2c4e82683157fb389f5bb4a0bae598ac174427f10abd5f842682c92372e","response_hash":"7a0efec667ecfd2572d9df9e8fd09f36b5602408b8c609ccbdc275531904b6f9","selected_tool":"ask_question","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":27,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":327,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":354}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-14","correct":true,"expected_tool":"read_wiki_contents","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:38.379873Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"fe19fd90eaf4a2ba74482760161b6022aa09182a4d2f50661caed82f500f1e11","response_hash":"9d8933d063b44ecdef043de3009eccd73d064a77f292d69485bf81581e088787","selected_tool":"read_wiki_contents","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":21,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":314,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":335}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-15","correct":true,"expected_tool":"read_wiki_structure","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:39.664975Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"116e2af90b7e7bdb19d384cea93193d8533f8d128ce79f4997adb9b526544982","response_hash":"5fb535b69fa012421e292833c6269bb472a43d3b42ad9368ba01d55f22ae896d","selected_tool":"read_wiki_structure","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":23,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":318,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":341}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-16","correct":true,"expected_tool":"ask_question","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:41.606447Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"f4a2cbcd273e201ddf6285b4a34e2436f7315cf4a7e0763defeb7d165a00810f","response_hash":"7ae1ea8cfb92ed3d494e70eb927d6c535ebeac470babdd63d8e940d265de0165","selected_tool":"ask_question","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":43,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":326,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":369}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-17","correct":true,"expected_tool":"read_wiki_contents","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:42.936927Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"758eb66e71b7584bc814baf4e4e4995942e838fa1a5dfd382b3d6d223237a6e7","response_hash":"e91440d57a3f0bf764b9b7e38cbede1722cee891a84c4cc27d870780d9e63a9f","selected_tool":"read_wiki_contents","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":313,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":335}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-18","correct":true,"expected_tool":"read_wiki_structure","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:44.240984Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"5a06a6ffa4582affc4efcfa58d0274061cb3a9baa9a13abda6467808aa41e725","response_hash":"7799cbb7433e00d1baa8405c941b1b7e177ea70c8f28bb9bf7a06e64d29608d5","selected_tool":"read_wiki_structure","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":21,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":317,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":338}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-19","correct":true,"expected_tool":"ask_question","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:45.530630Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"5c1fc2630b90a129500e361a168b19d265ead2b44a2601a4bb106bdc2c5935e3","response_hash":"21e4527775d418536a2798aa21d319033ec738a180e11aebcf9276df4d815031","selected_tool":"ask_question","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":24,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":328,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":352}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-20","correct":true,"expected_tool":"read_wiki_contents","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:46.893194Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"c2bae4595232694249e7b00827133d217e6574a196916c4d454d7d2fd431b146","response_hash":"f8852cda86c0094c2328a27ad70272764e6fc19d8d9f2b02a3764d1d3638ef18","selected_tool":"read_wiki_contents","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":315,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":337}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-21","correct":true,"expected_tool":"read_wiki_structure","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:48.162074Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"1d729fe71dbadd76407c35ea68004ac095c3c65387369d1247c5168860644630","response_hash":"df365931e991356d7448a9a2710b4c363d87c1a9d31fc71c315d5f143d2a396b","selected_tool":"read_wiki_structure","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":319,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":341}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-22","correct":true,"expected_tool":"ask_question","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:49.849280Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"c7d43793985a6fde037232d527b4ba6a13f87e71ad35f2416eb5d18dba9f5dac","response_hash":"acc205e1829d8cd45e523d76ea120345b9b252a1e7e1b7ef6452ed70403832c0","selected_tool":"ask_question","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":47,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":327,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":374}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-23","correct":true,"expected_tool":"read_wiki_contents","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:50.961617Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"2df651f9378f0c6626e69efa229d38938c9f7c0e0347a4e2d786344b78d341c4","response_hash":"331318cbff1612a09045a768550c089ec4e50e4b1fd8f7aed2819e7487301703","selected_tool":"read_wiki_contents","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":314,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":336}},{"attempt_count":1,"canonical_id":"deepwiki-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"deepwiki-mcp-24","correct":true,"expected_tool":"read_wiki_structure","frozen_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:52.137694Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"5e7d370e9e3a3c01ecb1c99d1bb01838e236907358e79458eaaef43a1d88fd9e","response_hash":"87d079acf22010a89b4e6f528b689749eb4db15ae49fa42a87beb5961aabe9f0","selected_tool":"read_wiki_structure","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":318,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":340}}],"provider_valid_semantic_case_count":24,"semantic_coverage":1.0,"semantic_outcome_counts":{"SELECTED_VALID_TOOL":24},"strict_accuracy":1.0,"strict_correct_count":24,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"}},"promotion":{"adjusted_accuracy":null,"badge":"selection-tested","checks":{"S1":{"full_frozen_case_coverage":true,"immutable_identity_hashes":true,"min_case_count":true,"no_business_tool_execution":true,"no_post_result_mutation":true,"no_target_mcp_network_calls":true,"prediction_count_equals_case_count":true,"semantic_failures_not_retried":true,"valid_semantic_selection_count_equals_case_count":true,"zero_provider_infrastructure_failures":true},"S2":{"external_benchmark_source":false,"inherits_s1":true,"misses_adjudicated_when_present":true,"no_adjusted_accuracy":true,"strict_score_authoritative":true},"S3":{"fresh_within_30_days":false,"inherits_s2":false,"min_case_count":false,"min_distinct_frozen_measurements":false,"min_strict_accuracy":true,"min_wilson_95_lower":true,"same_runtime_schema_epoch":false},"S4":{"inherits_s3":false,"min_distinct_model_families_or_providers":false,"min_distinct_models":false,"min_total_case_count":false,"no_unresolved_gold_schema_contradiction_latest":true,"pooled_thresholds_met":false}},"level":"S1","maximum_level_for_this_case_source":"S1","policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","policy_version":"SELECTION_EVIDENCE_PROMOTION_POLICY_V1","reasons":[],"s2_eligible":false,"status":"S1 selection-tested","strict_accuracy_authoritative":true}},{"canonical_id":"gitmcp-docs","subject_type":"LIVE_TOOLS_LIST","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","qualification_status":"FREEZE_READY","selection_evidence_status":"MEASURED","badge":"selection-tested","score":{"canonical_id":"gitmcp-docs","frozen_case_count":24,"infrastructure_failure_count":0,"infrastructure_outcome_counts":{},"predictions":[{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-01","correct":true,"expected_tool":"match_common_libs_owner_repo_mapping","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:53.790467Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"6bd096a7c7e26773942353ce355ede1d01f6a3aa8e8f02373ab8b1904da49282","response_hash":"3982daf26837b40e8e3c5c74fb200649c469642b07b40b9ba599a73c18b0e137","selected_tool":"match_common_libs_owner_repo_mapping","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":546,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":568}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-02","correct":true,"expected_tool":"fetch_generic_documentation","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:55.125294Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"6d4f586ade53590b6d194bb1195f057db23fecb9759eee97ad649b5c1909dfac","response_hash":"bf7870b2df184cb67695d5c2f8f823b841fd1b4d20a2781bb1d613c64fbbedcb","selected_tool":"fetch_generic_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":26,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":552,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":578}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-03","correct":true,"expected_tool":"search_generic_documentation","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:57.190456Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"bfec355c26bb9a9f1232284030e0cc5b13f8d00df53684331241db269465e3f4","response_hash":"52a32bd7416a422d879b48fcd01afad5182768e4c16bf4004ac9947774dcc9b9","selected_tool":"search_generic_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":33,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":561,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":594}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-04","correct":true,"expected_tool":"search_generic_code","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:58.681572Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"5c9df0f81c9c227f704f294706b45fea816166ee22725b0b55c4a22e78786086","response_hash":"ad489f9ed23f9917ff3d4233f33e9dc5e1536e74e536bfbfab5b656d14c34789","selected_tool":"search_generic_code","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":30,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":559,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":589}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-05","correct":true,"expected_tool":"fetch_generic_url_content","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:19:59.967293Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"2a76b602e8ad542c7a6994314d13bc712a4ada3d8e94052cc394fbcc3584a9c1","response_hash":"b4a3351dd0a7741c62cb55143c4ddd9cbedabf4db442887e4b4bafbc36d41ec7","selected_tool":"fetch_generic_url_content","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":551,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":573}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-06","correct":true,"expected_tool":"match_common_libs_owner_repo_mapping","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:01.377451Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"ee30b632e9d4d2cb2aff548d540995e69183a8e5df48867bde92d911000adaa8","response_hash":"43e950af3264fce0a7662ac97034b13c90be118a8d81da078ad9f453379b9918","selected_tool":"match_common_libs_owner_repo_mapping","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":547,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":569}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-07","correct":true,"expected_tool":"fetch_generic_documentation","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:03.256054Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"4b9c33485c3064f4712b52bb1b0ba070c69768445b9296a448e25f7344b481f7","response_hash":"1a65f8a7e74857b5fe6ef05598c946135b6bbf3ba65dfcb4cc2a45218c22d696","selected_tool":"fetch_generic_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":26,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":553,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":579}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-08","correct":true,"expected_tool":"search_generic_documentation","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:04.969328Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"6a2e070e22475239880c18116825473ee8e01ee5d5e467c925cfa3df4dd88704","response_hash":"1c7bf89d3a4fabfc193385ede22382f43d0235e7928e52311cf447219f4117e4","selected_tool":"search_generic_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":31,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":562,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":593}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-09","correct":true,"expected_tool":"search_generic_code","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:06.586525Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"50e85c5d8aabc4cf5868bfd71cee35e0d45ff233d1e1e2ba8feaada2e7df9031","response_hash":"985ce6c1a8cd3be6ffde2d16c6ee13843691e4ce1601ec2426fabedc7e821296","selected_tool":"search_generic_code","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":29,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":560,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":589}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-10","correct":true,"expected_tool":"fetch_generic_url_content","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:07.794076Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"c716bcbdba0642bc64a4996c05159e05ee19d2faf33cfc29039d32b622f65b32","response_hash":"31a05ecb576c2b6edb6a87575c9392faaa3fc2670121313391271a7f3a064bb4","selected_tool":"fetch_generic_url_content","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":552,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":574}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-11","correct":true,"expected_tool":"match_common_libs_owner_repo_mapping","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:09.156808Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"7925766247a9971bfcfe9033b9b25563c8efffa86bb9d4f93c0fe12547059805","response_hash":"8d9a188c9091caecf548994a699531220232a0c5b326bfe204a0f0b9d204d136","selected_tool":"match_common_libs_owner_repo_mapping","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":545,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":567}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-12","correct":true,"expected_tool":"fetch_generic_documentation","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:10.249842Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"4f9b0a04e8c934a75fe63f6162497a35c0e8e457381fc8771b7a1fb983ce9c0f","response_hash":"20eadf68c36316ac9e12ec730dbbed0f6a6501b8af6ba05053e3bdcb57b37e57","selected_tool":"fetch_generic_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":26,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":551,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":577}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-13","correct":true,"expected_tool":"search_generic_documentation","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:11.441360Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"353a4820693df6db5eb765ebe6605094d7cf83176d9cf2f56408643740656142","response_hash":"179400de79f047838d4b6ede9358a8b56f877bb75684c1a8a30961e881b096c6","selected_tool":"search_generic_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":30,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":560,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":590}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-14","correct":true,"expected_tool":"search_generic_code","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:13.026742Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"ca551419f26e3c25ef58f7342a4d50b5b1a62d513d0c0d64bed5832bc30dff8f","response_hash":"7b4106d6bd68d0ee4553c398c9ee0630480c8b8cca469dc428bf9f39cb478c1a","selected_tool":"search_generic_code","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":29,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":558,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":587}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-15","correct":true,"expected_tool":"fetch_generic_url_content","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:14.765758Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"613cbed406ffeb10fa10bc1207e94f17e6ac8172f3903b2d00098e1865d125a6","response_hash":"bc606b70ec7fb2e930444a4fe88acdf4f940bfbd24ef105655227a4e2d1dc41e","selected_tool":"fetch_generic_url_content","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":550,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":572}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-16","correct":true,"expected_tool":"match_common_libs_owner_repo_mapping","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:16.031687Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"699f6a6a70a09eddd5a43366f751c55a6236af2ebc5159baa11075fce5f55e1f","response_hash":"5f2dfb213231872a612a4e451408cce99eea49e971c6625b5aa924dba0f611a6","selected_tool":"match_common_libs_owner_repo_mapping","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":545,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":567}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-17","correct":true,"expected_tool":"fetch_generic_documentation","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:17.291605Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"438fe3538c2356669ed5a8d6dbc82ebe7fbbceced164376d4467ef75d3761236","response_hash":"adda6e82602bea92c53a93a37a01eadff7b6e74847128acf7d0e8dcaaccc77ef","selected_tool":"fetch_generic_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":25,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":551,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":576}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-18","correct":true,"expected_tool":"search_generic_documentation","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:18.938935Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"3002b342d82f5d1e5d63b50d6c1c0354e06abb179196e362ae8082f2a3b2a683","response_hash":"5bb7d936939e7e674679efb6fed4e70c85dac5a7b59d04fd00190b94e1a09d74","selected_tool":"search_generic_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":31,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":560,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":591}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-19","correct":true,"expected_tool":"search_generic_code","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:20.336554Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"f9477766253082669068ea621f4fe64ec8dff6a180ef6e1879c68707da6b987d","response_hash":"9501021cbd23626da8fd3cf9c30c50a4baf8dbbf45af5d2b8fa3cb8cd3c95aa7","selected_tool":"search_generic_code","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":34,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":558,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":592}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-20","correct":true,"expected_tool":"fetch_generic_url_content","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:21.512994Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"882d84cd11b71c6d36de4b783098f8f1eaa7e5fadd79c888a97df7942d171b02","response_hash":"e893c5df8d2eaea57a6cd7eed9b312a5d1c00885270b744ab9b7230e797acf59","selected_tool":"fetch_generic_url_content","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":550,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":572}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-21","correct":true,"expected_tool":"match_common_libs_owner_repo_mapping","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:22.620166Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"619dd66c3f7e23264abc8ecac6d866b08a149d6b7b472344c184297af331c5cf","response_hash":"dc4c5549cfa034cc2f07e4c3d8196ed2de21ae24ca8eb53407520776c8517e4e","selected_tool":"match_common_libs_owner_repo_mapping","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":545,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":567}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-22","correct":true,"expected_tool":"fetch_generic_documentation","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:24.110045Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"95d2e5e2e49b81649c94beb9fde29c81b58d42d8066b7e334b6eadb211d51fda","response_hash":"277775f52d1800a3258c80770e970df2fcdf5e9c0c223e7163581db34dfadf82","selected_tool":"fetch_generic_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":25,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":551,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":576}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-23","correct":true,"expected_tool":"search_generic_documentation","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:25.332134Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"ffe54fbcc51645373a799000ebd01b65ddaadd7c73150ba9237c9d095cef06f0","response_hash":"893d1593c2cd03a414fb68516095028b5047915057c3ae9bb2c93322c8d513f0","selected_tool":"search_generic_documentation","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":30,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":560,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":590}},{"attempt_count":1,"canonical_id":"gitmcp-docs","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"gitmcp-docs-24","correct":true,"expected_tool":"search_generic_code","frozen_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:26.618478Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"5ac19c253dcdd07b902b5c973d6caeedbcc70c016ad548b88ae38c7cf821e21a","response_hash":"ce02fa49f7d1524228e8990ff70103cca81d1dbfab4990c13226f5fd668f197b","selected_tool":"search_generic_code","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":32,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":558,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":590}}],"provider_valid_semantic_case_count":24,"semantic_coverage":1.0,"semantic_outcome_counts":{"SELECTED_VALID_TOOL":24},"strict_accuracy":1.0,"strict_correct_count":24,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"}},"promotion":{"adjusted_accuracy":null,"badge":"selection-tested","checks":{"S1":{"full_frozen_case_coverage":true,"immutable_identity_hashes":true,"min_case_count":true,"no_business_tool_execution":true,"no_post_result_mutation":true,"no_target_mcp_network_calls":true,"prediction_count_equals_case_count":true,"semantic_failures_not_retried":true,"valid_semantic_selection_count_equals_case_count":true,"zero_provider_infrastructure_failures":true},"S2":{"external_benchmark_source":false,"inherits_s1":true,"misses_adjudicated_when_present":true,"no_adjusted_accuracy":true,"strict_score_authoritative":true},"S3":{"fresh_within_30_days":false,"inherits_s2":false,"min_case_count":false,"min_distinct_frozen_measurements":false,"min_strict_accuracy":true,"min_wilson_95_lower":true,"same_runtime_schema_epoch":false},"S4":{"inherits_s3":false,"min_distinct_model_families_or_providers":false,"min_distinct_models":false,"min_total_case_count":false,"no_unresolved_gold_schema_contradiction_latest":true,"pooled_thresholds_met":false}},"level":"S1","maximum_level_for_this_case_source":"S1","policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","policy_version":"SELECTION_EVIDENCE_PROMOTION_POLICY_V1","reasons":[],"s2_eligible":false,"status":"S1 selection-tested","strict_accuracy_authoritative":true}},{"canonical_id":"uk-premises-licence-mcp","subject_type":"LIVE_TOOLS_LIST","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","qualification_status":"FREEZE_READY","selection_evidence_status":"MEASURED","badge":"selection-tested","score":{"canonical_id":"uk-premises-licence-mcp","frozen_case_count":24,"infrastructure_failure_count":0,"infrastructure_outcome_counts":{},"predictions":[{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-01","correct":true,"expected_tool":"premises_licence_info","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:28.372369Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"13c6864ed3f951b26615ddccb14a1902fa81acde67f7223cd691411c6d2bf251","response_hash":"88088b91d9a3405c9641a9aadf6989c28785c5144724fa96a2afa610491cb306","selected_tool":"premises_licence_info","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6240,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":6118,"cached_tokens":0},"total_tokens":6258}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-02","correct":true,"expected_tool":"list_supported_authorities","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:30.173418Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"59409cf05662bb75f9c32dda9471d9d394400830f01ceef93e48c37e9115684c","response_hash":"c32ad68f632872b3d1c2ecef9c8d9b5e750436cee8193465c9d9a00a9a8f2a9e","selected_tool":"list_supported_authorities","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":15,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6226,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":27,"cached_tokens":6077},"total_tokens":6241}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-03","correct":true,"expected_tool":"licensing_source_status","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:32.036000Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"e189e6a98fbe83ad092ea6f14a1a3c9439923c73e9381e81844d7f750c7f810c","response_hash":"8d70617b65c3abf31a73a4875cf982621914f8cd4e3fa42deb220c3825fef81e","selected_tool":"licensing_source_status","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":16,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6226,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":27,"cached_tokens":6077},"total_tokens":6242}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-04","correct":true,"expected_tool":"source_fingerprint_status","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:33.650708Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"7e884232d72b3f49c72652e91c70d7f5647313caab1c7f4438b22d6bbc9cc907","response_hash":"84c63778573e22bf046dd6605ddbde06422a71affab59ce76a275ca5ed2a7839","selected_tool":"source_fingerprint_status","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":15,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6229,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":30,"cached_tokens":6077},"total_tokens":6244}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-05","correct":true,"expected_tool":"premises_licence_preflight","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:35.499957Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"b474e008ca863439b64f324efb13afc8ca18aa57ef890014341e8ffea44b5923","response_hash":"4e1a822d3ff17dd43e98fd7c68fb05d3daf7a458144b072f9215c24414c2c323","selected_tool":"premises_licence_preflight","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":29,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6250,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":51,"cached_tokens":6077},"total_tokens":6279}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-06","correct":true,"expected_tool":"compare_licensing_requirements","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:37.318590Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"fe1db4ffb8362fff96ee479799272ace3c9c6c37e2e182e4cd45a16b1aa6b9ce","response_hash":"8a06112fc5d2b3baccd9a43d6291fb22237cc21ada2e1ec340c732adeeb60a00","selected_tool":"compare_licensing_requirements","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":27,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6252,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":53,"cached_tokens":6077},"total_tokens":6279}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-07","correct":true,"expected_tool":"local_policy_capabilities","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:39.067743Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"e30cc30f309f5c1b4b2c014518f8ef6cd0901fae6c343e54da1b882252a719b8","response_hash":"ab643828ce6c49a6514f221de7e6a568f492400f4da038e26db0d3aed3907928","selected_tool":"local_policy_capabilities","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":25,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6226,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":27,"cached_tokens":6077},"total_tokens":6251}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-08","correct":true,"expected_tool":"preflight_readiness","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:40.621724Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"9cd8903819bca68a7ac74cfeff2f026e01bba09def83250cd9dd77e48627eeba","response_hash":"16dfe6c141e0931db77ebf77b3c223b7a386f5eeba7221dc9dd1bfe0f60f3298","selected_tool":"preflight_readiness","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":19,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6240,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":41,"cached_tokens":6077},"total_tokens":6259}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-09","correct":true,"expected_tool":"premises_licence_info","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:42.229569Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"c61fcd2782a7cc522f5858b579ab3c7309e82e23fb02cacb8653932b470d8b77","response_hash":"081be2615e2898bfc4a493052d486547db1aa3684c4c39ed0902b82327e57d00","selected_tool":"premises_licence_info","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6241,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":42,"cached_tokens":6077},"total_tokens":6259}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-10","correct":true,"expected_tool":"list_supported_authorities","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:44.096666Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"2fd205dbb05f57417bd2618a76f3194c5883f77af8233d7895142415b113859e","response_hash":"c2f7d8ba77b053b699022c0e0f5dcb7569970c1a0a7b80742c9df59c593d4b27","selected_tool":"list_supported_authorities","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":15,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6227,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":28,"cached_tokens":6077},"total_tokens":6242}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-11","correct":true,"expected_tool":"licensing_source_status","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:45.646834Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"1f6bbaf541ff20c29c58c760f22d20858d30f5929dfe1fc33087786562d2a72d","response_hash":"4fcbe1eb2aeb74736a5b72ed96854d319c933b96bd10c9f0e3c3debcd367243d","selected_tool":"licensing_source_status","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":16,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6227,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":28,"cached_tokens":6077},"total_tokens":6243}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-12","correct":true,"expected_tool":"source_fingerprint_status","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:47.304573Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"244809aa1f3496a0b497f56d1f882284aa18421d33d7210a9a4c9ee01aeb5706","response_hash":"72ef3897780af9b33417ee2fbb0adfbb06a103daf7b2a22419f0ed0a9e17994c","selected_tool":"source_fingerprint_status","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":15,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6230,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":31,"cached_tokens":6077},"total_tokens":6245}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-13","correct":true,"expected_tool":"premises_licence_preflight","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:49.073844Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"4495fbbda14ce707c52804457f4c61ad78e864605e2e43c8c7ba33e6b64982e9","response_hash":"b40d4162be612b50c4269d900bb014a9859ac6ba8761aca07f5e85fcd77aabad","selected_tool":"premises_licence_preflight","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":29,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6251,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":52,"cached_tokens":6077},"total_tokens":6280}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-14","correct":true,"expected_tool":"compare_licensing_requirements","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:50.698218Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"b1942b8a0a2ad314f03d504e8263418f7ccc1a46a31349e82d90133a070916c3","response_hash":"235e0bf74d43ecb9bd64a58254761d2647c6b340a3c9e421137ff745b6ff7588","selected_tool":"compare_licensing_requirements","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":27,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6253,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":54,"cached_tokens":6077},"total_tokens":6280}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-15","correct":true,"expected_tool":"local_policy_capabilities","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:52.258494Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"e09ae1e5b704c5940c82c15b0373e63d844aa50d8ee0d70f9949af22fa51819e","response_hash":"a1e3ec364c10cda9e8978c09dd55bf6978a2b8a7201b1f81f75f588da390b771","selected_tool":"local_policy_capabilities","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":15,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6227,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":28,"cached_tokens":6077},"total_tokens":6242}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-16","correct":true,"expected_tool":"preflight_readiness","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:54.137370Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"cb003d39f5a9dfacb11249668f8a333046bec45e326dc82a3d4fe21224f52f96","response_hash":"ca3f15f17f6f3e57d1873ce0ee480e27e29c5b4c4396d2e568f17383b71db560","selected_tool":"preflight_readiness","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":19,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6241,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":42,"cached_tokens":6077},"total_tokens":6260}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-17","correct":true,"expected_tool":"premises_licence_info","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:55.439507Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"12f13eb6a660c08dbb2e4d4ab53ea9d9f119d970b9c3612d8833e3b4871839d5","response_hash":"b57807c08971debf6b345c2e0c50df04d2f5a4495b17b90f4c1466485f3b428a","selected_tool":"premises_licence_info","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6239,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":40,"cached_tokens":6077},"total_tokens":6257}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-18","correct":true,"expected_tool":"list_supported_authorities","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:56.934445Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"4ffd50172d1e05a389efc67d0463bcdb668ed69c6544c001aab1cb73662c410d","response_hash":"82c09538a84f40f2ade5cbca5f034c49c20c756768e43ecd293f99f504248826","selected_tool":"list_supported_authorities","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":15,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6225,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":26,"cached_tokens":6077},"total_tokens":6240}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-19","correct":true,"expected_tool":"licensing_source_status","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:20:58.552813Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"74c18ad6315bc7f9153ff9dcc99465821352891345476c501d54036d50b6e856","response_hash":"0567bb6ab14b8e6516ab57874edc198fa815f0c455e54cd5e1942d73456c2a6a","selected_tool":"licensing_source_status","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":16,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6225,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":26,"cached_tokens":6077},"total_tokens":6241}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-20","correct":true,"expected_tool":"source_fingerprint_status","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:00.014115Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"474981cc646d4e0f0d85b0101656daa976ea0291a3219903baeba842c43d286b","response_hash":"67daaf7d2ba1fd56d4f2bce08884c80c745e69dacee2f48334db631ee3d9f11f","selected_tool":"source_fingerprint_status","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":15,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6228,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":29,"cached_tokens":6077},"total_tokens":6243}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-21","correct":true,"expected_tool":"premises_licence_preflight","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:01.443762Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"7947e97c70ac50cb127d4210ecf4a47e6765513c36d61e7150918cc1d178002b","response_hash":"6d1b809babdff0a95c15306626b584deca387f7e447f077742ad4f3a756eb5e1","selected_tool":"premises_licence_preflight","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":29,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6249,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":50,"cached_tokens":6077},"total_tokens":6278}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-22","correct":true,"expected_tool":"compare_licensing_requirements","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:02.951659Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"8dfde3b51567378cbe2abba6a0e7089edbe421065241f5725c5878b1b62310de","response_hash":"c0a715f49fb07b16031a15b9f18026d958c08aaad136402dddcd10a6a3dcf737","selected_tool":"compare_licensing_requirements","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":27,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6251,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":52,"cached_tokens":6077},"total_tokens":6278}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-23","correct":true,"expected_tool":"local_policy_capabilities","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:04.454959Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"6e10380cf3643443ec40b54ce078fa326cf3d7f27dde38f74260a9763713aedd","response_hash":"7cfbe3126aca92f58bcd3025dbef19a9d643d5fa75cbfae3378a4bc3e5bef6c9","selected_tool":"local_policy_capabilities","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":15,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6225,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":26,"cached_tokens":6077},"total_tokens":6240}},{"attempt_count":1,"canonical_id":"uk-premises-licence-mcp","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"uk-premises-licence-mcp-24","correct":true,"expected_tool":"preflight_readiness","frozen_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:06.034830Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"49f451291477b70576ce0d1a0dfd8a848a7e4d4c203a03fa0dc327e8681b5d38","response_hash":"118109bfdeea2f04ea14100d00c3a897899b3e16dac0dd843fc7aa5cadd356a5","selected_tool":"preflight_readiness","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":19,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":6239,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":40,"cached_tokens":6077},"total_tokens":6258}}],"provider_valid_semantic_case_count":24,"semantic_coverage":1.0,"semantic_outcome_counts":{"SELECTED_VALID_TOOL":24},"strict_accuracy":1.0,"strict_correct_count":24,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"}},"promotion":{"adjusted_accuracy":null,"badge":"selection-tested","checks":{"S1":{"full_frozen_case_coverage":true,"immutable_identity_hashes":true,"min_case_count":true,"no_business_tool_execution":true,"no_post_result_mutation":true,"no_target_mcp_network_calls":true,"prediction_count_equals_case_count":true,"semantic_failures_not_retried":true,"valid_semantic_selection_count_equals_case_count":true,"zero_provider_infrastructure_failures":true},"S2":{"external_benchmark_source":false,"inherits_s1":true,"misses_adjudicated_when_present":true,"no_adjusted_accuracy":true,"strict_score_authoritative":true},"S3":{"fresh_within_30_days":false,"inherits_s2":false,"min_case_count":false,"min_distinct_frozen_measurements":false,"min_strict_accuracy":true,"min_wilson_95_lower":true,"same_runtime_schema_epoch":false},"S4":{"inherits_s3":false,"min_distinct_model_families_or_providers":false,"min_distinct_models":false,"min_total_case_count":false,"no_unresolved_gold_schema_contradiction_latest":true,"pooled_thresholds_met":false}},"level":"S1","maximum_level_for_this_case_source":"S1","policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","policy_version":"SELECTION_EVIDENCE_PROMOTION_POLICY_V1","reasons":[],"s2_eligible":false,"status":"S1 selection-tested","strict_accuracy_authoritative":true}},{"canonical_id":"nixos-mcp-source-snapshot","subject_type":"VERSION_BOUND_SOURCE_SNAPSHOT","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","qualification_status":"FREEZE_READY","selection_evidence_status":"MEASURED","badge":"selection-tested","score":{"canonical_id":"nixos-mcp-source-snapshot","frozen_case_count":24,"infrastructure_failure_count":0,"infrastructure_outcome_counts":{},"predictions":[{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-01","correct":true,"expected_tool":"nixos_search","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:07.541711Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"9c88cc9804ff7f162a329a0f93f704cd249c28638cf54ee34f91b64b9f734ce9","response_hash":"6d9b65729a346f8fff21386ed60d8bb12e9fad2cddbeb1fe7d56cf3c3934a8c5","selected_tool":"nixos_search","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":32,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1792,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":1670,"cached_tokens":0},"total_tokens":1824}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-02","correct":true,"expected_tool":"nixos_info","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:09.187793Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"4ad1896a9037dd495100741f72219fa50501aa0f062a395c95aae9424df499fe","response_hash":"09a6c62ba115ef8487a9b215f400f0dbe1addc5be56f2919ac87fd6c334b273b","selected_tool":"nixos_info","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1793,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":38,"cached_tokens":1633},"total_tokens":1815}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-03","correct":true,"expected_tool":"nixos_channels","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:10.252290Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"b1847ea2991e49ec9b3b76f567371c7680574a16bb96f3adf0cdd913ee87d034","response_hash":"533026fc44f1e2a31535623614f3ae117e484385eeeaf89b56eeed26305ae5e6","selected_tool":"nixos_channels","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":14,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1785,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":30,"cached_tokens":1633},"total_tokens":1799}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-04","correct":true,"expected_tool":"nixos_stats","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:12.252694Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"4a7b8a8ada0301df4935fdb8f98e01a81091a053153049d165d0df99ea73fd5d","response_hash":"42116ce85293830da8e1dd0baa48c20daf3c1cb717e90fa09c68273e97e2a2b4","selected_tool":"nixos_stats","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":19,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1784,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":29,"cached_tokens":1633},"total_tokens":1803}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-05","correct":true,"expected_tool":"home_manager_search","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:14.032173Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"29c751007b815fb7bf0e64b27c5f0beda6ef5fefed826abe42343a475f5fc353","response_hash":"19bdc63502e29065bd4d562b1da0baee7ecb3d06b9d66af5ab19f5e69bbee88a","selected_tool":"home_manager_search","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1787,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":32,"cached_tokens":1633},"total_tokens":1805}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-06","correct":true,"expected_tool":"home_manager_info","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:15.885281Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"5b76b0c78b6de91b8a97050a548cc6ed43e06cb73177cffe0d751bb6144cba26","response_hash":"ae71fbafc25eabb0ac514ea58a875acbb5fcdfc67b97f84d8828f9d89b2d4994","selected_tool":"home_manager_info","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1791,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":36,"cached_tokens":1633},"total_tokens":1809}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-07","correct":true,"expected_tool":"home_manager_stats","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:16.830130Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"2dbc32f0b150960934ef3d5ff912bc8375b7bc07531d0de6608312895c24c477","response_hash":"05500b1949e85943601ffcc3f8ca011a02e06a4c34f18640d6b73f50c7b0af6e","selected_tool":"home_manager_stats","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":14,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1782,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":27,"cached_tokens":1633},"total_tokens":1796}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-08","correct":true,"expected_tool":"home_manager_list_options","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:18.194368Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"e4ca9b8fdbeeeb405c44c52a9c99d01c5f9179310d79222337fd6b03ea9ef870","response_hash":"6936cedaac26a1a037cef4b2264200cfcf0e605a525d5a3db1cee2d868bfe2ef","selected_tool":"home_manager_list_options","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":15,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1782,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":27,"cached_tokens":1633},"total_tokens":1797}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-09","correct":true,"expected_tool":"home_manager_options_by_prefix","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:19.766095Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"0a1d7e3c4fee80522164d4031b9392e2c19ac39a844f32c36ac16ee18cc74787","response_hash":"e74401d6ef5864dd1b937dd05d0e362cf0cc95ccbec338eef4731db3d980bdfc","selected_tool":"home_manager_options_by_prefix","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1791,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":36,"cached_tokens":1633},"total_tokens":1813}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-10","correct":true,"expected_tool":"darwin_search","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:21.190699Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"09f206eb44491696dbceeef403d801796b3237181d1e2c7d657c28578aae45af","response_hash":"6c14e2944515f343926ea1f918c81a744483ed8d878b3c4e46fd7d3bfc490d55","selected_tool":"darwin_search","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":26,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1793,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":38,"cached_tokens":1633},"total_tokens":1819}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-11","correct":true,"expected_tool":"darwin_info","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:22.737045Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"f5e03e7380d903ec79e73c45db34b30cf8203e1d4a18f82eee4ddfff3093e5e3","response_hash":"08548a9660ba422ec98caafd09ca6537d9f5ac9152aac1d9f7cccceb2dd99843","selected_tool":"darwin_info","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":25,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1793,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":38,"cached_tokens":1633},"total_tokens":1818}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-12","correct":true,"expected_tool":"darwin_stats","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:23.629574Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"a451fad04626d6eb0320d26db0c47ff680ca972ea17ead569ef2dc41a255db79","response_hash":"86e5dc3cc337e835af0948a39db194ab325c72b37f31c12aa93a4514ce4b0bba","selected_tool":"darwin_stats","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":15,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1784,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":29,"cached_tokens":1633},"total_tokens":1799}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-13","correct":true,"expected_tool":"darwin_list_options","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:24.624329Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"4003e35915884cf0e7e029a015bc67f6d275cc9b0c813a0bd822958d1626ab9f","response_hash":"f97f180ad11c6144e3818848f9d46b86d214cff0f29c2be55fa3753b6085f719","selected_tool":"darwin_list_options","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":16,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1784,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":29,"cached_tokens":1633},"total_tokens":1800}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-14","correct":true,"expected_tool":"darwin_options_by_prefix","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:25.724620Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"9c915c3948982ad1e856ebed843664d3dd9dd4d66df560fa86be5a77267764bf","response_hash":"ba25b92712f42d6033692d6a8b8c7230d6b247aa224cd41b58c9a1500da0295f","selected_tool":"darwin_options_by_prefix","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":23,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1793,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":38,"cached_tokens":1633},"total_tokens":1816}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-15","correct":true,"expected_tool":"nixos_flakes_stats","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:27.152488Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"a70341155246e8d8e5a5340f1dcd9cd9b4866de31f7ee6bc536f4a73444af3e3","response_hash":"a3679cdc6819458bab96b5e1817773e4694b70a71123a91373c517975bca7876","selected_tool":"nixos_flakes_stats","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":16,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1784,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":29,"cached_tokens":1633},"total_tokens":1800}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-16","correct":true,"expected_tool":"nixos_flakes_search","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:28.436336Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"3fcebc1215044d13645078a6c2e9670573ebe1dbe557bc71d7c911b8f5b9ea6d","response_hash":"ef496ea794007538a20a010eb29da340825776a4227aa211feb55ef392121e8a","selected_tool":"nixos_flakes_search","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":27,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1796,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":41,"cached_tokens":1633},"total_tokens":1823}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-17","correct":true,"expected_tool":"nixhub_package_versions","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:30.020879Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"0f3e41d56520a58afe1960593584881136f5bc6985d3af6c12a1eaa2ba7e66a3","response_hash":"30392d5217d881bbab6b930477e852b713ce41c62f99ffdc73b467c700fbbdd9","selected_tool":"nixhub_package_versions","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1801,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":46,"cached_tokens":1633},"total_tokens":1823}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-18","correct":true,"expected_tool":"nixhub_find_version","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:31.419696Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"c10c31bd97048ab9ef6850fffd8ed1a2cb3de191d99d44a965eb3fd2077878a4","response_hash":"b1243d7efe543d0e08c4b29d3eb461ddc735d415ea6f6ec8afd89994cb8f153b","selected_tool":"nixhub_find_version","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":30,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1799,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":44,"cached_tokens":1633},"total_tokens":1829}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-19","correct":true,"expected_tool":"nixos_search","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:32.503146Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"e53757b6ce1961d4aba4401ed3ae72e0be0f139fd773b0f3e9bdee6777f724e7","response_hash":"0a84ae41c32f17b374b7a28e0fc67384e5330858880a95bab2d1a99df1a8834a","selected_tool":"nixos_search","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1793,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":38,"cached_tokens":1633},"total_tokens":1811}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-20","correct":true,"expected_tool":"nixos_info","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:33.873631Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"d5eed8af6e7b08e3e47c19e9f07e2e77823833ad503af71f2722bdbdce8e3cb3","response_hash":"c78d0b961708661e299f7087fa8578a465d6889cd9890bbfd9b70243afb68312","selected_tool":"nixos_info","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":22,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1794,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":39,"cached_tokens":1633},"total_tokens":1816}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-21","correct":true,"expected_tool":"nixos_channels","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:35.160187Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"5916a64b61d57070c1e398786f7a8173a349852ed213d5a0ed64af5000eb845e","response_hash":"6c0c421cdb21e2ff942a9692de7e26a7f5b5357f39a9fed11f5822b6c8bb7201","selected_tool":"nixos_channels","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":14,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1786,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":31,"cached_tokens":1633},"total_tokens":1800}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-22","correct":true,"expected_tool":"nixos_stats","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:36.540705Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"d4b4578b1788324381400bc917a58823d8af61ded9dad9c87c4eef9dc3d99ca0","response_hash":"2773f99d080c0eef92c7f0b99d92467f1a55844606f14ce6120b274db0c32aab","selected_tool":"nixos_stats","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":19,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1785,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":30,"cached_tokens":1633},"total_tokens":1804}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-23","correct":true,"expected_tool":"home_manager_search","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:38.001700Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"a673826145aa2f1ac3ccc48f4f40bb558bba172cc4d43e660599a628c50c7055","response_hash":"61a510e164f0fd71e7aebe0a0957549890d5701c925b8247d24543339211430d","selected_tool":"home_manager_search","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":21,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1788,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":33,"cached_tokens":1633},"total_tokens":1809}},{"attempt_count":1,"canonical_id":"nixos-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"nixos-mcp-source-snapshot-24","correct":true,"expected_tool":"home_manager_info","frozen_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:39.127606Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"881713823ec46476a260482eaab293fd33375271b8a9d243f98f2ea6e09b6204","response_hash":"736a839ee881be1704ebed51ab319b7f2b2a33a50813166ce8d02a6c1cb5d548","selected_tool":"home_manager_info","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1792,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":37,"cached_tokens":1633},"total_tokens":1810}}],"provider_valid_semantic_case_count":24,"semantic_coverage":1.0,"semantic_outcome_counts":{"SELECTED_VALID_TOOL":24},"strict_accuracy":1.0,"strict_correct_count":24,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"}},"promotion":{"adjusted_accuracy":null,"badge":"selection-tested","checks":{"S1":{"full_frozen_case_coverage":true,"immutable_identity_hashes":true,"min_case_count":true,"no_business_tool_execution":true,"no_post_result_mutation":true,"no_target_mcp_network_calls":true,"prediction_count_equals_case_count":true,"semantic_failures_not_retried":true,"valid_semantic_selection_count_equals_case_count":true,"zero_provider_infrastructure_failures":true},"S2":{"external_benchmark_source":false,"inherits_s1":true,"misses_adjudicated_when_present":true,"no_adjusted_accuracy":true,"strict_score_authoritative":true},"S3":{"fresh_within_30_days":false,"inherits_s2":false,"min_case_count":false,"min_distinct_frozen_measurements":false,"min_strict_accuracy":true,"min_wilson_95_lower":true,"same_runtime_schema_epoch":false},"S4":{"inherits_s3":false,"min_distinct_model_families_or_providers":false,"min_distinct_models":false,"min_total_case_count":false,"no_unresolved_gold_schema_contradiction_latest":true,"pooled_thresholds_met":false}},"level":"S1","maximum_level_for_this_case_source":"S1","policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","policy_version":"SELECTION_EVIDENCE_PROMOTION_POLICY_V1","reasons":[],"s2_eligible":false,"status":"S1 selection-tested","strict_accuracy_authoritative":true}},{"canonical_id":"dex-paprika-mcp-source-snapshot","subject_type":"VERSION_BOUND_SOURCE_SNAPSHOT","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","qualification_status":"FREEZE_READY","selection_evidence_status":"MEASURED","badge":"selection-tested","score":{"canonical_id":"dex-paprika-mcp-source-snapshot","frozen_case_count":24,"infrastructure_failure_count":0,"infrastructure_outcome_counts":{},"predictions":[{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-01","correct":true,"expected_tool":"getNetworks","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:40.614832Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"aebb75f40ad8ca081f5e21a30e92820e8da20586944cbbaed82969d1cf254b24","response_hash":"f08234707c5e5dd123c02349b834877a0479e4757750a52c1695c105b1b854af","selected_tool":"getNetworks","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":13,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1444,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":1322,"cached_tokens":0},"total_tokens":1457}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-02","correct":true,"expected_tool":"getNetworkDexes","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:42.002818Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"345cb14157c91eb69413627e238315efc9ca5a4fae719155235ccd0eced72828","response_hash":"d8be69bd7281cf8b832546669278933b01032b7894b69adcc8e386c849d02d92","selected_tool":"getNetworkDexes","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":27,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1450,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":36,"cached_tokens":1292},"total_tokens":1477}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-03","correct":true,"expected_tool":"getNetworkPools","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:43.383900Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"0043fa80aa080bb6fbecdd0bd73335c75d99eab7aeabfa910a91b4b023b73ad7","response_hash":"bb1e03cf245818d70661666b8c6663bf7eaee666aca26a45b764473e8ba3c28d","selected_tool":"getNetworkPools","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1454,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":40,"cached_tokens":1292},"total_tokens":1472}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-04","correct":true,"expected_tool":"getDexPools","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:44.975246Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"96945b529bee34bfc3d6a545723aae1fe12fcc85c69bfafd08de38b22178be07","response_hash":"c3e31915d8393ce2d033e59e748b4e50bc85d9537b4a0860838d5856dc68f162","selected_tool":"getDexPools","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":26,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1453,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":39,"cached_tokens":1292},"total_tokens":1479}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-05","correct":true,"expected_tool":"getPoolDetails","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:46.446829Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"fac75db0113abc98e48ee2a5c118bb261d16ffd6bd2f2fd9b39f8bd455e8e3bd","response_hash":"7796a7ce76627c29b1c6ab7d1a7d75b8e4e8b79467e6cb96adf32a9b74fd797f","selected_tool":"getPoolDetails","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":28,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1451,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":37,"cached_tokens":1292},"total_tokens":1479}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-06","correct":true,"expected_tool":"getTokenDetails","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:47.796819Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"196fc73f43251ed062224252d9c61747f47284cf0930fad0586ce7102fbe6098","response_hash":"50a3996e4023f58813dbe861e0f8f88ee79ffa8ebeec9ecad33363ccb1e61904","selected_tool":"getTokenDetails","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":28,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1454,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":40,"cached_tokens":1292},"total_tokens":1482}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-07","correct":true,"expected_tool":"getTokenPools","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:49.363765Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"bc5e465a23d840259ea97322f71cb546226d3e3f4d2962c48fa9f5feaf5edb36","response_hash":"1c6d923fabada19350f76bf6699ca4f7458a70944448622cb2d848f00ab31029","selected_tool":"getTokenPools","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":30,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1454,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":40,"cached_tokens":1292},"total_tokens":1484}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-08","correct":true,"expected_tool":"getPoolOHLCV","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:50.672840Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"dd2bd4e05ddd04063d62052bddb8fa7554c64d56223780c732b2535e702f4293","response_hash":"82c63a24f86f0c3aa3050613e7aa59e38b149a77d36095edaa3727c53955db62","selected_tool":"getPoolOHLCV","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":48,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1469,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":55,"cached_tokens":1292},"total_tokens":1517}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-09","correct":true,"expected_tool":"getPoolTransactions","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:52.156302Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"8e92d4b74267476a7e8d83778667121af58a4185a0c2d55d836ab6266f61e85b","response_hash":"d7580616b95be29549242df3efe0f9e22f90fdd8fc2cf15dea2620d1c7debbcf","selected_tool":"getPoolTransactions","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":30,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1451,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":37,"cached_tokens":1292},"total_tokens":1481}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-10","correct":true,"expected_tool":"search","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:53.334949Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"cbf0a5e43b1195da554ceb4fb85d9c3b3f0a9b7c3386201d93d72982e3ec58c7","response_hash":"b71b77c59a932f9ae8c8c1a4eccd7906c19183a9d39a328e4d2ebbcec6b5866a","selected_tool":"search","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":16,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1461,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":47,"cached_tokens":1292},"total_tokens":1477}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-11","correct":true,"expected_tool":"getStats","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:54.486858Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"75a2317fdb74d27482179337cc67e2d7ceeaf99024ed0493fd6693e238e2e9e9","response_hash":"6282e385aad731a8b1a6224b28eaa11dbb4612f52587f5c0f940ec629474f754","selected_tool":"getStats","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":13,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1458,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":44,"cached_tokens":1292},"total_tokens":1471}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-12","correct":true,"expected_tool":"getNetworks","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:55.603278Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"875d4a954b0984856146f893812efbc4ee014c8b7eda407b54150e1ab413da77","response_hash":"a79194a8730ac1eae6d9729c999f6394e34838ed14d4988a218c6dd2cdd73e42","selected_tool":"getNetworks","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":13,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1445,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":31,"cached_tokens":1292},"total_tokens":1458}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-13","correct":true,"expected_tool":"getNetworkDexes","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:56.661880Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"06d74794296d183aa584912f0870c17b0984b0675c4f479473d4a1da2ba3998d","response_hash":"9a896516a36ceb76e8a5fb280f07af2e8b0ad7cdb62a4139b3a037d4a487a34a","selected_tool":"getNetworkDexes","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":27,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1451,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":37,"cached_tokens":1292},"total_tokens":1478}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-14","correct":true,"expected_tool":"getNetworkPools","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:58.191461Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"1f6fcced1a83f598e7f51d4428030e8089c2a0f9d2698b19a82e5834dcd5d6dc","response_hash":"35d715a76d88d4436e8fbdc230a466ef02dc969dabf811bb77d74950fc541dea","selected_tool":"getNetworkPools","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1455,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":41,"cached_tokens":1292},"total_tokens":1473}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-15","correct":true,"expected_tool":"getDexPools","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:21:59.358173Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"c651da780ee999e6686a1d1508dea47dd04f12abbd2e1223515e61d8280f36d0","response_hash":"f13f99d5ad9c7cb343129d2a273ef4d64772909864e98ca8f58aac67acb04090","selected_tool":"getDexPools","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":26,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1454,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":40,"cached_tokens":1292},"total_tokens":1480}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-16","correct":true,"expected_tool":"getPoolDetails","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:00.911498Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"0ab6c911de272fb2474e8f21ed2e5b125eb6d36e7409714405d6014873b25d4f","response_hash":"877354e584d9a125115df1a386b6b143a1f98b8f134f48f3852592c428ae3601","selected_tool":"getPoolDetails","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":28,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1452,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":38,"cached_tokens":1292},"total_tokens":1480}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-17","correct":true,"expected_tool":"getTokenDetails","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:02.714160Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"a0d1c3abc92b854e5d1297dc9605fc88c9b5cd02ca0d9578ceeef92b116d6ac9","response_hash":"212987941e9980ae6d8c95441e633e07c60039632d92d31977d3bbba7c3f5c69","selected_tool":"getTokenDetails","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":28,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1455,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":41,"cached_tokens":1292},"total_tokens":1483}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-18","correct":true,"expected_tool":"getTokenPools","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:04.156528Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"247114e82630973d3606b6fc0a93614982753b00d357d6e8766dbde8d567a3f1","response_hash":"815b03064a82bfaa30b1b7156c4f230c65d9e73ab6c06348b2e3367ccbff42fb","selected_tool":"getTokenPools","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":30,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1455,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":41,"cached_tokens":1292},"total_tokens":1485}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-19","correct":true,"expected_tool":"getPoolOHLCV","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:05.567257Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"4cb9765cee182e792a4c4c83b8a44b3b9309fc90044c2b9167115b6d01f67800","response_hash":"bd7bdbf7333d0a14f9cf76db3a13e5f41bd87d5200e9945147161a86a97ac3d2","selected_tool":"getPoolOHLCV","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":39,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1470,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":56,"cached_tokens":1292},"total_tokens":1509}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-20","correct":true,"expected_tool":"getPoolTransactions","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:07.071529Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"73d172a8bc6dd430c10855df234d8a0d6bd3dcc2cd8b660d5f0cc20b546b6861","response_hash":"deb01d65b692c25083ddc2af65b0a7ae962ae0224415a6b33de71bcf58108ba8","selected_tool":"getPoolTransactions","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":28,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1452,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":38,"cached_tokens":1292},"total_tokens":1480}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-21","correct":true,"expected_tool":"search","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:08.172782Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"59a170d699de01b6cf0a2d8d98e617803636d17e28da4816bf624e724d9c4bdf","response_hash":"9c259a02d44fc1445d65d18715e80b718a2449645acc23b0682dd6370c27dd6a","selected_tool":"search","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":16,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1462,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":48,"cached_tokens":1292},"total_tokens":1478}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-22","correct":true,"expected_tool":"getStats","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:09.202215Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"a688a456f682f3bdfdb9c6dbbd07a6f2e6a619215b50c7736d91ad7001af4343","response_hash":"6471bb40292abc16dbc1adacc29b5a70a9710dcb36b8890b368bcee7acbc0248","selected_tool":"getStats","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":13,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1459,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":45,"cached_tokens":1292},"total_tokens":1472}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-23","correct":true,"expected_tool":"getNetworks","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:10.290324Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"3082ebae1afeebbf0a6c91b1ef4445ef338f4f33173c97848713fecdb90d2215","response_hash":"6f68e0ff67ea3ceb941895301929dd5b75f9fc29988ac740e0a0f8398070e48d","selected_tool":"getNetworks","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":13,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1443,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":29,"cached_tokens":1292},"total_tokens":1456}},{"attempt_count":1,"canonical_id":"dex-paprika-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"dex-paprika-mcp-source-snapshot-24","correct":true,"expected_tool":"getNetworkDexes","frozen_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:11.539950Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"2318d31e575e81210ce60989998e5068d8af3eeda69c834bccc6c5c8e8d277ca","response_hash":"46ae03191bca34ff121edc09641b32ba4116e742855b374c71072e4f00799807","selected_tool":"getNetworkDexes","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":27,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":1449,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":35,"cached_tokens":1292},"total_tokens":1476}}],"provider_valid_semantic_case_count":24,"semantic_coverage":1.0,"semantic_outcome_counts":{"SELECTED_VALID_TOOL":24},"strict_accuracy":1.0,"strict_correct_count":24,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"}},"promotion":{"adjusted_accuracy":null,"badge":"selection-tested","checks":{"S1":{"full_frozen_case_coverage":true,"immutable_identity_hashes":true,"min_case_count":true,"no_business_tool_execution":true,"no_post_result_mutation":true,"no_target_mcp_network_calls":true,"prediction_count_equals_case_count":true,"semantic_failures_not_retried":true,"valid_semantic_selection_count_equals_case_count":true,"zero_provider_infrastructure_failures":true},"S2":{"external_benchmark_source":false,"inherits_s1":true,"misses_adjudicated_when_present":true,"no_adjusted_accuracy":true,"strict_score_authoritative":true},"S3":{"fresh_within_30_days":false,"inherits_s2":false,"min_case_count":false,"min_distinct_frozen_measurements":false,"min_strict_accuracy":true,"min_wilson_95_lower":true,"same_runtime_schema_epoch":false},"S4":{"inherits_s3":false,"min_distinct_model_families_or_providers":false,"min_distinct_models":false,"min_total_case_count":false,"no_unresolved_gold_schema_contradiction_latest":true,"pooled_thresholds_met":false}},"level":"S1","maximum_level_for_this_case_source":"S1","policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","policy_version":"SELECTION_EVIDENCE_PROMOTION_POLICY_V1","reasons":[],"s2_eligible":false,"status":"S1 selection-tested","strict_accuracy_authoritative":true}},{"canonical_id":"national-parks-mcp-source-snapshot","subject_type":"VERSION_BOUND_SOURCE_SNAPSHOT","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","qualification_status":"FREEZE_READY","selection_evidence_status":"MEASURED","badge":"selection-tested","score":{"canonical_id":"national-parks-mcp-source-snapshot","frozen_case_count":24,"infrastructure_failure_count":0,"infrastructure_outcome_counts":{},"predictions":[{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-01","correct":true,"expected_tool":"findParks","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:12.652942Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"486269ce6c1a4621b40ee729b6b4dfa6a1a1fe67f0c899aa77a8dda3c83d6883","response_hash":"0b1f448a5c61f3ed54572fb314516cf5ddcfa26dc09c2ea2f0633985c0f9c900","selected_tool":"findParks","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":14,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":827,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":841}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-02","correct":true,"expected_tool":"getParkDetails","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:14.279114Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"ad964649151f961168f607498f7c867b558242fa185e4875762b77db24fab012","response_hash":"694551bda28bb4d623eb8f100d2e990125289f55fce623182d63c99868616c05","selected_tool":"getParkDetails","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":20,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":827,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":847}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-03","correct":true,"expected_tool":"getAlerts","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:15.676093Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"f84ab6af5099aee070dc2b40d3d7be0422ed2272fa8a6a26438721fe58eab58d","response_hash":"ed1a5bd684265ff02988b9473ef49e3e587f3b30cf74200b67c7298b5a8801d2","selected_tool":"getAlerts","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":17,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":826,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":843}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-04","correct":true,"expected_tool":"getVisitorCenters","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:16.742403Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"bbb3ccc494847ee27d11d4fcd54599d442708c8d0b7c5b2aad15691803cb8e73","response_hash":"bbd0d9d95526f02e7412316f661e19d945810b373a0773232732f9cccdc13b02","selected_tool":"getVisitorCenters","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":14,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":821,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":835}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-05","correct":true,"expected_tool":"getCampgrounds","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:17.850868Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"dcbe22ca2c80e7fd00c71c377840692cf3e57eaa018d5c748b36f5bab0c594d5","response_hash":"1a4d647488ab08bb8affdb080ac9269c315b6db751c5f4239a24b9b6e2b80fbb","selected_tool":"getCampgrounds","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":821,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":839}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-06","correct":true,"expected_tool":"getEvents","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:19.098741Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"2fe836c948d5a9a4e0982e49cff2b26e48efce23caca5bf92ab4987a8703adfd","response_hash":"43589501dbb969a86998437667c4779475c250b90b853b3d1fdf9907b86d111f","selected_tool":"getEvents","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":17,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":817,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":834}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-07","correct":true,"expected_tool":"findParks","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:20.428829Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"592bcf1133022f4e2d3cc8aad48838a045b7a8fc4ed93f8b2c372356cd52e550","response_hash":"4dbdee150d1ba2905ea2c852971e725e28f1f81037f702ed11bbbd3ceab5d197","selected_tool":"findParks","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":14,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":828,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":842}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-08","correct":true,"expected_tool":"getParkDetails","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:21.590243Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"fd0b42a04abcab83d2abb1a7d2770dc8f912546ea69805ba4a48794eba2ed3e7","response_hash":"f94628e9217898f2b948ffc6d16ebf27594b3fd2922cd51328274005dcb41405","selected_tool":"getParkDetails","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":20,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":828,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":848}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-09","correct":true,"expected_tool":"getAlerts","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:22.868775Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"62a598240026fa458795086f73d9005d8a14568aacc1820d2226e70582924164","response_hash":"870bb97a86dfdce3a6e8cc6ffc6413882fe9f5d16952f2171b9f617b2676d32c","selected_tool":"getAlerts","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":17,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":827,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":844}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-10","correct":true,"expected_tool":"getVisitorCenters","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:24.041543Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"e15dbf00451cf9d442d537db4af164f150c001b21367d05182637c0bd9851714","response_hash":"b482502f8545a3d53cd3812f2d8e09f0cc3fbe6339f0a4e3d5db76375b9be3d0","selected_tool":"getVisitorCenters","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":14,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":822,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":836}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-11","correct":true,"expected_tool":"getCampgrounds","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:25.360960Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"0cce1a22053eb6a24dece7e4b11d3d8ffd4155d9f4f5a3c531850b0485b30e33","response_hash":"af6dfbda83ba86df930202ddb9729945606a69fffa54c8fd75f1d566528b65ce","selected_tool":"getCampgrounds","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":14,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":822,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":836}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-12","correct":true,"expected_tool":"getEvents","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:26.537898Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"829cd971a218385c080d63bf495f26d632a1a8e769aa04dd2785f5e5e3b6ee78","response_hash":"9a03711fda0aab06cb0316b2fb6a7312690e5a460fb01ae82c4ccfdbdb2090a5","selected_tool":"getEvents","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":17,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":818,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":835}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-13","correct":true,"expected_tool":"findParks","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:27.638785Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"3822afb1cb732e7ffd9284e26393fb07c07f77bf4fb6ad5ad019d7263de92d71","response_hash":"59e79a94989c3f73877b137f264d928279aaaf9110b1c67163fe86d3da0bed5f","selected_tool":"findParks","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":826,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":844}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-14","correct":true,"expected_tool":"getParkDetails","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:28.629220Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"c0a8de2ce534ae6527334c107a60ca594f96cce8292b3a75630d8d64c2236a44","response_hash":"fb57230fd608ba53963f6e200bc0c6b3ebd30a38ed60bcadfbb80f00f83094eb","selected_tool":"getParkDetails","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":20,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":826,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":846}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-15","correct":true,"expected_tool":"getAlerts","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:29.751185Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"d9084a8f4e061d4b2dd7bc51ee9f0d73c22d6205f86ce84a38c41a1501ca565d","response_hash":"389e214a0462659251d7d5b563cda658fda4a4eb3701f6bb37674835530f2c3d","selected_tool":"getAlerts","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":17,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":825,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":842}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-16","correct":true,"expected_tool":"getVisitorCenters","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:30.920532Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"2c5e6ead206b51006a81e8a1f8205e6f007ea0f989b5265c40007595cf9eb938","response_hash":"9845dd6caf69151d261a63a7adf8252c3c77eb3b1367d5262d3397763d9551c2","selected_tool":"getVisitorCenters","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":14,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":820,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":834}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-17","correct":true,"expected_tool":"getCampgrounds","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:32.040746Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"0f03f41f98f273d883dec82450dad538c3c137742abbc62415b9eb2cb96601cf","response_hash":"dbacef958e1b2332fdcfc5c3adefb038209a27943a314db328bc837084214f49","selected_tool":"getCampgrounds","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":820,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":838}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-18","correct":true,"expected_tool":"getEvents","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:33.388954Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"b6b6af081ee400f2b6608877aca2ef484b503d29945b0e0f647f8391c48b403c","response_hash":"9e679a030210c9e0860b104d5d4cca1e6e907b07c4b485641f0909dfee5b557e","selected_tool":"getEvents","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":17,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":816,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":833}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-19","correct":true,"expected_tool":"findParks","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:34.732030Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"48fcfc7d0dabc2d70722dc9c05523cd9592c3233aee977865e4001f799b8326e","response_hash":"9b161b6433a7c6a7e84a75da535b6cde17ebbf3d06b6099428527e9c9e8deafd","selected_tool":"findParks","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":826,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":844}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-20","correct":true,"expected_tool":"getParkDetails","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:35.820332Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"7479e8450898bed4aa41e4ca2cd6a00b6d2cc84f96f022fc404f9694c1cf8fbd","response_hash":"363c3d67e4c756ce66998cc6738a1dc1334bdb38588853cba39c356602ac0ec0","selected_tool":"getParkDetails","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":20,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":826,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":846}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-21","correct":true,"expected_tool":"getAlerts","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:37.020781Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"a0cae671264e5b27d7bb67f76b72edf99062aa63d7fb8d6d33190131641b7c3a","response_hash":"0120c9c7e173e4cd4e4ebfbef92d58924d522b486526a15c13f806b4b4d0c890","selected_tool":"getAlerts","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":13,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":825,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":838}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-22","correct":true,"expected_tool":"getVisitorCenters","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:38.052063Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"5772acc4f2c72de6065fd3e3478308bed0f589940b5b4e380c2bf45ddcba4cce","response_hash":"0d5f15236ad0e3a7777858ea4352ca91e842f35c7ae558a8a275fcb166aa4b68","selected_tool":"getVisitorCenters","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":14,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":820,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":834}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-23","correct":true,"expected_tool":"getCampgrounds","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:39.209695Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"ed7ac91ec5b2cf1c603e53ac8f4250b6787e2f9f1a60c0ab0f6b8271fbbbe8d3","response_hash":"0bd1683a2447d3845c8eb547ebc26fbc711946f0f1d6a0a39597b3783b48fad1","selected_tool":"getCampgrounds","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":18,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":820,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":838}},{"attempt_count":1,"canonical_id":"national-parks-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"national-parks-mcp-source-snapshot-24","correct":true,"expected_tool":"getEvents","frozen_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:40.347732Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"7efe51277c014513099df6b5eb23f2f07a914aeaac7cbfc7eeecc36c2fd6ec13","response_hash":"8e5ee074fba3322a465b3a40c3d84c50ab4f94ec41f0abb75b3a4c006bd115ae","selected_tool":"getEvents","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":13,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":816,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":829}}],"provider_valid_semantic_case_count":24,"semantic_coverage":1.0,"semantic_outcome_counts":{"SELECTED_VALID_TOOL":24},"strict_accuracy":1.0,"strict_correct_count":24,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"}},"promotion":{"adjusted_accuracy":null,"badge":"selection-tested","checks":{"S1":{"full_frozen_case_coverage":true,"immutable_identity_hashes":true,"min_case_count":true,"no_business_tool_execution":true,"no_post_result_mutation":true,"no_target_mcp_network_calls":true,"prediction_count_equals_case_count":true,"semantic_failures_not_retried":true,"valid_semantic_selection_count_equals_case_count":true,"zero_provider_infrastructure_failures":true},"S2":{"external_benchmark_source":false,"inherits_s1":true,"misses_adjudicated_when_present":true,"no_adjusted_accuracy":true,"strict_score_authoritative":true},"S3":{"fresh_within_30_days":false,"inherits_s2":false,"min_case_count":false,"min_distinct_frozen_measurements":false,"min_strict_accuracy":true,"min_wilson_95_lower":true,"same_runtime_schema_epoch":false},"S4":{"inherits_s3":false,"min_distinct_model_families_or_providers":false,"min_distinct_models":false,"min_total_case_count":false,"no_unresolved_gold_schema_contradiction_latest":true,"pooled_thresholds_met":false}},"level":"S1","maximum_level_for_this_case_source":"S1","policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","policy_version":"SELECTION_EVIDENCE_PROMOTION_POLICY_V1","reasons":[],"s2_eligible":false,"status":"S1 selection-tested","strict_accuracy_authoritative":true}},{"canonical_id":"metropolitan-museum-mcp-source-snapshot","subject_type":"VERSION_BOUND_SOURCE_SNAPSHOT","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","qualification_status":"FREEZE_READY","selection_evidence_status":"MEASURED","badge":"selection-tested","score":{"canonical_id":"metropolitan-museum-mcp-source-snapshot","frozen_case_count":24,"infrastructure_failure_count":0,"infrastructure_outcome_counts":{},"predictions":[{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-01","correct":true,"expected_tool":"list-departments","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:41.436317Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"4f8183e60689e9c06ab561282e92e45fab5f208b30b17560ab772dfb42310c48","response_hash":"caca44b8fe04c4e9b1a02a35f8964793dc7580cb1feef350c572bbab568e0f12","selected_tool":"list-departments","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":15,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":446,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":461}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-02","correct":true,"expected_tool":"search-museum-objects","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:42.924615Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"c653c6fb3de85d0e81e33eec82dc75e9d08ff5e450a7949f887ba21d20861042","response_hash":"78df09769dd0a59b0ab448a8b15d73c0d61b6406e5eb028d7b3279c4ed8c3fe8","selected_tool":"search-museum-objects","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":24,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":452,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":476}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-03","correct":true,"expected_tool":"get-museum-object","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:44.261007Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"4e7802bb42dd3b9c21d53ff7542e5d15ef8792b81e9def8c04af06b3161a5991","response_hash":"bc673f98449d868b152de558663d7b0ea5ca9207c2eebe7037f69c7960d37dc1","selected_tool":"get-museum-object","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":20,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":456,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":476}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-04","correct":true,"expected_tool":"list-departments","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:45.905444Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"5c5528e4eaac7ff0f7b8db9eba8118aa5145d54a82326f5d031a4d87f60826ef","response_hash":"cf50859558f78542423543a2df546ae3d4b847d765307e95ba584d868fff97d4","selected_tool":"list-departments","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":15,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":447,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":462}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-05","correct":true,"expected_tool":"search-museum-objects","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:47.468331Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"653cdae8d9e097ef3955944ceef217a6514c737cb6e77d8c35c952c872fb25da","response_hash":"4345744261d746167066bbbdc98ee77038e72d1c3b7c7473015aa491282fe533","selected_tool":"search-museum-objects","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":20,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":453,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":473}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-06","correct":true,"expected_tool":"get-museum-object","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:48.421064Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"12e25288d183f96711607be1246787c702562aba6263d28229fda55b2af2afbd","response_hash":"9f96a449677a0dd6c6078be9124484518ec0b8af239a72e402a0d6df39e3ae93","selected_tool":"get-museum-object","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":20,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":457,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":477}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-07","correct":true,"expected_tool":"list-departments","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:49.720520Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"78a347b4c13158c971358f802915e7cfd9ea9b812a40a3dae79688aaa1660f2b","response_hash":"7478a049423e2c4d1346344d38275821abe7c2e85aae3f3fdf9bc3f3f5844fc1","selected_tool":"list-departments","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":15,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":445,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":460}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-08","correct":true,"expected_tool":"search-museum-objects","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:51.363793Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"a0fea1b5679cf4b523c929fe72336113af0a7ea8c8b04718b4756854a35df1fd","response_hash":"32fe8a6c6b7b04e37c5e43288376aed32c0e355fc4033b653e384314813d1816","selected_tool":"search-museum-objects","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":20,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":451,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":471}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-09","correct":true,"expected_tool":"get-museum-object","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:52.534050Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"4b4df9ade52750c2cbda9ea9f89bfdb76481debd118f934d49a8f73236903d0b","response_hash":"d4eb23066761697995a3a53dc93e8c1538a283879e85d5f8c9fc434002639cc8","selected_tool":"get-museum-object","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":20,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":455,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":475}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-10","correct":true,"expected_tool":"list-departments","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:53.558372Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"7376e1de3bae4941761916847c0655477b44491e8ed4940d5a5907a54f748120","response_hash":"82ab785242b42920d69143246826f30242d4e8d7be55678f0fae472746b90bf8","selected_tool":"list-departments","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":15,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":445,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":460}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-11","correct":true,"expected_tool":"search-museum-objects","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:54.738115Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"6b0d08d78bab111d4408cf2d7bb7ef4402f54443ccf8368d6fcf23377e58da39","response_hash":"48733eb9abd25182002ee0982fc9ff9656f30b8cc92e94f7df8baa8198a1b0da","selected_tool":"search-museum-objects","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":20,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":451,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":471}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-12","correct":true,"expected_tool":"get-museum-object","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:55.911829Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"1d295d14ae7ae5ef70761e03afdb3b602da35b8a7810c14e7e296d98c6bb4eaf","response_hash":"19083367f1d51cb8e117c456f136dbc7fc22cbdba243325a3058f409332a4eec","selected_tool":"get-museum-object","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":20,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":455,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":475}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-13","correct":true,"expected_tool":"list-departments","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:57.002097Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"ae22fce119a88e61fa9fc9cd5486e2b9d04807e5d229020ac639602c58ccc27e","response_hash":"1cbfec99ae0280422b741d237e88533077af4d0af364407c754629f027eb6e91","selected_tool":"list-departments","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":15,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":445,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":460}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-14","correct":true,"expected_tool":"search-museum-objects","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:58.550779Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"55bfc6a768506793acf45e6f4366268897ec0db54bfcba982832a89a8fb0912c","response_hash":"f70aed74d31fa4fd58dd72e1fcd3aa810e1b692b953aac10963a75aa0ee8cf60","selected_tool":"search-museum-objects","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":20,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":451,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":471}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-15","correct":true,"expected_tool":"get-museum-object","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:22:59.917733Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"820fbd99331b9f98dbf92b9b0372cdd6cb9136df0a72cae4c20b09f9ba365fda","response_hash":"afdcc31993b97fa40db00667ec1b5501ccfe8d0ab1cf2ad972d595b982bc3def","selected_tool":"get-museum-object","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":20,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":455,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":475}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-16","correct":true,"expected_tool":"list-departments","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:23:01.022704Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"dccd2ed38089cfb3810d87f6bf9fe35c40b55d428181e5e8b338f3b6ecf177d0","response_hash":"6988f12100d0c6500f26f152e0201dfbaf5bd31f3b85a09339c12df32565c99a","selected_tool":"list-departments","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":15,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":444,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":459}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-17","correct":true,"expected_tool":"search-museum-objects","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:23:02.569355Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"09d93c5775094f010fa360a9ee368d7f198c2a842a6d8da60baa3cf9a1c46041","response_hash":"7865a6300f816adfcb21776400b483c80cc2e5092945ca34a1bd48cbe8d39886","selected_tool":"search-museum-objects","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":20,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":450,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":470}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-18","correct":true,"expected_tool":"get-museum-object","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:23:04.019046Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"1df10af80b64d4b336b46f6f89c347abc75a6fd14834b9fced4cc11e19b3060f","response_hash":"3c00b28bf223d6e1bb3362caf338d3be1d1f7ca2ca2f126ed301208f5e226c39","selected_tool":"get-museum-object","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":25,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":454,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":479}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-19","correct":true,"expected_tool":"list-departments","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:23:05.852615Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"271dd61722721be91122a8f4ead7f18fa664a1f88ea2201e9072064dfccdfac3","response_hash":"76f9addc3c72697eddcdb64abbb90f6addf3bba92fc7ff81512b5c2b6a754f46","selected_tool":"list-departments","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":15,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":446,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":461}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-20","correct":true,"expected_tool":"search-museum-objects","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:23:07.298727Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"0d1ab60e4ec94eb8294161fdffe7c8370389d20ac0540c38f25b3d23ed5924c9","response_hash":"e37d14cc6ab9d933ec3d9f208219c6fe5acf7ce84144e213e610ec2244756f40","selected_tool":"search-museum-objects","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":20,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":452,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":472}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-21","correct":true,"expected_tool":"get-museum-object","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:23:08.452020Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"ef702577bc9c02b6daccb6bf8fe34596ed9534fb4b44b9294e5d99b9feaab0ab","response_hash":"3b39f22d39883c2bb491f6ffcc83868b2cc9a444aee040027f73121ab92d7b57","selected_tool":"get-museum-object","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":25,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":456,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":481}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-22","correct":true,"expected_tool":"list-departments","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:23:09.536631Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"8b924e79a8dfe2fea9bc362d4bb86d9f17ccb132cde6b34f077ec060ef7de9ae","response_hash":"c811ae0a332b6f837e841c0281a4c6345e84615118a92ce2be5714e7fd221413","selected_tool":"list-departments","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":15,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":445,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":460}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-23","correct":true,"expected_tool":"search-museum-objects","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:23:10.833903Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"48b162b8fcbf09712149495719ae23090b336f824cf58475c79eca1d27fc46e2","response_hash":"4aa95f589dd92dcda7a140f22b10030759fb5c837ca7c8382ef68cc5402ecf7f","selected_tool":"search-museum-objects","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":21,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":451,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":472}},{"attempt_count":1,"canonical_id":"metropolitan-museum-mcp-source-snapshot","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","case_id":"metropolitan-museum-mcp-source-snapshot-24","correct":true,"expected_tool":"get-museum-object","frozen_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","infrastructure_error_classification":null,"measured_at":"2026-09-10T13:23:12.069736Z","model":"gpt-5.6-luna","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","provider_outcome":"PROVIDER_RESPONSE_RECEIVED","request_hash":"0755fefd00c115e60d5f5eb3bf15c72d2551dc6ab215fe3b95e5dd6def962115","response_hash":"823d475096f97ce04eeab4618ccddcf6810986306fc1869b3c65d6e6d6afa2ea","selected_tool":"get-museum-object","semantic_status":"SELECTED_VALID_TOOL","usage":{"completion_tokens":20,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":455,"prompt_tokens_details":{"audio_tokens":0,"cache_write_tokens":0,"cached_tokens":0},"total_tokens":475}}],"provider_valid_semantic_case_count":24,"semantic_coverage":1.0,"semantic_outcome_counts":{"SELECTED_VALID_TOOL":24},"strict_accuracy":1.0,"strict_correct_count":24,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"}},"promotion":{"adjusted_accuracy":null,"badge":"selection-tested","checks":{"S1":{"full_frozen_case_coverage":true,"immutable_identity_hashes":true,"min_case_count":true,"no_business_tool_execution":true,"no_post_result_mutation":true,"no_target_mcp_network_calls":true,"prediction_count_equals_case_count":true,"semantic_failures_not_retried":true,"valid_semantic_selection_count_equals_case_count":true,"zero_provider_infrastructure_failures":true},"S2":{"external_benchmark_source":false,"inherits_s1":true,"misses_adjudicated_when_present":true,"no_adjusted_accuracy":true,"strict_score_authoritative":true},"S3":{"fresh_within_30_days":false,"inherits_s2":false,"min_case_count":false,"min_distinct_frozen_measurements":false,"min_strict_accuracy":true,"min_wilson_95_lower":true,"same_runtime_schema_epoch":false},"S4":{"inherits_s3":false,"min_distinct_model_families_or_providers":false,"min_distinct_models":false,"min_total_case_count":false,"no_unresolved_gold_schema_contradiction_latest":true,"pooled_thresholds_met":false}},"level":"S1","maximum_level_for_this_case_source":"S1","policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","policy_version":"SELECTION_EVIDENCE_PROMOTION_POLICY_V1","reasons":[],"s2_eligible":false,"status":"S1 selection-tested","strict_accuracy_authoritative":true}}],"qualification_evidence":{"blocked_count":1,"candidate_count":11,"checked_at":"2026-09-10T10:15:52Z","claim_boundaries":{"model_results_consulted":false,"partial_subject_freeze_allowed":false,"prior_external_gold_v1_mutated":false,"qualification_is_measurement":false,"target_business_tool_execution":false},"freeze_ready_count":10,"prior_qualification_receipt":"validation/selection-evidence-expansion-v1/qualification-round-1.json","qualification_round":2,"rejected_count":0,"schema_version":"mcp-selection-lab.selection-evidence-expansion-v1.qualification.v1","subjects":[{"business_tool_execution":false,"canonical_id":"context7","checked_at":"2026-09-10T10:15:52Z","identity_mode":"LIVE_TOOLS_LIST","name":"Context7","protocol_operations":["tools/list","initialize_if_required","notifications/initialized_if_required"],"reason_codes":[],"schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","status":"FREEZE_READY","tool_count":2},{"business_tool_execution":false,"canonical_id":"exa-mcp","checked_at":"2026-09-10T10:15:52Z","identity_mode":"LIVE_TOOLS_LIST","name":"Exa MCP","protocol_operations":["tools/list","initialize_if_required","notifications/initialized_if_required"],"reason_codes":[],"schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","status":"FREEZE_READY","tool_count":2},{"business_tool_execution":false,"canonical_id":"cloudflare-docs-mcp","checked_at":"2026-09-10T10:15:52Z","identity_mode":"LIVE_TOOLS_LIST","name":"Cloudflare Docs MCP","protocol_operations":["tools/list","initialize_if_required","notifications/initialized_if_required"],"reason_codes":[],"schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","status":"FREEZE_READY","tool_count":2},{"business_tool_execution":false,"canonical_id":"deepwiki-mcp","checked_at":"2026-09-10T10:15:52Z","identity_mode":"LIVE_TOOLS_LIST","name":"DeepWiki MCP","protocol_operations":["tools/list","initialize_if_required","notifications/initialized_if_required"],"reason_codes":[],"schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","status":"FREEZE_READY","tool_count":3},{"business_tool_execution":false,"canonical_id":"gitmcp-docs","checked_at":"2026-09-10T10:15:52Z","identity_mode":"LIVE_TOOLS_LIST","name":"GitMCP Docs","protocol_operations":["tools/list","initialize_if_required","notifications/initialized_if_required"],"reason_codes":[],"schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","status":"FREEZE_READY","tool_count":5},{"business_tool_execution":false,"canonical_id":"uk-taxi-phv-mcp","checked_at":"2026-09-10T10:15:52Z","error_summary":"<urlopen error [Errno -2] Name or service not known>","error_type":"URLError","identity_mode":"LIVE_TOOLS_LIST","name":"UK Taxi / PHV MCP","reason_codes":["LIVE_TOOLS_LIST_UNAVAILABLE"],"schema_sha256":null,"status":"BLOCKED","tool_count":null},{"business_tool_execution":false,"canonical_id":"uk-premises-licence-mcp","checked_at":"2026-09-10T10:15:52Z","identity_mode":"LIVE_TOOLS_LIST","name":"UK Premises Licence MCP","protocol_operations":["tools/list","initialize_if_required","notifications/initialized_if_required"],"reason_codes":[],"schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","status":"FREEZE_READY","tool_count":8},{"business_tool_execution":false,"canonical_id":"nixos-mcp-source-snapshot","checked_at":"2026-09-10T10:15:52Z","current_live_status_claimed":false,"identity_mode":"VERSION_BOUND_SOURCE_SNAPSHOT","name":"NixOS MCP (source snapshot)","reason_codes":[],"schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","status":"FREEZE_READY","tool_count":18},{"business_tool_execution":false,"canonical_id":"dex-paprika-mcp-source-snapshot","checked_at":"2026-09-10T10:15:52Z","current_live_status_claimed":false,"identity_mode":"VERSION_BOUND_SOURCE_SNAPSHOT","name":"DEX Paprika MCP (source snapshot)","reason_codes":[],"schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","status":"FREEZE_READY","tool_count":11},{"business_tool_execution":false,"canonical_id":"national-parks-mcp-source-snapshot","checked_at":"2026-09-10T10:15:52Z","current_live_status_claimed":false,"identity_mode":"VERSION_BOUND_SOURCE_SNAPSHOT","name":"National Parks MCP (source snapshot)","reason_codes":[],"schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","status":"FREEZE_READY","tool_count":6},{"business_tool_execution":false,"canonical_id":"metropolitan-museum-mcp-source-snapshot","checked_at":"2026-09-10T10:15:52Z","current_live_status_claimed":false,"identity_mode":"VERSION_BOUND_SOURCE_SNAPSHOT","name":"Metropolitan Museum MCP (source snapshot)","reason_codes":[],"schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","status":"FREEZE_READY","tool_count":3}],"track":"SELECTION_EVIDENCE_EXPANSION_V1"},"blocked_subjects_remain_visible":true},"grade_separation":{"transport_grade":{"status":"SEPARATE","changed_by_this_artifact":false},"profile_evidence_grade":{"status":"SEPARATE","changed_by_this_artifact":false},"selection_evidence_grade":{"status":"MEASURED","source":"selection-evidence-expansion-v1"}},"version_bound_source_snapshot_disclosure":"VERSION_BOUND_SOURCE_SNAPSHOT subjects are bound to their recorded source receipt and do not claim current live MCP status.","blocked_taxonomy_disclosure":"The historical uk-taxi-phv-mcp qualification remains BLOCKED with LIVE_TOOLS_LIST_UNAVAILABLE; it is not silently replaced or reinterpreted.","artifact":{"available":true,"artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826"},"selection_evidence_expansion_v2":{"schema_version":"mcp-selection-lab.agent-trust-lab.selection-evidence-status-v2","track":"EXTERNAL_GOLD_EXPANSION_V2","status":"MEASURED","benchmark_version":"external-gold-expansion-v2-batch1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","badge":"external-selection-tested","promotion_level":"S2","promotion_status":"S2 external-selection-tested","case_count":24,"strict_correct":23,"strict_accuracy":0.9583333333333334,"wilson_95":{"lower":0.7975819352134406,"upper":0.9926065438344942},"provider_valid_count":24,"provider_valid_coverage":1.0,"semantic_coverage":1.0,"infrastructure_failure_count":0,"s2_external_selection_tested":true,"subject_level_promotions":[],"subject_level_promotion_note":"Policy V1 evaluates this 24-case benchmark at benchmark level. Subject partitions are descriptive only and do not receive independent S2 badges.","subjects":[{"canonical_id":"datagouv-mcp","qualification_status":"FREEZE_READY","measurement_status":"MEASURED","frozen_cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.6096657120978346,"upper":1.0},"coverage":1.0,"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"microsoft-learn","qualification_status":"FREEZE_READY","measurement_status":"MEASURED","frozen_cases":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"coverage":1.0,"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"microsoft-release-communications","qualification_status":"FREEZE_READY","measurement_status":"MEASURED","frozen_cases":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"coverage":1.0,"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"exa-mcp","qualification_status":"FREEZE_READY","measurement_status":"MEASURED","frozen_cases":4,"provider_valid_cases":4,"infrastructure_failures":0,"strict_correct":4,"strict_accuracy":1.0,"wilson_95":{"lower":0.5101091635454027,"upper":1.0},"coverage":1.0,"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"deepwiki-mcp","qualification_status":"FREEZE_READY","measurement_status":"MEASURED","frozen_cases":4,"provider_valid_cases":4,"infrastructure_failures":0,"strict_correct":3,"strict_accuracy":0.75,"wilson_95":{"lower":0.30064184258240184,"upper":0.9544127391902995},"coverage":1.0,"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"}],"frozen_measurement_identity":{"freeze_input_commit":"c614ae51750d12ce113b5e3d51429f52e529f08c","final_freeze_receipt_commit":"b691170","measurement_commit":"b9f358b1a65aba348fa1c0e625eac6da1f31b651","policy_evaluation_commit":"3e560e5767f181831c65569b893ca789cd3bdff0","case_freeze_sha256":"17777d2c3983cd444353b8d69dcc98836a1909ee6fde6f5c8cf18cf9b059427e","model_lock_sha256":"c4330165deb3d8587e224a2ca1b3d886db60801ca436710a0edace55525923d2","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ab085cedf13eef5724f4e160b0f49f8402d62bc5b10203de0f8d81e41f2a017e","measurement_artifact_sha256":"01243422aa946296f235810aaee6f065ec21d9e78a01017ee8161e5e61a8b5e8","policy_evaluation_sha256":"4e710a9c5205660cc07603d2fd7538aa063eda4c2574c29308f9f255e4706f82"},"semantic_miss_count":1,"semantic_miss_adjudication":"validation/external-gold-expansion-v2/measurement-v2-batch1/semantic-miss-adjudication-v2-batch1.json","strict_score_authoritative":true,"adjusted_accuracy_used":false,"target_mcp_business_tools_called":false,"transport_grade_changed":false,"profile_evidence_grade_changed":false,"reserves_used_post_measurement":false},"selection_evidence_expansion_v2_batch2":{"schema_version":"mcp-selection-lab.agent-trust-lab.selection-evidence-status-v2","track":"EXTERNAL_GOLD_EXPANSION_V2","status":"MEASURED","benchmark":"External Gold Expansion V2 Batch 2","benchmark_version":"external-gold-expansion-v2-batch2","batch_characterization":"HARD_SELECTION_BOUNDARY_BATCH","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","badge":"external-selection-tested","promotion_level":"S2","promotion_status":"S2 external-selection-tested","case_count":26,"strict_correct":24,"strict_accuracy":0.9230769230769231,"wilson_95":{"lower":0.7585844622837095,"upper":0.9786449115277238},"provider_valid_count":26,"provider_valid_coverage":1.0,"semantic_coverage":1.0,"infrastructure_failure_count":0,"difficulty_metrics":{"H2":{"frozen_cases":17,"infrastructure_failures":0,"provider_coverage":1.0,"provider_valid_cases":17,"semantic_coverage":1.0,"semantic_valid_selection_cases":17,"strict_accuracy":1.0,"strict_correct":17,"wilson_95":{"lower":0.8156818649911483,"upper":1.0}},"H3":{"frozen_cases":9,"infrastructure_failures":0,"provider_coverage":1.0,"provider_valid_cases":9,"semantic_coverage":1.0,"semantic_valid_selection_cases":9,"strict_accuracy":0.7777777777777778,"strict_correct":7,"wilson_95":{"lower":0.45258896910698887,"upper":0.9367748928821533}}},"boundary_family_metrics":{"additional_frozen_directional_labels":{"AZURE_COLLECTION_VS_SPECIFIC_ITEM":{"case_count":1,"correct_count":1,"interpretation":"descriptive directional label; retained separately because the frozen register does not contain this label","strict_accuracy":1.0},"CODE_SAMPLE_VS_FULL_DOC":{"case_count":1,"correct_count":1,"interpretation":"descriptive directional label; retained separately because the frozen register does not contain this label","strict_accuracy":1.0},"CODE_VS_CONCEPTUAL_DOCS":{"case_count":1,"correct_count":1,"interpretation":"descriptive directional label; retained separately because the frozen register does not contain this label","strict_accuracy":1.0},"CONTENTS_VS_STRUCTURE":{"case_count":1,"correct_count":1,"interpretation":"descriptive directional label; retained separately because the frozen register does not contain this label","strict_accuracy":1.0},"DATASET_VS_RESOURCE_LIST":{"case_count":1,"correct_count":0,"interpretation":"descriptive directional label; retained separately because the frozen register does not contain this label","strict_accuracy":0.0},"DISCOVERY_VS_KNOWN_PAGE":{"case_count":1,"correct_count":1,"interpretation":"descriptive directional label; retained separately because the frozen register does not contain this label","strict_accuracy":1.0},"DOCUMENTED_BEHAVIOR_VS_IMPLEMENTATION":{"case_count":1,"correct_count":0,"interpretation":"descriptive directional label; retained separately because the frozen register does not contain this label","strict_accuracy":0.0},"M365_SPECIFIC_VS_AZURE_SPECIFIC":{"case_count":1,"correct_count":1,"interpretation":"descriptive directional label; retained separately because the frozen register does not contain this label","strict_accuracy":1.0},"M365_VS_AZURE":{"case_count":1,"correct_count":1,"interpretation":"descriptive directional label; retained separately because the frozen register does not contain this label","strict_accuracy":1.0},"OVERVIEW_VS_TARGETED_DOCS":{"case_count":1,"correct_count":1,"interpretation":"descriptive directional label; retained separately because the frozen register does not contain this label","strict_accuracy":1.0},"QUESTION_VS_CONTENTS":{"case_count":1,"correct_count":1,"interpretation":"descriptive directional label; retained separately because the frozen register does not contain this label","strict_accuracy":1.0},"QUESTION_VS_DOCUMENT_CONTENTS":{"case_count":1,"correct_count":1,"interpretation":"descriptive directional label; retained separately because the frozen register does not contain this label","strict_accuracy":1.0},"RESOURCE_DETAIL_VS_QUERY":{"case_count":1,"correct_count":1,"interpretation":"descriptive directional label; retained separately because the frozen register does not contain this label","strict_accuracy":1.0},"STRUCTURE_VS_QUESTION":{"case_count":1,"correct_count":1,"interpretation":"descriptive directional label; retained separately because the frozen register does not contain this label","strict_accuracy":1.0},"TARGETED_DOCS_VS_OVERVIEW":{"case_count":1,"correct_count":1,"interpretation":"descriptive directional label; retained separately because the frozen register does not contain this label","strict_accuracy":1.0}},"post_measurement_merging_performed":false,"registered_families":{"AZURE_VS_M365":{"case_count":1,"correct_count":1,"interpretation":"descriptive; small cells are not statistically conclusive","strict_accuracy":1.0},"COLLECTION_VS_SPECIFIC_ITEM":{"case_count":1,"correct_count":1,"interpretation":"descriptive; small cells are not statistically conclusive","strict_accuracy":1.0},"CONTENTS_VS_QUESTION":{"case_count":0,"correct_count":0,"interpretation":"descriptive; small cells are not statistically conclusive","strict_accuracy":null},"DATASET_METADATA_VS_RESOURCES":{"case_count":1,"correct_count":1,"interpretation":"descriptive; small cells are not statistically conclusive","strict_accuracy":1.0},"DATASET_VS_DATASERVICE":{"case_count":1,"correct_count":1,"interpretation":"descriptive; small cells are not statistically conclusive","strict_accuracy":1.0},"DOCS_VS_CODE":{"case_count":2,"correct_count":2,"interpretation":"descriptive; small cells are not statistically conclusive","strict_accuracy":1.0},"ENTITY_VS_DATASET":{"case_count":1,"correct_count":1,"interpretation":"descriptive; small cells are not statistically conclusive","strict_accuracy":1.0},"KNOWN_PAGE_VS_DISCOVERY":{"case_count":1,"correct_count":1,"interpretation":"descriptive; small cells are not statistically conclusive","strict_accuracy":1.0},"LIBRARY_RESOLUTION_VS_REPOSITORY_DOCS":{"case_count":1,"correct_count":1,"interpretation":"descriptive; small cells are not statistically conclusive","strict_accuracy":1.0},"RESOURCE_METADATA_VS_QUERY":{"case_count":1,"correct_count":1,"interpretation":"descriptive; small cells are not statistically conclusive","strict_accuracy":1.0},"STRUCTURE_VS_CONTENTS":{"case_count":1,"correct_count":1,"interpretation":"descriptive; small cells are not statistically conclusive","strict_accuracy":1.0}},"registered_family_count":11},"semantic_miss_count":2,"semantic_misses":[{"case_id":"datagouv-hard-004","classification":"SEMANTIC_MISS_GOLD_REMAINS_VALID","competing_plausible_tools":["get_dataset_info","get_resource_info"],"difficulty":"H3","expected_tool":"list_dataset_resources","reason":"The model selected get_dataset_info although the request asks for the complete inventory of attached files before choosing one; the frozen external catalog evidence supports list_dataset_resources.","response_hash":"63c2d98ec50c7ae0bdb4583fae2e98f55ef4c99e699e87e21cdc2b96b563f5d5","selected_tool":"get_dataset_info","selection_boundary":"DATASET_VS_RESOURCE_LIST","subject_id":"datagouv-mcp"},{"case_id":"gitmcp-hard-005","classification":"SEMANTIC_MISS_GOLD_REMAINS_VALID","competing_plausible_tools":["search_generic_code"],"difficulty":"H3","expected_tool":"search_generic_documentation","reason":"The model selected fetch_generic_documentation although the request asks for a prose explanation of documented fallback behavior and explicitly excludes implementation location; the frozen external README evidence supports search_generic_documentation.","response_hash":"8897806f7ffa3cbfe6c0b54e450d39df75351214b4401aac1fc8ca6fc07b8df3","selected_tool":"fetch_generic_documentation","selection_boundary":"DOCUMENTED_BEHAVIOR_VS_IMPLEMENTATION","subject_id":"gitmcp-docs"}],"subject_level_promotions":[],"subject_level_promotion_note":"Policy V1 defines the S1/S2 gate at benchmark level. The five subject partitions have 6, 5, 5, 5, and 5 cases and are descriptive only; none receives an individual S2 badge.","subjects":[{"canonical_id":"datagouv-mcp","frozen_cases":6,"strict_correct":5,"strict_accuracy":0.8333333333333334,"wilson_95":{"lower":0.43649717781352987,"upper":0.9699466302516935},"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"coverage":1.0,"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"gitmcp-docs","frozen_cases":5,"strict_correct":4,"strict_accuracy":0.8,"wilson_95":{"lower":0.37553462976252533,"upper":0.9637758913675698},"provider_valid_cases":5,"infrastructure_failures":0,"semantic_coverage":1.0,"coverage":1.0,"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"microsoft-release-communications","frozen_cases":5,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"provider_valid_cases":5,"infrastructure_failures":0,"semantic_coverage":1.0,"coverage":1.0,"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"microsoft-learn","frozen_cases":5,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"provider_valid_cases":5,"infrastructure_failures":0,"semantic_coverage":1.0,"coverage":1.0,"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"deepwiki-mcp","frozen_cases":5,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"provider_valid_cases":5,"infrastructure_failures":0,"semantic_coverage":1.0,"coverage":1.0,"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"}],"frozen_measurement_identity":{"freeze_input_commit":"61f6511c41562c24019cd441b37bc6ee07283951","final_freeze_receipt_commit":"d3c55d2f96a4e45bf2577225ea16929877f2ded1","measurement_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","policy_evaluation_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","case_freeze_sha256":"ce21fc085bd9a0185c9e37d0d795221f286397cbc477ee941e876be24f7ba1b4","model_lock_sha256":"153647f94eed0ee3d3069229545846eef864fd43fa821f2d311dc3fb5d265ba9","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"8c4364ab598241ddbe4adb16453ba2e12b0917a1d1852caedb07884d1a54570b","measurement_artifact_sha256":"dd13aec8a90b2b5490c934c9ad9650c37733c65f18d824e2ae6066202176ced9","policy_evaluation_sha256":"736f154a5d5c3b8f0d77cd58fa0644ca97ad94c7a6b46af98d394c1dc1578f9a"},"strict_score_authoritative":true,"adjusted_accuracy_used":false,"target_mcp_business_tools_called":false,"transport_grade_changed":false,"profile_evidence_grade_changed":false,"reserves_used_post_measurement":false,"batch_1_untouched":true},"selection_evidence_expansion_v2_batch3":{"schema_version":"mcp-selection-lab.agent-trust-lab.selection-evidence-status-v3","track":"EXTERNAL_GOLD_EXPANSION_V2","status":"MEASURED","benchmark":"External Gold Expansion V2 Batch 3","benchmark_version":"external-gold-expansion-v2-batch3","batch_characterization":"BOUNDARY_RISK_VALIDATION_BATCH","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","subject_level_promotions":[],"case_count":36,"strict_correct":34,"strict_score":"34/36","strict_accuracy":0.944444,"strict_accuracy_display":"94.4444%","wilson_95":{"lower":0.818553,"upper":0.98463},"wilson_95_display":"81.8553%–98.4630%","provider_valid_count":36,"provider_valid_coverage":1.0,"semantic_coverage":1.0,"infrastructure_failure_count":0,"risk_bucket_metrics":{"HIGH":{"cases":15,"provider_valid_cases":15,"infrastructure_failures":0,"semantic_valid_selection_cases":15,"semantic_coverage":1.0,"correct":13,"misses":2,"strict_accuracy":0.866667,"miss_rate":0.133333,"wilson_95":{"lower":0.62118,"upper":0.962639},"provider_coverage":1.0},"MEDIUM":{"cases":12,"provider_valid_cases":12,"infrastructure_failures":0,"semantic_valid_selection_cases":12,"semantic_coverage":1.0,"correct":12,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.757506,"upper":1.0},"provider_coverage":1.0},"LOW":{"cases":9,"provider_valid_cases":9,"infrastructure_failures":0,"semantic_valid_selection_cases":9,"semantic_coverage":1.0,"correct":9,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.700855,"upper":1.0},"provider_coverage":1.0}},"risk_score_ordering":{"method":"Spearman rank correlation between frozen risk score and binary semantic failure; exploratory only.","spearman_rho":0.299238,"high_vs_low_miss_rate_difference":0.133333,"high_vs_low_risk_ratio":null,"interpretation":"HIGH had two misses while MEDIUM and LOW had none; the zero denominator and two total misses make the association exploratory, not statistically conclusive."},"scanner_evaluation":{"scanner_id":"BOUNDARY_RISK_SCAN_V1","decision":"PROMISING_BUT_MORE_DATA_REQUIRED","decision_reason":"Both observed Batch 3 misses were in HIGH, while MEDIUM and LOW had no misses, but the total of two misses and concentration in one subject is too small for validation or production-grade prediction claims.","statistical_claim":false},"taxonomy_signal_metrics":{"PAIRWISE_DESCRIPTION_LEXICAL_OVERLAP":{"exposure_count":13,"miss_count":1,"miss_rate":0.076923,"case_ids":["b3-dg-003","b3-gm-002","b3-gm-003","b3-gm-004","b3-gm-006","b3-dw-001","b3-dw-002","b3-dw-003","b3-dw-005","b3-dw-006","b3-mrc-001","b3-mrc-002","b3-mrc-006"],"interpretation":"descriptive exposure analysis; not a causal estimate"},"REQUIRED_INPUT_SCHEMA_OVERLAP":{"exposure_count":15,"miss_count":2,"miss_rate":0.133333,"case_ids":["b3-dg-003","b3-dg-004","b3-dg-005","b3-dg-006","b3-gm-001","b3-gm-002","b3-gm-003","b3-gm-004","b3-gm-005","b3-gm-006","b3-dw-001","b3-dw-002","b3-mrc-001","b3-mrc-002","b3-mrc-006"],"interpretation":"descriptive exposure analysis; not a causal estimate"},"GENERIC_SPECIFIC_DESCRIPTION_IMBALANCE":{"exposure_count":5,"miss_count":0,"miss_rate":0.0,"case_ids":["b3-dg-001","b3-dg-002","b3-dg-003","b3-mrc-001","b3-mrc-002"],"interpretation":"descriptive exposure analysis; not a causal estimate"},"OUTPUT_GRANULARITY_BOUNDARY_ABSENCE":{"exposure_count":12,"miss_count":2,"miss_rate":0.166667,"case_ids":["b3-gm-001","b3-gm-002","b3-gm-003","b3-gm-004","b3-gm-005","b3-gm-006","b3-dw-001","b3-dw-002","b3-dw-003","b3-dw-004","b3-dw-005","b3-dw-006"],"interpretation":"descriptive exposure analysis; not a causal estimate"},"BOUNDARY_LANGUAGE_COVERAGE":{"exposure_count":29,"miss_count":2,"miss_rate":0.068966,"case_ids":["b3-dg-001","b3-dg-002","b3-dg-003","b3-dg-004","b3-dg-005","b3-dg-006","b3-gm-001","b3-gm-002","b3-gm-003","b3-gm-004","b3-gm-005","b3-gm-006","b3-dw-001","b3-dw-002","b3-dw-003","b3-dw-004","b3-dw-005","b3-dw-006","b3-cf-001","b3-cf-002","b3-cf-003","b3-cf-004","b3-cf-005","b3-cf-006","b3-mrc-001","b3-mrc-002","b3-mrc-003","b3-mrc-004","b3-mrc-005"],"interpretation":"descriptive exposure analysis; not a causal estimate"}},"taxonomy_validation":[{"stable_id":"RETRIEVAL_GRANULARITY_CONFUSION","status":"INSUFFICIENT_EVIDENCE","batch3_evidence":"No Batch 3 miss in the data.gouv resource/list cases; prior Batch 2 observation remains unchanged.","confidence":"low"},{"stable_id":"DEFAULT_GENERAL_TOOL_PREFERENCE","status":"SUPPORTED","batch3_evidence":"Both Batch 3 misses selected GitMCP's broad fetch tool over a more specific neighboring operation.","confidence":"low; two misses in one subject"},{"stable_id":"DESCRIPTION_ASYMMETRY","status":"PARTIALLY_SUPPORTED","batch3_evidence":"Both misses involved the short generic fetch description competing with more specific retrieval modes; the scanner exposed incomplete boundary language.","confidence":"low"},{"stable_id":"QUESTION_VS_STRUCTURE_CONFUSION","status":"INSUFFICIENT_EVIDENCE","batch3_evidence":"DeepWiki Batch 3 cases were all correct; Batch 1 miss remains preserved and unchanged.","confidence":"low"},{"stable_id":"SCHEMA_SEMANTIC_GAP","status":"INSUFFICIENT_EVIDENCE","batch3_evidence":"No controlled schema rewrite or parameter-ablation experiment was performed.","confidence":"low"}],"semantic_miss_count":2,"semantic_misses":[{"case_id":"b3-gm-005","subject_id":"gitmcp-docs","frozen_prompt":"Read the external documentation URL linked from the GitMCP repository documentation and summarize that page.","expected_tool":"fetch_generic_url_content","selected_tool":"fetch_generic_documentation","competing_plausible_tools":["fetch_generic_documentation"],"boundary_family":"DOCUMENTATION_VS_REFERENCED_URL","risk_bucket":"HIGH","risk_score":65.5556,"frozen_external_source_id":"gitmcp-readme-c487a29-2026-05-08","frozen_source_url":"https://github.com/idosal/git-mcp/blob/c487a29895dcfcb5b672247e646426a56e2051c1/README.md","frozen_source_pointer":"README.md lines 365-371; referenced URL retrieval workflow","provider_response_hash":"74d2208690baab941886bb7f8147f0b2454222194f5cbde02252d3a3a982e492","review_conclusion":"GOLD_REMAINS_VALID; semantic miss preserved in strict score. The public source supports the requested neighboring operation, and the selected tool is a plausible but incorrect neighboring retrieval mode.","gold_label_changed":false,"original_prediction_preserved":true,"semantic_retry":false},{"case_id":"b3-gm-006","subject_id":"gitmcp-docs","frozen_prompt":"How does GitMCP decide which repository documents to use when llms.txt is absent? Explain the documented fallback behavior; I am not asking where the code implements it.","expected_tool":"search_generic_documentation","selected_tool":"fetch_generic_documentation","competing_plausible_tools":["search_generic_code"],"boundary_family":"DOCUMENTED_BEHAVIOR_VS_IMPLEMENTATION","risk_bucket":"HIGH","risk_score":83.1884,"frozen_external_source_id":"gitmcp-readme-c487a29-2026-05-08","frozen_source_url":"https://github.com/idosal/git-mcp/blob/c487a29895dcfcb5b672247e646426a56e2051c1/README.md","frozen_source_pointer":"README.md lines 365-371 and 463-465; documentation fallback discussion","provider_response_hash":"6676c39505e520f919e7f7747aa86481a74bb947a846580fb0dfc0247411f73c","review_conclusion":"GOLD_REMAINS_VALID; semantic miss preserved in strict score. The public source supports the requested neighboring operation, and the selected tool is a plausible but incorrect neighboring retrieval mode.","gold_label_changed":false,"original_prediction_preserved":true,"semantic_retry":false}],"subjects":[{"canonical_id":"datagouv-mcp","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_valid_selection_cases":6,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.609666,"upper":1.0},"provider_coverage":1.0,"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"gitmcp-docs","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_valid_selection_cases":6,"semantic_coverage":1.0,"correct":4,"misses":2,"strict_accuracy":0.666667,"miss_rate":0.333333,"wilson_95":{"lower":0.299993,"upper":0.903229},"provider_coverage":1.0,"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"deepwiki-mcp","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_valid_selection_cases":6,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.609666,"upper":1.0},"provider_coverage":1.0,"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"cloudflare-docs-mcp","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_valid_selection_cases":6,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.609666,"upper":1.0},"provider_coverage":1.0,"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"exa-mcp","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_valid_selection_cases":6,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.609666,"upper":1.0},"provider_coverage":1.0,"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"microsoft-release-communications","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_valid_selection_cases":6,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.609666,"upper":1.0},"provider_coverage":1.0,"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"}],"frozen_measurement_identity":{"freeze_input_commit":"8450ed889756784a12ca4ebb8d84974d73b31778","final_freeze_receipt_commit":"a309bda7e8dd2867ccc7894dc7d42efbfc48395e","measurement_commit":"b81cc816158f7c0fbe7f8ac0a6fcf2b3f38f4c86","immutable_measurement_receipt_commit":"9d8290160849a5124d1896fc318d53470585ef41","case_freeze_sha256":"c1595743f48fe8ae6b3f70f2ea401731052b3bd7614297feabaa8172ce190ae5","model_lock_sha256":"48b5ed9c98f16cc2b3b76bc0da2e7ec8072ecd89ba1914683181b1c20074a3a8","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"c5cfe765f6b72489220cc73f689acb1c1c2c9284ffd8d093de0e597b645f6fbf","measurement_artifact_sha256":"7dcab9d4b71e7de59094c07cbca830ebf674b601ac6de024db0d3e677e7c0067","policy_evaluation_sha256":"3ea5b62044b4da6e92812831ebb00205970849accffe763a270248a24a0adef4"},"cross_batch_separation":{"batch1_untouched":true,"batch2_untouched":true,"expansion_v1_untouched":true,"external_gold_v1_untouched":true,"no_combined_headline_score":true},"strict_score_authoritative":true,"adjusted_accuracy_used":false,"target_mcp_business_tools_called":false,"transport_grade_changed":false,"profile_evidence_grade_changed":false,"scanner_production_validity_claimed":false},"selection_evidence_expansion_v2_batch4":{"schema_version":"mcp-selection-lab.agent-trust-lab.selection-evidence-status-v5","track":"EXTERNAL_GOLD_EXPANSION_V2","status":"MEASURED","benchmark":"External Gold Expansion V2 Batch 4","benchmark_version":"external-gold-expansion-v2-batch4","batch_characterization":"CROSS_SUBJECT_BOUNDARY_RISK_VALIDATION_BATCH","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","subject_level_promotions":[],"case_count":48,"strict_correct":46,"strict_score":"46/48","strict_accuracy":0.958333,"strict_accuracy_display":"95.8333%","wilson_95":{"lower":0.860243,"upper":0.988498,"lower_percent":86.0243,"upper_percent":98.8498},"wilson_95_display":"86.0243%–98.8498%","provider_valid_count":48,"provider_valid_coverage":1.0,"semantic_coverage":1.0,"infrastructure_failure_count":0,"risk_bucket_metrics":{"HIGH":{"cases":19,"provider_valid_cases":19,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":17,"misses":2,"strict_accuracy":0.894737,"wilson_95":{"lower":0.686059,"upper":0.970641,"lower_percent":68.6059,"upper_percent":97.0641}},"MEDIUM":{"cases":12,"provider_valid_cases":12,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":12,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.757506,"upper":1.0,"lower_percent":75.7506,"upper_percent":100.0}},"LOW":{"cases":17,"provider_valid_cases":17,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":17,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.815682,"upper":1.0,"lower_percent":81.5682,"upper_percent":100.0}}},"risk_comparison":{"high":{"cases":19,"provider_valid_cases":19,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":17,"misses":2,"strict_accuracy":0.894737,"wilson_95":{"lower":0.686059,"upper":0.970641,"lower_percent":68.6059,"upper_percent":97.0641},"miss_rate":0.10526315789473684},"non_high":{"cases":29,"provider_valid_cases":29,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":29,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.88303,"upper":1.0,"lower_percent":88.303,"upper_percent":100.0},"miss_rate":0.0},"miss_rate_difference_high_minus_non_high":0.10526315789473684,"risk_ratio":null,"interpretation":"EXPLORATORY; zero non-HIGH misses prevents a stable finite risk ratio and the number of failures is small."},"scanner_status":"PROMISING_CROSS_SUBJECT_SIGNAL_MORE_DATA_REQUIRED","scanner_validation":{"decision":"PROMISING_CROSS_SUBJECT_SIGNAL_MORE_DATA_REQUIRED","cross_subject_validated":false,"production_validity_claimed":false,"disclosure":"Batch 4 was a promising exploratory observation only; it did not validate BOUNDARY_RISK_SCAN_V1."},"failure_dispersion":{"subjects_with_miss":["deepwiki-mcp","nixos-mcp"],"subject_count_with_miss":2,"boundary_family_count_with_miss":2},"leave_one_subject_out":{"all_subjects_directionally_consistent":true,"results_artifact":"leave-one-subject-out-evaluation-v1.json"},"boundary_family_sensitivity":{"benchmark_version":"external-gold-expansion-v2-batch4","metrics":{"ALERTS_VS_PARK_DISCOVERY":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"AZURE_COLLECTION_VS_AZURE_ITEM":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"AZURE_ITEM_VS_AZURE_COLLECTION":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"AZURE_VS_M365_COLLECTION":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"CAMPGROUNDS_VS_VISITOR_CENTERS":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"CODE_SAMPLE_VS_FULL_DOC":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"CODE_VS_CONCEPTUAL_DOCS":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"CONTENTS_VS_QUESTION":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":0,"misses":1,"strict_accuracy":0.0,"wilson_95":{"lower":0.0,"upper":0.793451,"lower_percent":0.0,"upper_percent":79.3451},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"CONTENTS_VS_STRUCTURE":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"CURRENT_DISCOVERY_VS_KNOWN_URL":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"DARWIN_VS_HOME_MANAGER_SEARCH":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"DATASET_METADATA_VS_USAGE_METRICS":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"DATASET_RECORD_VS_RESOURCE_COLLECTION":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"DATASET_VS_DATASERVICE":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"DATASET_VS_RESOURCE_COLLECTION":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"DEDICATED_GUIDE_VS_GENERAL_SEARCH":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"DISCOVERY_VS_KNOWN_PAGE":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"DISCOVERY_VS_PAGE_READ":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"DOCS_VS_CODE":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"EVENTS_VS_ALERTS":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"FLAKES_VS_PACKAGES_OPTIONS":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"GENERAL_DOCS_VS_MIGRATION_GUIDE":{"cases":4,"provider_valid_cases":4,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":4,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.510109,"upper":1.0,"lower_percent":51.0109,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"HOME_MANAGER_OPTION_VS_DARWIN_OPTION":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"HOME_MANAGER_VS_DARWIN_SEARCH":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":0,"misses":1,"strict_accuracy":0.0,"wilson_95":{"lower":0.0,"upper":0.793451,"lower_percent":0.0,"upper_percent":79.3451},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"KNOWN_PAGE_VS_DISCOVERY":{"cases":2,"provider_valid_cases":2,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":2,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0,"lower_percent":34.238,"upper_percent":100.0},"subject_count":2,"interpretation":"DESCRIPTIVE_ONLY"},"KNOWN_PAGE_VS_SEARCH":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"KNOWN_URL_VS_OPEN_SEARCH":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"KNOWN_URL_VS_SEARCH":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"M365_ITEM_VS_AZURE_ITEM":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"M365_ITEM_VS_M365_COLLECTION":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"M365_VS_AZURE_COLLECTION":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"MIGRATION_GUIDE_VS_GENERAL_SEARCH":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"NIXOS_SEARCH_VS_INFO":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"OPTION_CATEGORIES_VS_PREFIX_ENTRIES":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"PARK_DETAIL_VS_DISCOVERY":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"PARK_DISCOVERY_VS_ALERTS":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"QUESTION_VS_CONTENTS":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"QUESTION_VS_STRUCTURE":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"RESOURCE_METADATA_VS_QUERY":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"SERVICE_METADATA_VS_OPENAPI":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"STRUCTURE_VS_CONTENTS":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"STRUCTURE_VS_QUESTION":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"VISITOR_CENTERS_VS_CAMPGROUNDS":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"WEB_DISCOVERY_VS_KNOWN_PAGE":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"}},"recurring_exact_boundary_labels_across_subjects":{"KNOWN_PAGE_VS_DISCOVERY":["exa-mcp","microsoft-learn"]},"cross_subject_recurrence_note":"Only exact frozen boundary labels recurring across subjects are listed; no labels were merged after measurement."},"semantic_miss_count":2,"semantic_misses":[{"case_id":"b4-dw-005","subject_id":"deepwiki-mcp","frozen_prompt":"Read the repository documentation pages covering its release process and return their contents verbatim enough for me to inspect the procedure.","expected_tool":"read_wiki_contents","selected_tool":"read_wiki_structure","plausible_competing_tools":["ask_question"],"selection_boundary":"CONTENTS_VS_QUESTION","difficulty":"H2","risk_bucket":"HIGH","risk_score":51.6667,"risk_factors":["DESCRIPTION_LEXICAL_OVERLAP","SHARED_REQUIRED_INPUTS","MISSING_OUTPUT_CONTRAST","INCOMPLETE_BOUNDARY_LANGUAGE"],"external_source_id":"deepwiki-aws-guide-2026-09-10","source_url":"https://docs.devin.ai/work-with-devin/deepwiki-mcp","source_version":"public guide and repository evidence captured 2026-09-10","source_excerpt_reference":"DeepWiki MCP guide; page contents request","gold_label_changed":false,"original_prediction_preserved":true,"gold_validity":"VALID_GOLD_RETAINED","adjudication_reason":"The user wants source documentation contents, not an answer generated from them."},{"case_id":"b4-nx-001","subject_id":"nixos-mcp","frozen_prompt":"Find the Home Manager option documentation for enabling the Git program module. I am configuring a user home, not a macOS system.","expected_tool":"home_manager_search","selected_tool":"home_manager_info","plausible_competing_tools":["darwin_search"],"selection_boundary":"HOME_MANAGER_VS_DARWIN_SEARCH","difficulty":"H3","risk_bucket":"HIGH","risk_score":70.0,"risk_factors":["DESCRIPTION_LEXICAL_OVERLAP","IDENTICAL_REQUIRED_INPUTS_OR_BOTH_NULL","OVERLAPPING_ACTION_VERBS","INCOMPLETE_BOUNDARY_LANGUAGE"],"external_source_id":"home-manager-manual-2026-09-10","source_url":"https://nix-community.github.io/home-manager/options.html","source_version":"Home Manager options manual accessed 2026-09-10","source_excerpt_reference":"Home Manager's published option index distinguishes user-home modules from nix-darwin system options.","gold_label_changed":false,"original_prediction_preserved":true,"gold_validity":"VALID_GOLD_RETAINED","adjudication_reason":"The requested platform and configuration domain are Home Manager, not nix-darwin."}],"taxonomy_validation":{"RETRIEVAL_GRANULARITY_CONFUSION":{"status":"PARTIALLY_SUPPORTED","batch4_evidence":["b4-dw-005 contents-versus-structure miss","b4-nx-001 search-versus-detail miss"],"confidence":"LOW_TO_MEDIUM"},"DEFAULT_GENERAL_TOOL_PREFERENCE":{"status":"SUPPORTED_SINGLE_SUBJECT_ONLY","batch4_evidence":[],"confidence":"LOW"},"QUESTION_VS_STRUCTURE_CONFUSION":{"status":"INSUFFICIENT_EVIDENCE","batch4_evidence":["b4-dw-005 is contents-versus-structure rather than a synthesized-question case"],"confidence":"LOW"},"DESCRIPTION_ASYMMETRY":{"status":"PARTIALLY_SUPPORTED","batch4_evidence":["Both misses occurred in scanner-high neighboring pairs whose distinctions are primarily descriptive; this is metadata evidence, not causal proof."],"confidence":"LOW"},"SCHEMA_SEMANTIC_GAP":{"status":"PARTIALLY_SUPPORTED","batch4_evidence":["Both misses were valid externally clear requests represented by neighboring schema operations; only two failures limit inference."],"confidence":"LOW_TO_MEDIUM"}},"subjects":[{"canonical_id":"nixos-mcp","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":5,"misses":1,"strict_accuracy":0.833333,"wilson_95":{"lower":0.436497,"upper":0.969947,"lower_percent":43.6497,"upper_percent":96.9947},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"national-parks-mcp","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"datagouv-mcp","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"deepwiki-mcp","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":5,"misses":1,"strict_accuracy":0.833333,"wilson_95":{"lower":0.436497,"upper":0.969947,"lower_percent":43.6497,"upper_percent":96.9947},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"microsoft-release-communications","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"cloudflare-docs-mcp","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"microsoft-learn","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"exa-mcp","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"}],"frozen_measurement_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"},"cross_batch_separation":{"expansion_v1_untouched":true,"external_gold_v1_untouched":true,"batch1_untouched":true,"batch2_untouched":true,"batch3_untouched":true,"batch5_untouched":true,"batch6_untouched":true,"no_combined_headline_score":true},"strict_score_authoritative":true,"adjusted_accuracy_used":false,"target_mcp_business_tools_called":false,"isolated_real_model_executor_exposed":false,"scanner_production_validity_claimed":false,"interpretation":"Batch 4 is benchmark-level S2 evidence with a promising exploratory scanner observation. It is not a validated predictor or production risk score."},"selection_evidence_expansion_v2_batch5":{"schema_version":"mcp-selection-lab.agent-trust-lab.selection-evidence-status-v5","track":"EXTERNAL_GOLD_EXPANSION_V2","status":"MEASURED","benchmark":"External Gold Expansion V2 Batch 5","benchmark_version":"external-gold-expansion-v2-batch5","batch_characterization":"PREDICTIVE_VALIDATION_POWER_EXPANSION_BATCH","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","subject_level_promotions":[],"case_count":77,"strict_correct":74,"strict_score":"74/77","strict_accuracy":0.961039,"strict_accuracy_display":"96.1039%","wilson_95":{"lower":0.8916,"upper":0.986662,"lower_percent":89.16,"upper_percent":98.6662},"provider_valid_count":77,"provider_valid_coverage":1.0,"semantic_coverage":1.0,"infrastructure_failure_count":0,"risk_bucket_metrics":{"HIGH":{"cases":32,"provider_valid_cases":32,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":29,"misses":3,"strict_accuracy":0.90625,"miss_rate":0.09375,"wilson_95":{"lower":0.757818,"upper":0.967598,"lower_percent":75.7818,"upper_percent":96.7598}},"MEDIUM":{"cases":21,"provider_valid_cases":21,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":21,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.845361,"upper":1.0,"lower_percent":84.5361,"upper_percent":100.0}},"LOW":{"cases":24,"provider_valid_cases":24,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":24,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.862024,"upper":1.0,"lower_percent":86.2024,"upper_percent":100.0}}},"risk_comparison":{"high":{"cases":32,"provider_valid_cases":32,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":29,"misses":3,"strict_accuracy":0.90625,"miss_rate":0.09375,"wilson_95":{"lower":0.757818,"upper":0.967598,"lower_percent":75.7818,"upper_percent":96.7598}},"low":{"cases":24,"provider_valid_cases":24,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":24,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.862024,"upper":1.0,"lower_percent":86.2024,"upper_percent":100.0}},"non_high":{"cases":45,"provider_valid_cases":45,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":45,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.921348,"upper":1.0,"lower_percent":92.1348,"upper_percent":100.0}},"high_minus_low_miss_rate":0.09375,"high_vs_low_risk_ratio":null,"high_vs_low_odds_ratio":null,"continuity_correction":{"method":"Haldane-Anscombe add 0.5 to each 2x2 cell","corrected_risk_ratio":5.30303,"corrected_odds_ratio":5.813559},"interpretation":"Exploratory only: LOW has zero misses, so uncorrected RR/OR are not finite and the corrected estimates are unstable with the observed failure count."},"scanner_status":"PROMISING_CROSS_SUBJECT_SIGNAL_MORE_DATA_REQUIRED","scanner_validation":{"decision":"PROMISING_CROSS_SUBJECT_SIGNAL_MORE_DATA_REQUIRED","criteria":{"results":{"C1_HIGH_EXCEEDS_LOW":true,"C2_FAILURE_SUBJECT_DISPERSION":false,"C3_LEAVE_ONE_SUBJECT_OUT":true,"C4_BOUNDARY_FAMILY_NONDOMINANCE":true,"C5_EVIDENCE_INTEGRITY":true,"C6_PROVIDER_COMPLETENESS":true,"C7_SCANNER_IMMUTABILITY":true},"failed_criteria":["C2_FAILURE_SUBJECT_DISPERSION"],"criterion_pass_count":6,"criterion_count":7},"cross_subject_validated":false,"production_validity_claimed":false,"disclosure":"All three Batch 5 semantic misses occurred in DeepWiki; pre-frozen cross-subject dispersion criterion C2 failed."},"failure_dispersion":{"miss_count":3,"subjects_with_miss":["deepwiki-mcp"],"subject_count_with_miss":1,"boundary_families_with_miss":["CONTENTS_VS_QUESTION","CONTENTS_VS_STRUCTURE","QUESTION_VS_CONTENTS"],"boundary_family_count_with_miss":3,"concentration_note":"All observed misses are from one subject; no cross-subject failure dispersion was observed."},"leave_one_subject_out":{"results":{"datagouv-mcp":{"omitted_subject":"datagouv-mcp","high_cases":27,"high_misses":3,"high_miss_rate":0.1111111111111111,"non_high_cases":43,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true},"deepwiki-mcp":{"omitted_subject":"deepwiki-mcp","high_cases":27,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":43,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":false,"informative_failure_signal_remains":false},"microsoft-release-communications":{"omitted_subject":"microsoft-release-communications","high_cases":28,"high_misses":3,"high_miss_rate":0.10714285714285714,"non_high_cases":42,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true},"microsoft-learn":{"omitted_subject":"microsoft-learn","high_cases":32,"high_misses":3,"high_miss_rate":0.09375,"non_high_cases":38,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true},"cloudflare-docs-mcp":{"omitted_subject":"cloudflare-docs-mcp","high_cases":32,"high_misses":3,"high_miss_rate":0.09375,"non_high_cases":38,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true},"exa-mcp":{"omitted_subject":"exa-mcp","high_cases":32,"high_misses":3,"high_miss_rate":0.09375,"non_high_cases":38,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true},"nixos-mcp":{"omitted_subject":"nixos-mcp","high_cases":27,"high_misses":3,"high_miss_rate":0.1111111111111111,"non_high_cases":43,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true},"national-parks-mcp":{"omitted_subject":"national-parks-mcp","high_cases":26,"high_misses":3,"high_miss_rate":0.11538461538461539,"non_high_cases":44,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true},"wikipedia-mcp-reference":{"omitted_subject":"wikipedia-mcp-reference","high_cases":27,"high_misses":3,"high_miss_rate":0.1111111111111111,"non_high_cases":43,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true},"google-maps-mcp":{"omitted_subject":"google-maps-mcp","high_cases":30,"high_misses":3,"high_miss_rate":0.1,"non_high_cases":40,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true},"metropolitan-museum-mcp":{"omitted_subject":"metropolitan-museum-mcp","high_cases":32,"high_misses":3,"high_miss_rate":0.09375,"non_high_cases":38,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true}},"directional_fraction":1.0,"evaluable_omissions":11,"direction_count":11,"all_direction_at_least_non_high":true,"strict_signal_remains_omissions":10,"criterion_pass":true,"interpretation":"Direction is non-reversed in every omission, but omitting DeepWiki removes all failures; cross-subject predictive validity is therefore not established."},"boundary_family_sensitivity":{"recurring_family_count":11,"high_miss_counts_by_family":{"CONTENTS_VS_QUESTION":1,"CONTENTS_VS_STRUCTURE":1,"QUESTION_VS_CONTENTS":1},"max_high_miss_share":0.333333,"sensitivity_direction_count":11,"sensitivity_evaluable_count":11,"sensitivity_direction_fraction":1.0,"criterion_pass":true},"semantic_miss_count":3,"semantic_misses":[{"case_id":"b5-dw-003","subject_id":"deepwiki-mcp","frozen_prompt":"Read the documented procedure for running the repository's test suite; do not summarize it or list other topics.","expected_tool":"read_wiki_contents","selected_tool":"read_wiki_structure","plausible_competing_tools":["ask_question"],"selection_boundary":"CONTENTS_VS_QUESTION","difficulty":"H3","risk_score":51.6667,"risk_bucket":"HIGH","risk_factors":["DESCRIPTION_LEXICAL_OVERLAP","SHARED_REQUIRED_INPUTS","MISSING_OUTPUT_CONTRAST","INCOMPLETE_BOUNDARY_LANGUAGE"],"external_source_id":"deepwiki-aws-guide-2026-09-10","source_url":"https://docs.devin.ai/work-with-devin/deepwiki-mcp","source_version":"public guide and repository evidence captured 2026-09-10","source_excerpt_reference":"guide section on documentation contents","gold_label_changed":false,"original_prediction_preserved":true,"gold_validity":"VALID_GOLD_RETAINED","adjudication_reason":"The user requests source page content, while synthesized Q&A would produce an explanation instead."},{"case_id":"b5-dw-004","subject_id":"deepwiki-mcp","frozen_prompt":"Explain why the repository separates build and release stages, using its documentation as context.","expected_tool":"ask_question","selected_tool":"read_wiki_contents","plausible_competing_tools":["read_wiki_contents"],"selection_boundary":"QUESTION_VS_CONTENTS","difficulty":"H3","risk_score":51.6667,"risk_bucket":"HIGH","risk_factors":["DESCRIPTION_LEXICAL_OVERLAP","SHARED_REQUIRED_INPUTS","MISSING_OUTPUT_CONTRAST","INCOMPLETE_BOUNDARY_LANGUAGE"],"external_source_id":"deepwiki-aws-guide-2026-09-10","source_url":"https://docs.devin.ai/work-with-devin/deepwiki-mcp","source_version":"public guide and repository evidence captured 2026-09-10","source_excerpt_reference":"guide section on repository Q&A","gold_label_changed":false,"original_prediction_preserved":true,"gold_validity":"VALID_GOLD_RETAINED","adjudication_reason":"The requested output is an explanation, not a page dump."},{"case_id":"b5-dw-006","subject_id":"deepwiki-mcp","frozen_prompt":"Read the documentation page describing the repository's contribution workflow in full.","expected_tool":"read_wiki_contents","selected_tool":"read_wiki_structure","plausible_competing_tools":["read_wiki_structure"],"selection_boundary":"CONTENTS_VS_STRUCTURE","difficulty":"H3","risk_score":73.5714,"risk_bucket":"HIGH","risk_factors":["DESCRIPTION_LEXICAL_OVERLAP","IDENTICAL_REQUIRED_INPUTS_OR_BOTH_NULL","MISSING_OUTPUT_CONTRAST","INCOMPLETE_BOUNDARY_LANGUAGE"],"external_source_id":"deepwiki-aws-guide-2026-09-10","source_url":"https://docs.devin.ai/work-with-devin/deepwiki-mcp","source_version":"public guide and repository evidence captured 2026-09-10","source_excerpt_reference":"guide documentation-page example","gold_label_changed":false,"original_prediction_preserved":true,"gold_validity":"VALID_GOLD_RETAINED","adjudication_reason":"The requested artifact is page content, not the list of pages."}],"taxonomy_validation":{"RETRIEVAL_GRANULARITY_CONFUSION":{"status":"SUPPORTED_SINGLE_SUBJECT_ONLY","evidence":["b5-dw-003","b5-dw-006"],"confidence":"LOW"},"DEFAULT_GENERAL_TOOL_PREFERENCE":{"status":"SUPPORTED_SINGLE_SUBJECT_ONLY","evidence":["The three misses selected neighboring DeepWiki retrieval tools, but none establishes a general fallback preference across subjects."],"confidence":"LOW"},"QUESTION_VS_STRUCTURE_CONFUSION":{"status":"SUPPORTED_SINGLE_SUBJECT_ONLY","evidence":["b5-dw-003 and b5-dw-004 are contents/question/structure boundary misses within DeepWiki."],"confidence":"LOW_TO_MEDIUM"},"DESCRIPTION_ASYMMETRY":{"status":"PARTIALLY_SUPPORTED","evidence":["All misses carry scanner factors MISSING_OUTPUT_CONTRAST and INCOMPLETE_BOUNDARY_LANGUAGE, but all occur in one schema."],"confidence":"LOW"},"SCHEMA_SEMANTIC_GAP":{"status":"SUPPORTED_SINGLE_SUBJECT_ONLY","evidence":["DeepWiki's neighboring tools share repoName and rely on output semantics; no cross-subject recurrence of failure was observed."],"confidence":"LOW"}},"subjects":[{"canonical_id":"datagouv-mcp","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"deepwiki-mcp","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":4,"misses":3,"strict_accuracy":0.571429,"miss_rate":0.428571,"wilson_95":{"lower":0.250458,"upper":0.84178,"lower_percent":25.0458,"upper_percent":84.178},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"microsoft-release-communications","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"microsoft-learn","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"cloudflare-docs-mcp","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"exa-mcp","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"nixos-mcp","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"national-parks-mcp","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"wikipedia-mcp-reference","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"google-maps-mcp","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"metropolitan-museum-mcp","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"}],"frozen_measurement_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"},"cross_batch_separation":{"expansion_v1_untouched":true,"external_gold_v1_untouched":true,"batch1_untouched":true,"batch2_untouched":true,"batch3_untouched":true,"batch4_untouched":true,"no_combined_headline_score":true},"strict_score_authoritative":true,"adjusted_accuracy_used":false,"target_mcp_business_tools_called":false,"isolated_real_model_executor_exposed":false,"scanner_production_validity_claimed":false,"interpretation":"This is benchmark-level S2 evidence only. Batch 5 provides a promising but incomplete cross-subject signal; it does not validate BOUNDARY_RISK_SCAN_V1 for experimental use because C2 failed."},"selection_evidence_expansion_v2_batch6":{"schema_version":"mcp-selection-lab.agent-trust-lab.selection-evidence-status-v6","track":"EXTERNAL_GOLD_EXPANSION_V2","status":"MEASURED","benchmark":"External Gold Expansion V2 Batch 6","benchmark_version":"external-gold-expansion-v2-batch6","batch_characterization":"NEW_SUBJECT_DISPERSION_HOLDOUT","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","subject_level_promotions":[],"case_count":80,"strict_correct":80,"strict_score":"80/80","strict_accuracy":1.0,"strict_accuracy_display":"100%","wilson_95":{"lower":0.954182,"upper":1.0,"lower_percent":95.4182,"upper_percent":100.0},"wilson_95_display":"95.4182%–100%","provider_valid_count":80,"provider_valid_coverage":1.0,"semantic_coverage":1.0,"infrastructure_failure_count":0,"risk_bucket_metrics":{"HIGH":{"cases":32,"provider_valid_cases":32,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":32,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.892821,"upper":1.0,"lower_percent":89.2821,"upper_percent":100.0}},"MEDIUM":{"cases":20,"provider_valid_cases":20,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":20,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.838875,"upper":1.0,"lower_percent":83.8875,"upper_percent":100.0}},"LOW":{"cases":28,"provider_valid_cases":28,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":28,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.879357,"upper":1.0,"lower_percent":87.9357,"upper_percent":100.0}}},"risk_comparison":{"high":{"cases":32,"provider_valid_cases":32,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":32,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.892821,"upper":1.0,"lower_percent":89.2821,"upper_percent":100.0}},"low":{"cases":28,"provider_valid_cases":28,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":28,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.879357,"upper":1.0,"lower_percent":87.9357,"upper_percent":100.0}},"non_high":{"cases":48,"provider_valid_cases":48,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":48,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.9259,"upper":1.0,"lower_percent":92.59,"upper_percent":100.0}},"high_minus_low_miss_rate":0.0,"high_vs_low_risk_ratio":null,"high_vs_low_odds_ratio":null,"continuity_correction":{"method":"Haldane-Anscombe add 0.5 to each 2x2 cell","corrected_risk_ratio":0.878788,"corrected_odds_ratio":0.876923},"interpretation":"No semantic misses were observed in HIGH or LOW; uncorrected ratios are undefined and corrected estimates are unstable and non-validating."},"scanner_status":"WEAK_PREDICTIVE_SIGNAL","scanner_validation":{"decision":"WEAK_PREDICTIVE_SIGNAL","criteria":{"results":{"C1_HIGH_EXCEEDS_LOW":false,"C2_NON_DEEPWIKI_SUBJECT_DISPERSION":false,"C3_HIGH_MISS_CONCENTRATION":false,"C4_LEAVE_ONE_SUBJECT_OUT":true,"C5_BOUNDARY_FAMILY_ROBUSTNESS":true,"C6_NO_SINGLE_SUBJECT_DOMINATION":true,"C7_GOLD_INTEGRITY":true,"C8_PROVIDER_COMPLETENESS":true,"C9_SCANNER_IMMUTABILITY":true},"pass_count":6,"criterion_count":9,"failed_criteria":["C1_HIGH_EXCEEDS_LOW","C2_NON_DEEPWIKI_SUBJECT_DISPERSION","C3_HIGH_MISS_CONCENTRATION"]},"cross_subject_validated":false,"production_validity_claimed":false,"disclosure":"DeepWiki was excluded prospectively; no semantic failures occurred in the holdout, HIGH-risk cases did not underperform, and pre-frozen C1/C2/C3 failed. BOUNDARY_RISK_SCAN_V1 was therefore not validated."},"primary_holdout_excluded_subjects":["deepwiki-mcp"],"failure_dispersion":{"miss_count":0,"subject_count_with_miss":0,"subjects_with_miss":[],"largest_subject_miss_count":0,"largest_subject_miss_share":null,"boundary_family_count_with_miss":0,"boundary_families_with_miss":[],"interpretation":"No provider-valid semantic misses occurred in the primary non-DeepWiki holdout; cross-subject predictive failure dispersion is not demonstrated."},"subject_adjusted_analysis":{"method":"Within-subject HIGH versus non-HIGH comparison; all primary subjects with both partitions are retained.","evaluable_subject_count":6,"results":[{"subject_id":"datagouv-mcp","high_cases":6,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":2,"non_high_misses":0,"non_high_miss_rate":0.0,"high_at_least_non_high":true,"strictly_higher":false},{"subject_id":"microsoft-release-communications","high_cases":4,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":4,"non_high_misses":0,"non_high_miss_rate":0.0,"high_at_least_non_high":true,"strictly_higher":false},{"subject_id":"nixos-mcp","high_cases":6,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":2,"non_high_misses":0,"non_high_miss_rate":0.0,"high_at_least_non_high":true,"strictly_higher":false},{"subject_id":"national-parks-mcp","high_cases":6,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":2,"non_high_misses":0,"non_high_miss_rate":0.0,"high_at_least_non_high":true,"strictly_higher":false},{"subject_id":"wikipedia-mcp-reference","high_cases":6,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":2,"non_high_misses":0,"non_high_miss_rate":0.0,"high_at_least_non_high":true,"strictly_higher":false},{"subject_id":"google-maps-mcp","high_cases":4,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":4,"non_high_misses":0,"non_high_miss_rate":0.0,"high_at_least_non_high":true,"strictly_higher":false}],"high_at_least_non_high_count":6,"strictly_higher_count":0,"directional_fraction":1.0,"interpretation":"Descriptive subject-adjusted comparison only; all observed partitions have zero semantic misses, so no independent failure signal is established."},"leave_one_subject_out":{"results":{"datagouv-mcp":{"omitted_subject":"datagouv-mcp","high_cases":26,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":46,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false},"microsoft-release-communications":{"omitted_subject":"microsoft-release-communications","high_cases":28,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":44,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false},"microsoft-learn":{"omitted_subject":"microsoft-learn","high_cases":32,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":40,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false},"cloudflare-docs-mcp":{"omitted_subject":"cloudflare-docs-mcp","high_cases":32,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":40,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false},"exa-mcp":{"omitted_subject":"exa-mcp","high_cases":32,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":40,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false},"nixos-mcp":{"omitted_subject":"nixos-mcp","high_cases":26,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":46,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false},"national-parks-mcp":{"omitted_subject":"national-parks-mcp","high_cases":26,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":46,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false},"wikipedia-mcp-reference":{"omitted_subject":"wikipedia-mcp-reference","high_cases":26,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":46,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false},"google-maps-mcp":{"omitted_subject":"google-maps-mcp","high_cases":28,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":44,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false},"metropolitan-museum-mcp":{"omitted_subject":"metropolitan-museum-mcp","high_cases":32,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":40,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false}},"evaluable_omissions":10,"direction_count":10,"directional_fraction":1.0,"criterion_pass":true,"interpretation":"Direction is non-reversed but entirely uninformative because no semantic failures occurred."},"boundary_family_sensitivity":{"results":{"ADDRESS_TO_COORDINATES_VS_COORDINATES_TO_ADDRESS":{"excluded_family":"ADDRESS_TO_COORDINATES_VS_COORDINATES_TO_ADDRESS","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"ALERTS_VS_EVENTS":{"excluded_family":"ALERTS_VS_EVENTS","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"ALERTS_VS_PARK_DISCOVERY":{"excluded_family":"ALERTS_VS_PARK_DISCOVERY","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"ARTICLE_DISCOVERY_VS_FULL_CONTENT":{"excluded_family":"ARTICLE_DISCOVERY_VS_FULL_CONTENT","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"AZURE_COLLECTION_VS_ITEM":{"excluded_family":"AZURE_COLLECTION_VS_ITEM","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"AZURE_ITEM_VS_COLLECTION":{"excluded_family":"AZURE_ITEM_VS_COLLECTION","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"AZURE_ITEM_VS_M365_ITEM":{"excluded_family":"AZURE_ITEM_VS_M365_ITEM","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"AZURE_VS_M365_COLLECTION":{"excluded_family":"AZURE_VS_M365_COLLECTION","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"CAMPGROUNDS_VS_VISITOR_CENTERS":{"excluded_family":"CAMPGROUNDS_VS_VISITOR_CENTERS","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"CODE_SAMPLE_VS_FULL_DOC":{"excluded_family":"CODE_SAMPLE_VS_FULL_DOC","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"CODE_VS_CONCEPTUAL_DOCS":{"excluded_family":"CODE_VS_CONCEPTUAL_DOCS","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"COORDINATES_TO_ADDRESS_VS_ADDRESS_TO_COORDINATES":{"excluded_family":"COORDINATES_TO_ADDRESS_VS_ADDRESS_TO_COORDINATES","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"CURRENT_DISCOVERY_VS_KNOWN_URL":{"excluded_family":"CURRENT_DISCOVERY_VS_KNOWN_URL","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DARWIN_OPTION_VS_HOME_MANAGER_OPTION":{"excluded_family":"DARWIN_OPTION_VS_HOME_MANAGER_OPTION","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DARWIN_VS_HOME_MANAGER_SEARCH":{"excluded_family":"DARWIN_VS_HOME_MANAGER_SEARCH","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DATASERVICE_VS_DATASET":{"excluded_family":"DATASERVICE_VS_DATASET","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DATASET_METADATA_VS_USAGE_METRICS":{"excluded_family":"DATASET_METADATA_VS_USAGE_METRICS","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DATASET_VS_DATASERVICE":{"excluded_family":"DATASET_VS_DATASERVICE","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DEDICATED_GUIDE_VS_GENERAL_SEARCH":{"excluded_family":"DEDICATED_GUIDE_VS_GENERAL_SEARCH","remaining_cases":77,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DEPARTMENT_INDEX_VS_OBJECT_DETAIL":{"excluded_family":"DEPARTMENT_INDEX_VS_OBJECT_DETAIL","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DEPARTMENT_INDEX_VS_OBJECT_SEARCH":{"excluded_family":"DEPARTMENT_INDEX_VS_OBJECT_SEARCH","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DISCOVERY_VS_KNOWN_PAGE":{"excluded_family":"DISCOVERY_VS_KNOWN_PAGE","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DISCOVERY_VS_PAGE_READ":{"excluded_family":"DISCOVERY_VS_PAGE_READ","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DISCOVERY_VS_PARK_DETAIL":{"excluded_family":"DISCOVERY_VS_PARK_DETAIL","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DOCS_VS_CODE":{"excluded_family":"DOCS_VS_CODE","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"EVENTS_VS_ALERTS":{"excluded_family":"EVENTS_VS_ALERTS","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"FLAKE_SEARCH_VS_HOME_MANAGER_SEARCH":{"excluded_family":"FLAKE_SEARCH_VS_HOME_MANAGER_SEARCH","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"FULL_ARTICLE_VS_SECTIONS":{"excluded_family":"FULL_ARTICLE_VS_SECTIONS","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"FULL_ARTICLE_VS_SUMMARY":{"excluded_family":"FULL_ARTICLE_VS_SUMMARY","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"GENERAL_DOCS_VS_MIGRATION_GUIDE":{"excluded_family":"GENERAL_DOCS_VS_MIGRATION_GUIDE","remaining_cases":75,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"HOME_MANAGER_OPTION_VS_DARWIN_OPTION":{"excluded_family":"HOME_MANAGER_OPTION_VS_DARWIN_OPTION","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"HOME_MANAGER_SEARCH_VS_FLAKE_SEARCH":{"excluded_family":"HOME_MANAGER_SEARCH_VS_FLAKE_SEARCH","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"HOME_MANAGER_VS_DARWIN_SEARCH":{"excluded_family":"HOME_MANAGER_VS_DARWIN_SEARCH","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"KEY_FACTS_VS_GENERAL_SUMMARY":{"excluded_family":"KEY_FACTS_VS_GENERAL_SUMMARY","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"KNOWN_PAGE_VS_DISCOVERY":{"excluded_family":"KNOWN_PAGE_VS_DISCOVERY","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"KNOWN_PAGE_VS_SEARCH":{"excluded_family":"KNOWN_PAGE_VS_SEARCH","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"KNOWN_URL_VS_OPEN_SEARCH":{"excluded_family":"KNOWN_URL_VS_OPEN_SEARCH","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"KNOWN_URL_VS_SEARCH":{"excluded_family":"KNOWN_URL_VS_SEARCH","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"M365_COLLECTION_VS_ITEM":{"excluded_family":"M365_COLLECTION_VS_ITEM","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"M365_ITEM_VS_AZURE_ITEM":{"excluded_family":"M365_ITEM_VS_AZURE_ITEM","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"M365_ITEM_VS_COLLECTION":{"excluded_family":"M365_ITEM_VS_COLLECTION","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"M365_VS_AZURE_COLLECTION":{"excluded_family":"M365_VS_AZURE_COLLECTION","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"NIXOS_INFO_VS_SEARCH":{"excluded_family":"NIXOS_INFO_VS_SEARCH","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"NIXOS_SEARCH_VS_INFO":{"excluded_family":"NIXOS_SEARCH_VS_INFO","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"OBJECT_DETAIL_VS_OBJECT_SEARCH":{"excluded_family":"OBJECT_DETAIL_VS_OBJECT_SEARCH","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"OBJECT_SEARCH_VS_OBJECT_DETAIL":{"excluded_family":"OBJECT_SEARCH_VS_OBJECT_DETAIL","remaining_cases":77,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"OPENAPI_VS_SERVICE_METADATA":{"excluded_family":"OPENAPI_VS_SERVICE_METADATA","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"PARK_DETAIL_VS_DISCOVERY":{"excluded_family":"PARK_DETAIL_VS_DISCOVERY","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"PARK_DISCOVERY_VS_ALERTS":{"excluded_family":"PARK_DISCOVERY_VS_ALERTS","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"PLACE_DETAIL_VS_PLACE_DISCOVERY":{"excluded_family":"PLACE_DETAIL_VS_PLACE_DISCOVERY","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"PLACE_DISCOVERY_VS_PLACE_DETAIL":{"excluded_family":"PLACE_DISCOVERY_VS_PLACE_DETAIL","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"RESOURCE_METADATA_VS_QUERY":{"excluded_family":"RESOURCE_METADATA_VS_QUERY","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"RESOURCE_QUERY_VS_METADATA":{"excluded_family":"RESOURCE_QUERY_VS_METADATA","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"SECTIONS_VS_FULL_ARTICLE":{"excluded_family":"SECTIONS_VS_FULL_ARTICLE","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"SERVICE_METADATA_VS_OPENAPI":{"excluded_family":"SERVICE_METADATA_VS_OPENAPI","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"SUMMARY_VS_FULL_ARTICLE":{"excluded_family":"SUMMARY_VS_FULL_ARTICLE","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"SUMMARY_VS_SECTIONS":{"excluded_family":"SUMMARY_VS_SECTIONS","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"USAGE_METRICS_VS_DATASET_METADATA":{"excluded_family":"USAGE_METRICS_VS_DATASET_METADATA","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"VISITOR_CENTERS_VS_CAMPGROUNDS":{"excluded_family":"VISITOR_CENTERS_VS_CAMPGROUNDS","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"WEB_DISCOVERY_VS_KNOWN_PAGE":{"excluded_family":"WEB_DISCOVERY_VS_KNOWN_PAGE","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false}},"evaluable_count":60,"direction_count":60,"directional_fraction":1.0},"semantic_miss_count":0,"semantic_misses":[],"taxonomy_validation":{"RETRIEVAL_GRANULARITY_CONFUSION":{"status":"INSUFFICIENT_EVIDENCE","confidence":"LOW","evidence":["Batch 6 primary holdout excluded DeepWiki and observed zero semantic misses; no new cross-subject recurrence supports this signal."]},"DEFAULT_GENERAL_TOOL_PREFERENCE":{"status":"INSUFFICIENT_EVIDENCE","confidence":"LOW","evidence":["Batch 6 primary holdout excluded DeepWiki and observed zero semantic misses; no new cross-subject recurrence supports this signal."]},"QUESTION_VS_STRUCTURE_CONFUSION":{"status":"INSUFFICIENT_EVIDENCE","confidence":"LOW","evidence":["Batch 6 primary holdout excluded DeepWiki and observed zero semantic misses; no new cross-subject recurrence supports this signal."]},"DESCRIPTION_ASYMMETRY":{"status":"INSUFFICIENT_EVIDENCE","confidence":"LOW","evidence":["Batch 6 primary holdout excluded DeepWiki and observed zero semantic misses; no new cross-subject recurrence supports this signal."]},"SCHEMA_SEMANTIC_GAP":{"status":"INSUFFICIENT_EVIDENCE","confidence":"LOW","evidence":["Batch 6 primary holdout excluded DeepWiki and observed zero semantic misses; no new cross-subject recurrence supports this signal."]}},"subjects":[{"canonical_id":"datagouv-mcp","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"HIGH":6,"LOW":2},"HIGH":{"cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0}},"MEDIUM":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"LOW":{"cases":2,"provider_valid_cases":2,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":2,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.34238,"upper":1.0,"lower_percent":34.238,"upper_percent":100.0}},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"microsoft-release-communications","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"HIGH":4,"LOW":4},"HIGH":{"cases":4,"provider_valid_cases":4,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":4,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.510109,"upper":1.0,"lower_percent":51.0109,"upper_percent":100.0}},"MEDIUM":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"LOW":{"cases":4,"provider_valid_cases":4,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":4,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.510109,"upper":1.0,"lower_percent":51.0109,"upper_percent":100.0}},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"microsoft-learn","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"LOW":8},"HIGH":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"MEDIUM":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"LOW":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"cloudflare-docs-mcp","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"LOW":8},"HIGH":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"MEDIUM":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"LOW":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"exa-mcp","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"MEDIUM":8},"HIGH":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"MEDIUM":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"LOW":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"nixos-mcp","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"HIGH":6,"MEDIUM":2},"HIGH":{"cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0}},"MEDIUM":{"cases":2,"provider_valid_cases":2,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":2,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.34238,"upper":1.0,"lower_percent":34.238,"upper_percent":100.0}},"LOW":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"national-parks-mcp","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"HIGH":6,"LOW":2},"HIGH":{"cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0}},"MEDIUM":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"LOW":{"cases":2,"provider_valid_cases":2,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":2,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.34238,"upper":1.0,"lower_percent":34.238,"upper_percent":100.0}},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"wikipedia-mcp-reference","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"HIGH":6,"MEDIUM":2},"HIGH":{"cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0}},"MEDIUM":{"cases":2,"provider_valid_cases":2,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":2,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.34238,"upper":1.0,"lower_percent":34.238,"upper_percent":100.0}},"LOW":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"google-maps-mcp","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"HIGH":4,"LOW":4},"HIGH":{"cases":4,"provider_valid_cases":4,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":4,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.510109,"upper":1.0,"lower_percent":51.0109,"upper_percent":100.0}},"MEDIUM":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"LOW":{"cases":4,"provider_valid_cases":4,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":4,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.510109,"upper":1.0,"lower_percent":51.0109,"upper_percent":100.0}},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"metropolitan-museum-mcp","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"MEDIUM":8},"HIGH":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"MEDIUM":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"LOW":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"}],"frozen_measurement_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"},"cross_batch_separation":{"expansion_v1_untouched":true,"external_gold_v1_untouched":true,"batch1_untouched":true,"batch2_untouched":true,"batch3_untouched":true,"batch4_untouched":true,"batch5_untouched":true,"no_combined_headline_score":true},"strict_score_authoritative":true,"adjusted_accuracy_used":false,"target_mcp_business_tools_called":false,"isolated_real_model_executor_exposed":false,"scanner_production_validity_claimed":false,"interpretation":"This is benchmark-level S2 evidence and measured historical selection performance from a frozen holdout. It does not validate BOUNDARY_RISK_SCAN_V1, predict future errors, or provide a reliable pre-measurement failure forecast."},"empirical_selection_reliability":{"schema_version":"mcp-selection-lab.empirical-selection-reliability.v1","capability":"EMPIRICAL_SELECTION_RELIABILITY","status":"MEASURED_HISTORICAL_EVIDENCE","interpretation":"This is measured historical selection reliability from frozen benchmarks.","predictive_claim":false,"scanner_predictions_included":false,"aggregation_policy":{"schema_version":"mcp-selection-lab.empirical-selection-reliability.aggregation-policy.v1","capability":"EMPIRICAL_SELECTION_RELIABILITY","policy_id":"EMPIRICAL_SELECTION_RELIABILITY_AGGREGATION_V1","status":"FROZEN_IMPLEMENTATION_POLICY","interpretation":"This is measured historical selection reliability from frozen benchmarks.","eligible_evidence":{"include_only":"immutable completed benchmark evidence with a bound measurement identity","excluded":["abandoned drafts","uncommitted experiments","rejected gold cases","scanner predictions","provider infrastructure failures as semantic misses"],"included_tracks":["selection-evidence-expansion-v1","external-gold-v1","external-gold-expansion-v2-batch1","external-gold-expansion-v2-batch2","external-gold-expansion-v2-batch3","external-gold-expansion-v2-batch4","external-gold-expansion-v2-batch5","external-gold-expansion-v2-batch6"]},"subject_identity":{"merge_key":"exact canonical subject identity within the same provenance and identity mode","cross_provenance_aliasing":false,"version_bound_to_current_live_merge":false,"mixed_identity_status":"MIXED"},"aggregation":{"batch_independence":"each immutable benchmark/batch remains a separate batch_metrics record","pooling":"NONE","combined_headline_allowed":false,"combined_accuracy_allowed":false,"summary_fields":["total_measured_cases","number_of_batches","strict_accuracy_range_across_batches","latest_strict_accuracy","lowest_observed_strict_accuracy"],"provider_failures":"count separately; exclude from semantic miss count and strict semantic denominator","misses":"retain adjudicated observed semantic misses and case identities; do not infer predictive categories"},"reliability_status_rules":{"NO_MEASUREMENT_EVIDENCE":"zero eligible frozen cases","LIMITED_MEASUREMENT_EVIDENCE":"one eligible independent completed batch with fewer than 24 frozen cases","MEASURED":"one eligible independent completed batch with at least 24 frozen cases","MULTI_BATCH_MEASURED":"at least two eligible independent completed batches"},"public_claim_boundaries":{"predictive_claim":false,"future_failure_claim":false,"transport_health_claim":false,"profile_trust_grade_claim":false,"availability_claim":false,"current_live_status_claim":false,"benchmark_promotion_claim":false,"automatic_subject_ranking":false},"frozen_at":"2026-09-10T00:00:00Z"},"separation":{"transport_runtime_health":"SEPARATE_PROVIDER_VALID_AND_INFRASTRUCTURE_FIELDS","profile_trust_grade":"SEPARATE","availability":"NOT_EVALUATED","current_live_status":"NOT_CLAIMED","benchmark_promotion_level":"REPORTED_AS_SOURCE_EVIDENCE_ONLY","automatic_subject_ranking":false},"batch_records":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","metrics":{"frozen_case_count":240,"provider_valid_cases":240,"infrastructure_failures":0,"strict_correct":240,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.9842460800584412,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"240/240"},"measured_at":"2026-09-10T13:23:12.077611Z","latest_measured_at":"2026-09-10T13:23:12.077611Z","subject_metrics":[{"canonical_subject_id":"cloudflare-docs-mcp","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}},{"canonical_subject_id":"context7","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}},{"canonical_subject_id":"deepwiki-mcp","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}},{"canonical_subject_id":"dex-paprika-mcp-source-snapshot","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}},{"canonical_subject_id":"exa-mcp","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}},{"canonical_subject_id":"gitmcp-docs","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}},{"canonical_subject_id":"metropolitan-museum-mcp-source-snapshot","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}},{"canonical_subject_id":"national-parks-mcp-source-snapshot","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}},{"canonical_subject_id":"nixos-mcp-source-snapshot","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}},{"canonical_subject_id":"uk-premises-licence-mcp","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24"}}],"semantic_misses":[],"evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"},"source_files":[{"path":"validation/selection-evidence-expansion-v1/measurement-artifact-v1.json","sha256":"b18516e83f30484f70165e3360504c0228512631058894bb2d39f12a137d6974"},{"path":"validation/selection-evidence-expansion-v1/case-freeze-v1.json","sha256":"d55b04998bc670a68e944e4b4534e6d65c840520ac93c721b3700c12c4f1c0a7"},{"path":"validation/selection-evidence-expansion-v1/model-lock-v1.json","sha256":"adf4abc13e2313e4d2cc08b27ab96fed43c9bae93e0f23d29ba0d8cd6eb3e02c"},{"path":"validation/selection-evidence-expansion-v1/immutable-measurement-receipt-v1.json","sha256":"bc0a61269913e0207132ad6135d89667ce6ccb00bdd01a25e31e9d1129b9c5d3"},{"path":"validation/selection-evidence-expansion-v1/promotion-evaluation-receipt-v1.json","sha256":"1b4495a16c34723f01aa01d696dd27987fea913bdbd93b39cc7942b72bf05e4d"}],"subject_identity_metadata":{"context7":{"candidate_schema_sha256":"5405cff0b45dd3bbc5f2aad93607b2e6e6afd7869d3cb6806e1d7cec22ed2de4","canonical_id":"context7","case_count":24,"identity_mode":"LIVE_TOOLS_LIST","name":"Context7","schema_path":"validation/selection-evidence-expansion-v1/subjects/context7/schema-v1.json","tool_count":2},"exa-mcp":{"candidate_schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","canonical_id":"exa-mcp","case_count":24,"identity_mode":"LIVE_TOOLS_LIST","name":"Exa MCP","schema_path":"validation/selection-evidence-expansion-v1/subjects/exa-mcp/schema-v1.json","tool_count":2},"cloudflare-docs-mcp":{"candidate_schema_sha256":"84c02eb4bd72c3f2ca868986de39a7c9eeb5b70aaa361f67031b61f6a27769d9","canonical_id":"cloudflare-docs-mcp","case_count":24,"identity_mode":"LIVE_TOOLS_LIST","name":"Cloudflare Docs MCP","schema_path":"validation/selection-evidence-expansion-v1/subjects/cloudflare-docs-mcp/schema-v1.json","tool_count":2},"deepwiki-mcp":{"candidate_schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","canonical_id":"deepwiki-mcp","case_count":24,"identity_mode":"LIVE_TOOLS_LIST","name":"DeepWiki MCP","schema_path":"validation/selection-evidence-expansion-v1/subjects/deepwiki-mcp/schema-v1.json","tool_count":3},"gitmcp-docs":{"candidate_schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","canonical_id":"gitmcp-docs","case_count":24,"identity_mode":"LIVE_TOOLS_LIST","name":"GitMCP Docs","schema_path":"validation/selection-evidence-expansion-v1/subjects/gitmcp-docs/schema-v1.json","tool_count":5},"uk-premises-licence-mcp":{"candidate_schema_sha256":"4e942418380e030109da94b2d20595f742cca6f5437d927fa8f785fcb9ab31e7","canonical_id":"uk-premises-licence-mcp","case_count":24,"identity_mode":"LIVE_TOOLS_LIST","name":"UK Premises Licence MCP","schema_path":"validation/selection-evidence-expansion-v1/subjects/uk-premises-licence-mcp/schema-v1.json","tool_count":8},"nixos-mcp-source-snapshot":{"candidate_schema_sha256":"56b5bcfe699ef5d8176b9dcba45bdedb7d1a8ea347d94c4dc4b0cbd795caeb15","canonical_id":"nixos-mcp-source-snapshot","case_count":24,"identity_mode":"VERSION_BOUND_SOURCE_SNAPSHOT","name":"NixOS MCP (source snapshot)","schema_path":"validation/selection-evidence-expansion-v1/subjects/nixos-mcp-source-snapshot/schema-v1.json","tool_count":18},"dex-paprika-mcp-source-snapshot":{"candidate_schema_sha256":"00a0ccdd5ed751b4006c93eeac208ef770fbb7d11f8d1346588272d750da89f3","canonical_id":"dex-paprika-mcp-source-snapshot","case_count":24,"identity_mode":"VERSION_BOUND_SOURCE_SNAPSHOT","name":"DEX Paprika MCP (source snapshot)","schema_path":"validation/selection-evidence-expansion-v1/subjects/dex-paprika-mcp-source-snapshot/schema-v1.json","tool_count":11},"national-parks-mcp-source-snapshot":{"candidate_schema_sha256":"2ed30d6d2c5200de284ff53688f94b03c2cd8ec7bf4819ecb6c26beb3ad2bb2a","canonical_id":"national-parks-mcp-source-snapshot","case_count":24,"identity_mode":"VERSION_BOUND_SOURCE_SNAPSHOT","name":"National Parks MCP (source snapshot)","schema_path":"validation/selection-evidence-expansion-v1/subjects/national-parks-mcp-source-snapshot/schema-v1.json","tool_count":6},"metropolitan-museum-mcp-source-snapshot":{"candidate_schema_sha256":"b521becc7182000b5b3f2c9b7da95a131002bb5759c7a1ec437f48ac00f4b5ea","canonical_id":"metropolitan-museum-mcp-source-snapshot","case_count":24,"identity_mode":"VERSION_BOUND_SOURCE_SNAPSHOT","name":"Metropolitan Museum MCP (source snapshot)","schema_path":"validation/selection-evidence-expansion-v1/subjects/metropolitan-museum-mcp-source-snapshot/schema-v1.json","tool_count":3}}},{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":18,"strict_accuracy":0.75,"wilson_95":{"lower":0.551006,"upper":0.880006},"semantic_coverage":1.0,"observed_semantic_miss_count":6,"strict_score":"18/24"},"measured_at":"2026-09-09T22:51:26.259023Z","latest_measured_at":"2026-09-09T22:51:26.259023Z","subject_metrics":[{"canonical_subject_id":"external-gold-v1:Context7","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2"}},{"canonical_subject_id":"external-gold-v1:DEX Paprika","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":1,"strict_accuracy":0.5,"wilson_95":{"lower":0.094531,"upper":0.905469},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"1/2"}},{"canonical_subject_id":"external-gold-v1:Google Maps","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":0,"strict_accuracy":0.0,"wilson_95":{"lower":0.0,"upper":0.65762},"semantic_coverage":1.0,"observed_semantic_miss_count":2,"strict_score":"0/2"}},{"canonical_subject_id":"external-gold-v1:Huge Icons","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":1,"strict_accuracy":0.5,"wilson_95":{"lower":0.094531,"upper":0.905469},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"1/2"}},{"canonical_subject_id":"external-gold-v1:Medical Calculator","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2"}},{"canonical_subject_id":"external-gold-v1:Metropolitan Museum","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2"}},{"canonical_subject_id":"external-gold-v1:National Parks","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2"}},{"canonical_subject_id":"external-gold-v1:NixOS","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":1,"strict_accuracy":0.5,"wilson_95":{"lower":0.094531,"upper":0.905469},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"1/2"}},{"canonical_subject_id":"external-gold-v1:OKX Exchange","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":1,"strict_accuracy":0.5,"wilson_95":{"lower":0.094531,"upper":0.905469},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"1/2"}},{"canonical_subject_id":"external-gold-v1:Reddit","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2"}},{"canonical_subject_id":"external-gold-v1:Scientific Computing","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2"}},{"canonical_subject_id":"external-gold-v1:Wikipedia","metrics":{"frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2"}}],"semantic_misses":[{"case_id":"dex_paprika_001","subject_id":"external-gold-v1:DEX Paprika","expected_tool":"getStats","selected_tool":"getNetworks","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"GOLD_SCHEMA_CONTRADICTION","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json"},{"case_id":"google_maps_000","subject_id":"external-gold-v1:Google Maps","expected_tool":"maps_geocode","selected_tool":"search_nearby","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"NON_UNIQUE_FIRST_TOOL","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json"},{"case_id":"google_maps_001","subject_id":"external-gold-v1:Google Maps","expected_tool":"maps_geocode","selected_tool":"search_nearby","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"NON_UNIQUE_FIRST_TOOL","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json"},{"case_id":"huge_icons_000","subject_id":"external-gold-v1:Huge Icons","expected_tool":"list_icons","selected_tool":"search_icons","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"NON_UNIQUE_FIRST_TOOL","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json"},{"case_id":"nixos_001","subject_id":"external-gold-v1:NixOS","expected_tool":"nixos_search","selected_tool":"nixos_info","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"GENUINE_SEQUENCE_ERROR","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json"},{"case_id":"okx_exchange_000","subject_id":"external-gold-v1:OKX Exchange","expected_tool":"get_price","selected_tool":"get_candlesticks","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"PROCEDURE_ORDER_DISAGREEMENT","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json"}],"evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"},"source_files":[{"path":"validation/independent-holdout-v1/external-gold-v1/real-model-measurement-v2.manifest.json","sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd"},{"path":"validation/independent-holdout-v1/external-gold-v1/real-model-measurement-v2.sha256","sha256":"2ab0e9a4904d027d5ee863309ecda835c322a7b68b053b422d6650b4303b5969"},{"path":"validation/independent-holdout-v1/external-gold-v1/real-model-lock-v2.json","sha256":"b5d5209172c6b65e6b13daa348a1992355370760d5d5cadda9620cd2455eece5"},{"path":"validation/independent-holdout-v1/external-gold-v1/final-verification-receipt.json","sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"},{"path":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json","sha256":"bb64f7d7a59cdbbb5ccda4ea1deb7feb52691fb1aff63b951d93964e2df49a74"},{"path":"validation/independent-holdout-v1/external-gold-v1/real-model-measurement-v2.parts/000.b64","sha256":"9366cfa5bb43fb4c0a6d2ae70e8a147362c493a4fd19ae23763d9eedce5f9271"},{"path":"validation/independent-holdout-v1/external-gold-v1/real-model-measurement-v2.parts/001.b64","sha256":"59876d5d611548e62d815f11a2945a3a40b9cb2308f89c823af9539c9905cfae"},{"path":"validation/independent-holdout-v1/external-gold-v1/real-model-measurement-v2.parts/002.b64","sha256":"cd654ae8d99361878efad7be33036daced6367681522724a90e878d55a6ef689"},{"path":"validation/independent-holdout-v1/external-gold-v1/real-model-measurement-v2.parts/003.b64","sha256":"b02ee1a9db9b5264aeac7279caee4e36c25c04bd87d0005f779907ddae0c3019"},{"path":"validation/independent-holdout-v1/external-gold-v1/real-model-measurement-v2.parts/004.b64","sha256":"158694907e71c928fc760972d24f09fae717643776fd07f5deecf5727a359a80"}]},{"batch_id":"external-gold-expansion-v2-batch1","benchmark_id":"external-gold-expansion-v2-batch1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","metrics":{"frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":23,"strict_accuracy":0.9583333333333334,"wilson_95":{"lower":0.7975819352134406,"upper":0.9926065438344942},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"23/24"},"measured_at":"2026-09-10T15:38:03.968297Z","latest_measured_at":"2026-09-10T15:38:03.968297Z","subject_metrics":[{"canonical_subject_id":"datagouv-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.6096657120978346,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"deepwiki-mcp","metrics":{"frozen_case_count":4,"provider_valid_cases":4,"infrastructure_failures":0,"strict_correct":3,"strict_accuracy":0.75,"wilson_95":{"lower":0.30064184258240184,"upper":0.9544127391902995},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"3/4"}},{"canonical_subject_id":"exa-mcp","metrics":{"frozen_case_count":4,"provider_valid_cases":4,"infrastructure_failures":0,"strict_correct":4,"strict_accuracy":1.0,"wilson_95":{"lower":0.5101091635454027,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"4/4"}},{"canonical_subject_id":"microsoft-learn","metrics":{"frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5"}},{"canonical_subject_id":"microsoft-release-communications","metrics":{"frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5"}}],"semantic_misses":[{"case_id":"deepwiki-004","subject_id":"deepwiki-mcp","expected_tool":"ask_question","selected_tool":"read_wiki_structure","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"SEMANTIC_MISS","source_adjudication_artifact":"validation/external-gold-expansion-v2/measurement-v2-batch1/semantic-miss-adjudication-v2-batch1.json"}],"evidence_identity":{"freeze_input_commit":"c614ae51750d12ce113b5e3d51429f52e529f08c","final_freeze_receipt_commit":"b691170","measurement_commit":"b9f358b1a65aba348fa1c0e625eac6da1f31b651","policy_evaluation_commit":"3e560e5767f181831c65569b893ca789cd3bdff0","case_freeze_sha256":"17777d2c3983cd444353b8d69dcc98836a1909ee6fde6f5c8cf18cf9b059427e","model_lock_sha256":"c4330165deb3d8587e224a2ca1b3d886db60801ca436710a0edace55525923d2","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ab085cedf13eef5724f4e160b0f49f8402d62bc5b10203de0f8d81e41f2a017e","measurement_artifact_sha256":"01243422aa946296f235810aaee6f065ec21d9e78a01017ee8161e5e61a8b5e8","policy_evaluation_sha256":"4e710a9c5205660cc07603d2fd7538aa063eda4c2574c29308f9f255e4706f82"},"subject_identity_metadata":{"datagouv-mcp":{"subject_order":1,"canonical_subject_id":"datagouv-mcp","provider_repository":"data.gouv.fr; https://github.com/datagouv/datagouv-mcp","mcp_url":"https://mcp.data.gouv.fr/mcp","identity_mode":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_SNAPSHOT","schema_acquisition":"Local Docker initialize/session handshake followed by tools/list only; no tools/call.","schema_sha256":"fa394d9f6dd207dd3f15ac012a75a0029bdc456e2c1614a7db79c3cd2046dc1d","tools_list_raw_sha256":"bc8d083019728eab35ae94c628f6506d17104c759bfb559f9e494e30a87d0a07","tool_count":10,"tool_names":["get_dataservice_info","get_dataservice_openapi_spec","get_dataset_info","get_metrics","get_resource_info","list_dataset_resources","query_resource_data","search_dataservices","search_datasets","search_organizations"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots/datagouv-mcp/schema-v2.json","schema_snapshot_sha256":"50617b0c4cfb612d275d131eaa63bbe1d778a27f4ed923ef46daf260cbed29d4","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"},"microsoft-learn":{"subject_order":2,"canonical_subject_id":"microsoft-learn","provider_repository":"MicrosoftDocs; https://github.com/MicrosoftDocs/mcp","mcp_url":"https://learn.microsoft.com/api/mcp","identity_mode":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_SNAPSHOT","schema_acquisition":"Local Docker initialize/session handshake followed by tools/list only; no tools/call.","schema_sha256":"9729e31773dce5de4c9a08080af54658b13ff9cf96a078159fe234ef86581b7f","tools_list_raw_sha256":"380a2bbd84cafc59c9c7507baef0cedde7b40220dfd74b6cb8db601c473e73e0","tool_count":3,"tool_names":["microsoft_code_sample_search","microsoft_docs_fetch","microsoft_docs_search"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots/microsoft-learn/schema-v2.json","schema_snapshot_sha256":"dda9b6cfc3a8a93a2f5109e0772456016e586fb8bb40cfea42a607bedac46a1a","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"},"microsoft-release-communications":{"subject_order":3,"canonical_subject_id":"microsoft-release-communications","provider_repository":"MicrosoftDocs; https://github.com/MicrosoftDocs/microsoft-365-docs","mcp_url":"https://www.microsoft.com/releasecommunications/mcp","identity_mode":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_SNAPSHOT","schema_acquisition":"Local Docker initialize/session handshake followed by tools/list only; no tools/call.","schema_sha256":"f934d7f2ed46be9a4fc338c68259683b701c58cf0839e6cf2e2769dd02aa8b97","tools_list_raw_sha256":"d6de197436f503f59595ce18073b4471b4b008b48dbeb1bfbda2c0d8c3d8692b","tool_count":4,"tool_names":["get_azure_update_by_id","get_m365_roadmap_by_id","get_recent_azure_updates","get_recent_m365_roadmaps"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots/microsoft-release-communications/schema-v2.json","schema_snapshot_sha256":"94ed8a0076091d64801956b0215cab38725da7c33aa42f5d4c023c9394ad87b4","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"},"exa-mcp":{"subject_order":4,"canonical_subject_id":"exa-mcp","provider_repository":"Exa Labs; https://github.com/exa-labs/exa-mcp-server","mcp_url":"https://mcp.exa.ai/mcp","identity_mode":"CURRENT_LIVE_TOOLS_LIST_INHERITED_V1_AND_FROZEN_BY_COPY","schema_acquisition":"Exact immutable Expansion V1 tools/list schema snapshot copied into the additive V2 freeze directory; no V2 contact made.","schema_sha256":"14e876b19eb1117acbfba0443e0ec3fe9439700b998c61619c86ab3681ae6d28","tool_count":2,"tool_names":["web_fetch_exa","web_search_exa"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots/exa-mcp/schema-v2.json","schema_snapshot_sha256":"d98db3a5825b1d97d2611369e8bfa3c63bfff19a834d14a686bd099c9a13e252","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"},"deepwiki-mcp":{"subject_order":5,"canonical_subject_id":"deepwiki-mcp","provider_repository":"Cognition/Devin; https://github.com/CognitionAI/deepwiki","mcp_url":"https://mcp.deepwiki.com/mcp","identity_mode":"CURRENT_LIVE_TOOLS_LIST_INHERITED_V1_AND_FROZEN_BY_COPY","schema_acquisition":"Exact immutable Expansion V1 tools/list schema snapshot copied into the additive V2 freeze directory; no V2 contact made.","schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","tool_count":3,"tool_names":["ask_question","read_wiki_contents","read_wiki_structure"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots/deepwiki-mcp/schema-v2.json","schema_snapshot_sha256":"bb02856d9514fd8f03237739275812ceba29916159607e92a5e29380876df479","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"}},"source_files":[{"path":"validation/external-gold-expansion-v2/measurement-v2-batch1/measurement-artifact-v2-batch1.json","sha256":"01243422aa946296f235810aaee6f065ec21d9e78a01017ee8161e5e61a8b5e8"},{"path":"validation/external-gold-expansion-v2/case-freeze-v2-batch1.json","sha256":"17777d2c3983cd444353b8d69dcc98836a1909ee6fde6f5c8cf18cf9b059427e"},{"path":"validation/external-gold-expansion-v2/final-subject-set-v2-batch1.json","sha256":"183b1f1222626d63185eb386a198e90c99d68be75829f2ce780a5b4f906112ce"},{"path":"validation/external-gold-expansion-v2/measurement-v2-batch1/semantic-miss-adjudication-v2-batch1.json","sha256":"bc596d74a02c59066e4db5b8b05f371ff96be118132b17e357ea2962f5ef7df5"},{"path":"validation/external-gold-expansion-v2/measurement-v2-batch1/raw-predictions-v2-batch1.jsonl","sha256":"ab085cedf13eef5724f4e160b0f49f8402d62bc5b10203de0f8d81e41f2a017e"}]},{"batch_id":"external-gold-expansion-v2-batch2","benchmark_id":"external-gold-expansion-v2-batch2","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","metrics":{"frozen_case_count":26,"provider_valid_cases":26,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":0.9230769230769231,"wilson_95":{"lower":0.7585844622837095,"upper":0.9786449115277238},"semantic_coverage":1.0,"observed_semantic_miss_count":2,"strict_score":"24/26"},"measured_at":"2026-09-10T16:49:53.346446Z","latest_measured_at":"2026-09-10T16:49:53.346446Z","subject_metrics":[{"canonical_subject_id":"datagouv-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":0.8333333333333334,"wilson_95":{"lower":0.43649717781352987,"upper":0.9699466302516935},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"5/6"}},{"canonical_subject_id":"deepwiki-mcp","metrics":{"frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5"}},{"canonical_subject_id":"gitmcp-docs","metrics":{"frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":4,"strict_accuracy":0.8,"wilson_95":{"lower":0.37553462976252533,"upper":0.9637758913675698},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"4/5"}},{"canonical_subject_id":"microsoft-learn","metrics":{"frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5"}},{"canonical_subject_id":"microsoft-release-communications","metrics":{"frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5"}}],"semantic_misses":[{"case_id":"datagouv-hard-004","subject_id":"datagouv-mcp","expected_tool":"list_dataset_resources","selected_tool":"get_dataset_info","boundary_family":"DATASET_VS_RESOURCE_LIST","adjudication":"SEMANTIC_MISS_GOLD_REMAINS_VALID","source_adjudication_artifact":"validation/external-gold-expansion-v2/measurement-v2-batch2/semantic-miss-adjudication-v2-batch2.json"},{"case_id":"gitmcp-hard-005","subject_id":"gitmcp-docs","expected_tool":"search_generic_documentation","selected_tool":"fetch_generic_documentation","boundary_family":"DOCUMENTED_BEHAVIOR_VS_IMPLEMENTATION","adjudication":"SEMANTIC_MISS_GOLD_REMAINS_VALID","source_adjudication_artifact":"validation/external-gold-expansion-v2/measurement-v2-batch2/semantic-miss-adjudication-v2-batch2.json"}],"evidence_identity":{"freeze_input_commit":"61f6511c41562c24019cd441b37bc6ee07283951","final_freeze_receipt_commit":"d3c55d2f96a4e45bf2577225ea16929877f2ded1","measurement_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","policy_evaluation_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","case_freeze_sha256":"ce21fc085bd9a0185c9e37d0d795221f286397cbc477ee941e876be24f7ba1b4","model_lock_sha256":"153647f94eed0ee3d3069229545846eef864fd43fa821f2d311dc3fb5d265ba9","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"8c4364ab598241ddbe4adb16453ba2e12b0917a1d1852caedb07884d1a54570b","measurement_artifact_sha256":"dd13aec8a90b2b5490c934c9ad9650c37733c65f18d824e2ae6066202176ced9","policy_evaluation_sha256":"736f154a5d5c3b8f0d77cd58fa0644ca97ad94c7a6b46af98d394c1dc1578f9a"},"subject_identity_metadata":{"datagouv-mcp":{"subject_order":1,"canonical_subject_id":"datagouv-mcp","provider_repository":"data.gouv.fr; https://github.com/datagouv/datagouv-mcp","mcp_url":"https://mcp.data.gouv.fr/mcp","identity_mode":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT","schema_acquisition":"Existing immutable V2 tools/list snapshot reused; no remote contact and no tools/call in Batch 2 preparation.","schema_sha256":"fa394d9f6dd207dd3f15ac012a75a0029bdc456e2c1614a7db79c3cd2046dc1d","tools_list_raw_sha256":"bc8d083019728eab35ae94c628f6506d17104c759bfb559f9e494e30a87d0a07","tool_count":10,"tool_names":["get_dataservice_info","get_dataservice_openapi_spec","get_dataset_info","get_metrics","get_resource_info","list_dataset_resources","query_resource_data","search_dataservices","search_datasets","search_organizations"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch2/datagouv-mcp/schema-v2-batch2.json","schema_snapshot_sha256":"50617b0c4cfb612d275d131eaa63bbe1d778a27f4ed923ef46daf260cbed29d4","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"},"gitmcp-docs":{"subject_order":2,"canonical_subject_id":"gitmcp-docs","provider_repository":"Ido Salomon; https://github.com/idosal/git-mcp","mcp_url":"https://gitmcp.io/docs","identity_mode":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT","schema_acquisition":"Existing stable corpus tools/list metadata snapshot reused; no remote contact and no tools/call in Batch 2 preparation.","schema_sha256":"1b658aa07223e97b7855369a816f4026cc337ab189149d2be1495294363524ee","tools_list_raw_sha256":null,"tool_count":5,"tool_names":["fetch_generic_documentation","fetch_generic_url_content","match_common_libs_owner_repo_mapping","search_generic_code","search_generic_documentation"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch2/gitmcp-docs/schema-v2-batch2.json","schema_snapshot_sha256":"b90eb7608858850eff2594ab88943ab2f7be83630c7447dc1f34461a726c429b","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"},"microsoft-release-communications":{"subject_order":3,"canonical_subject_id":"microsoft-release-communications","provider_repository":"Microsoft; https://github.com/MicrosoftDocs/microsoft-365-docs","mcp_url":"https://www.microsoft.com/releasecommunications/mcp","identity_mode":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT","schema_acquisition":"Existing immutable V2 tools/list snapshot reused; no remote contact and no tools/call in Batch 2 preparation.","schema_sha256":"f934d7f2ed46be9a4fc338c68259683b701c58cf0839e6cf2e2769dd02aa8b97","tools_list_raw_sha256":"d6de197436f503f59595ce18073b4471b4b008b48dbeb1bfbda2c0d8c3d8692b","tool_count":4,"tool_names":["get_azure_update_by_id","get_m365_roadmap_by_id","get_recent_azure_updates","get_recent_m365_roadmaps"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch2/microsoft-release-communications/schema-v2-batch2.json","schema_snapshot_sha256":"94ed8a0076091d64801956b0215cab38725da7c33aa42f5d4c023c9394ad87b4","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"},"microsoft-learn":{"subject_order":4,"canonical_subject_id":"microsoft-learn","provider_repository":"MicrosoftDocs; https://github.com/MicrosoftDocs/mcp","mcp_url":"https://learn.microsoft.com/api/mcp","identity_mode":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT","schema_acquisition":"Existing immutable V2 tools/list snapshot reused; no remote contact and no tools/call in Batch 2 preparation.","schema_sha256":"9729e31773dce5de4c9a08080af54658b13ff9cf96a078159fe234ef86581b7f","tools_list_raw_sha256":"380a2bbd84cafc59c9c7507baef0cedde7b40220dfd74b6cb8db601c473e73e0","tool_count":3,"tool_names":["microsoft_code_sample_search","microsoft_docs_fetch","microsoft_docs_search"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch2/microsoft-learn/schema-v2-batch2.json","schema_snapshot_sha256":"dda9b6cfc3a8a93a2f5109e0772456016e586fb8bb40cfea42a607bedac46a1a","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"},"deepwiki-mcp":{"subject_order":5,"canonical_subject_id":"deepwiki-mcp","provider_repository":"Cognition/Devin; https://github.com/CognitionAI/deepwiki","mcp_url":"https://mcp.deepwiki.com/mcp","identity_mode":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT","schema_acquisition":"Existing immutable V2 tools/list snapshot reused; no remote contact and no tools/call in Batch 2 preparation.","schema_sha256":"0fedc4c75474d3c8068b9d0a2575a6dd5003ad7cf8617207f350f7aed9a0b555","tools_list_raw_sha256":null,"tool_count":3,"tool_names":["ask_question","read_wiki_contents","read_wiki_structure"],"schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch2/deepwiki-mcp/schema-v2-batch2.json","schema_snapshot_sha256":"bb02856d9514fd8f03237739275812ceba29916159607e92a5e29380876df479","qualification_receipt":"validation/external-gold-expansion-v2/qualification-gold-design-v2.json"}},"source_files":[{"path":"validation/external-gold-expansion-v2/measurement-v2-batch2/measurement-artifact-v2-batch2.json","sha256":"dd13aec8a90b2b5490c934c9ad9650c37733c65f18d824e2ae6066202176ced9"},{"path":"validation/external-gold-expansion-v2/case-freeze-v2-batch2.json","sha256":"ce21fc085bd9a0185c9e37d0d795221f286397cbc477ee941e876be24f7ba1b4"},{"path":"validation/external-gold-expansion-v2/final-subject-set-v2-batch2.json","sha256":"2713da3262e799306433f95555cdc19d14e5d54d25a976f9ed2279f07481ee19"},{"path":"validation/external-gold-expansion-v2/measurement-v2-batch2/semantic-miss-adjudication-v2-batch2.json","sha256":"064e119f4d197b5129051360f7c23ef9d7520a2074b77dc4a70daa474494db59"},{"path":"validation/external-gold-expansion-v2/measurement-v2-batch2/raw-predictions-v2-batch2.jsonl","sha256":"8c4364ab598241ddbe4adb16453ba2e12b0917a1d1852caedb07884d1a54570b"}]},{"batch_id":"external-gold-expansion-v2-batch3","benchmark_id":"external-gold-expansion-v2-batch3","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","metrics":{"frozen_case_count":36,"provider_valid_cases":36,"infrastructure_failures":0,"strict_correct":34,"strict_accuracy":0.944444,"wilson_95":{"lower":0.818553,"upper":0.98463},"semantic_coverage":1.0,"observed_semantic_miss_count":2,"strict_score":"34/36"},"measured_at":"2026-09-10T17:45:38.245198Z","latest_measured_at":"2026-09-10T17:46:44.384994Z","subject_metrics":[{"canonical_subject_id":"cloudflare-docs-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"datagouv-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"deepwiki-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"exa-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"gitmcp-docs","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":4,"strict_accuracy":0.666667,"wilson_95":{"lower":0.299993,"upper":0.903229},"semantic_coverage":1.0,"observed_semantic_miss_count":2,"strict_score":"4/6"}},{"canonical_subject_id":"microsoft-release-communications","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}}],"semantic_misses":[{"case_id":"b3-gm-005","subject_id":"gitmcp-docs","expected_tool":"fetch_generic_url_content","selected_tool":"fetch_generic_documentation","boundary_family":"DOCUMENTATION_VS_REFERENCED_URL","adjudication":"GOLD_REMAINS_VALID; semantic miss preserved in strict score. The public source supports the requested neighboring operation, and the selected tool is a plausible but incorrect neighboring retrieval mode.","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch3/measurement-v2-batch3/semantic-miss-adjudication-v2-batch3.json"},{"case_id":"b3-gm-006","subject_id":"gitmcp-docs","expected_tool":"search_generic_documentation","selected_tool":"fetch_generic_documentation","boundary_family":"DOCUMENTED_BEHAVIOR_VS_IMPLEMENTATION","adjudication":"GOLD_REMAINS_VALID; semantic miss preserved in strict score. The public source supports the requested neighboring operation, and the selected tool is a plausible but incorrect neighboring retrieval mode.","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch3/measurement-v2-batch3/semantic-miss-adjudication-v2-batch3.json"}],"evidence_identity":{"freeze_input_commit":"8450ed889756784a12ca4ebb8d84974d73b31778","final_freeze_receipt_commit":"a309bda7e8dd2867ccc7894dc7d42efbfc48395e","measurement_commit":"b81cc816158f7c0fbe7f8ac0a6fcf2b3f38f4c86","immutable_measurement_receipt_commit":"9d8290160849a5124d1896fc318d53470585ef41","case_freeze_sha256":"c1595743f48fe8ae6b3f70f2ea401731052b3bd7614297feabaa8172ce190ae5","model_lock_sha256":"48b5ed9c98f16cc2b3b76bc0da2e7ec8072ecd89ba1914683181b1c20074a3a8","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"c5cfe765f6b72489220cc73f689acb1c1c2c9284ffd8d093de0e597b645f6fbf","measurement_artifact_sha256":"7dcab9d4b71e7de59094c07cbca830ebf674b601ac6de024db0d3e677e7c0067","policy_evaluation_sha256":"3ea5b62044b4da6e92812831ebb00205970849accffe763a270248a24a0adef4"},"subject_identity_metadata":{"datagouv-mcp":{"subject_order":1,"canonical_subject_id":"datagouv-mcp","risk_group":"HIGH","provider_repository":"data.gouv.fr; https://github.com/datagouv/datagouv-mcp","mcp_url":"https://mcp.data.gouv.fr/mcp","source_schema":"schema-snapshots-batch2/datagouv-mcp/schema-v2-batch2.json","source_identity":"datagouv-readme-5a37d6f-2026-09-10","source_url":"https://github.com/datagouv/datagouv-mcp/blob/5a37d6f0f1da5504fd12918883498c26d6c28320/README.md","source_version":"repository commit 5a37d6f0f1da5504fd12918883498c26d6c28320; checked 2026-09-10","selection_rationale":"Largest qualified tool set with repeated catalog, metadata, resource, and query neighbors; useful high-risk boundary spread.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/datagouv-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"50617b0c4cfb612d275d131eaa63bbe1d778a27f4ed923ef46daf260cbed29d4","tool_count":10,"tool_names":["search_datasets","search_organizations","search_dataservices","get_dataservice_info","get_dataservice_openapi_spec","query_resource_data","get_dataset_info","list_dataset_resources","get_resource_info","get_metrics"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"},"gitmcp-docs":{"subject_order":2,"canonical_subject_id":"gitmcp-docs","risk_group":"HIGH","provider_repository":"Ido Salomon; https://github.com/idosal/git-mcp","mcp_url":"https://gitmcp.io/docs","source_schema":"schema-snapshots-batch2/gitmcp-docs/schema-v2-batch2.json","source_identity":"gitmcp-readme-c487a29-2026-05-08","source_url":"https://github.com/idosal/git-mcp/blob/c487a29895dcfcb5b672247e646426a56e2051c1/README.md","source_version":"repository commit c487a29895dcfcb5b672247e646426a56e2051c1; checked 2026-05-08","selection_rationale":"Multiple neighboring retrieval modes, including generic fetch, documentation search, code search, and library resolution, with null schemas.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/gitmcp-docs/schema-v2-batch3.json","schema_snapshot_sha256":"b90eb7608858850eff2594ab88943ab2f7be83630c7447dc1f34461a726c429b","tool_count":5,"tool_names":["match_common_libs_owner_repo_mapping","fetch_generic_documentation","search_generic_documentation","search_generic_code","fetch_generic_url_content"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"},"deepwiki-mcp":{"subject_order":3,"canonical_subject_id":"deepwiki-mcp","risk_group":"HIGH","provider_repository":"Cognition/Devin; https://github.com/CognitionAI/deepwiki","mcp_url":"https://mcp.deepwiki.com/mcp","source_schema":"schema-snapshots-batch2/deepwiki-mcp/schema-v2-batch2.json","source_identity":"deepwiki-aws-guide-2026-09-10","source_url":"https://docs.devin.ai/work-with-devin/deepwiki-mcp","source_version":"public guide and repository evidence captured 2026-09-10; repository commit 10f554ce266db71f3679a3c37eb140490384a81b","selection_rationale":"Structure, contents, and synthesized-answer tools share repository identity and exercise the observed question/structure boundary.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/deepwiki-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"bb02856d9514fd8f03237739275812ceba29916159607e92a5e29380876df479","tool_count":3,"tool_names":["ask_question","read_wiki_contents","read_wiki_structure"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"},"cloudflare-docs-mcp":{"subject_order":4,"canonical_subject_id":"cloudflare-docs-mcp","risk_group":"MEDIUM","provider_repository":"Cloudflare managed Documentation server; https://developers.cloudflare.com/agents/model-context-protocol/cloudflare/servers-for-cloudflare/","mcp_url":"https://docs.mcp.cloudflare.com/mcp","source_schema":"../selection-evidence-expansion-v1/subjects/cloudflare-docs-mcp/schema-v1.json","source_identity":"cloudflare-docs-official-2026-09-10","source_url":"https://developers.cloudflare.com/agents/model-context-protocol/cloudflare/servers-for-cloudflare/","source_version":"official pages last updated 2026-06-03 and 2026-07-28; checked 2026-09-10","selection_rationale":"A dedicated migration guide sits beside broad documentation search, providing a medium-risk control with explicit scope distinction.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/cloudflare-docs-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"5a02f1abdfa9ac3467c065d85fce654cc0debd79defebd3d4ed6ca198e1fad9c","tool_count":2,"tool_names":["search_cloudflare_documentation","migrate_pages_to_workers_guide"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"},"exa-mcp":{"subject_order":5,"canonical_subject_id":"exa-mcp","risk_group":"LOW","provider_repository":"Exa Labs; https://github.com/exa-labs/exa-mcp-server","mcp_url":"https://mcp.exa.ai/mcp","source_schema":"../selection-evidence-expansion-v1/subjects/exa-mcp/schema-v1.json","source_identity":"exa-official-reference-2026-09-10","source_url":"https://exa.ai/docs/reference/exa-mcp","source_version":"official reference crawled 2026-09-10; documentation index reports https://exa.ai/docs/llms.txt","selection_rationale":"Search and known-URL fetch are related retrieval operations but required input shapes provide comparatively explicit control separation.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/exa-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"d98db3a5825b1d97d2611369e8bfa3c63bfff19a834d14a686bd099c9a13e252","tool_count":2,"tool_names":["web_search_exa","web_fetch_exa"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"},"microsoft-release-communications":{"subject_order":6,"canonical_subject_id":"microsoft-release-communications","risk_group":"LOW","provider_repository":"Microsoft; https://github.com/MicrosoftDocs/microsoft-365-docs","mcp_url":"https://www.microsoft.com/releasecommunications/mcp","source_schema":"schema-snapshots-batch2/microsoft-release-communications/schema-v2-batch2.json","source_identity":"mrc-official-examples-2026-09-10","source_url":"https://learn.microsoft.com/en-us/microsoft-365/admin/manage/mrc-mcp?view=o365-worldwide","source_version":"documentation last updated 2026-08-25; source commit 03b138b88a789e34d5930f0a814d30ced515db4d","selection_rationale":"Azure/M365 and collection/specific-item pairs form explicit low-risk controls with distinct required identifiers or product families.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/microsoft-release-communications/schema-v2-batch3.json","schema_snapshot_sha256":"94ed8a0076091d64801956b0215cab38725da7c33aa42f5d4c023c9394ad87b4","tool_count":4,"tool_names":["get_m365_roadmap_by_id","get_azure_update_by_id","get_recent_m365_roadmaps","get_recent_azure_updates"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"},"microsoft-learn":{"subject_order":7,"canonical_subject_id":"microsoft-learn","risk_group":"RESERVE","provider_repository":"MicrosoftDocs; https://github.com/MicrosoftDocs/mcp","mcp_url":"https://learn.microsoft.com/api/mcp","source_schema":"schema-snapshots-batch2/microsoft-learn/schema-v2-batch2.json","source_identity":"microsoft-learn-official-2026-09-10","source_url":"https://learn.microsoft.com/en-us/training/support/mcp-developer-reference","source_version":"official page last updated 2025-12-03; source commit f8ffde185dfd232dbf5d187c22ce299eadc3d583","selection_rationale":"Qualified reserve with meaningful docs/code/fetch boundaries, held out to preserve a deterministic reserve and avoid overconcentrating the batch.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/microsoft-learn/schema-v2-batch3.json","schema_snapshot_sha256":"dda9b6cfc3a8a93a2f5109e0772456016e586fb8bb40cfea42a607bedac46a1a","tool_count":3,"tool_names":["microsoft_docs_search","microsoft_code_sample_search","microsoft_docs_fetch"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"}},"source_files":[{"path":"validation/external-gold-expansion-v2/batch3/measurement-v2-batch3/measurement-artifact-v2-batch3.json","sha256":"7dcab9d4b71e7de59094c07cbca830ebf674b601ac6de024db0d3e677e7c0067"},{"path":"validation/external-gold-expansion-v2/batch3/case-freeze-v2-batch3.json","sha256":"c1595743f48fe8ae6b3f70f2ea401731052b3bd7614297feabaa8172ce190ae5"},{"path":"validation/external-gold-expansion-v2/batch3/final-subject-set-v2-batch3.json","sha256":"2c84b0b96247869f90835e46c7b3922d7da664561e041bb3d1f43961afbd40e3"},{"path":"validation/external-gold-expansion-v2/batch3/measurement-v2-batch3/semantic-miss-adjudication-v2-batch3.json","sha256":"09e12b8cb3a996e51807bbe0a470cba0aec86a99a30d15674b192424e3e18308"},{"path":"validation/external-gold-expansion-v2/batch3/measurement-v2-batch3/raw-predictions-v2-batch3.jsonl","sha256":"c5cfe765f6b72489220cc73f689acb1c1c2c9284ffd8d093de0e597b645f6fbf"}]},{"batch_id":"external-gold-expansion-v2-batch4","benchmark_id":"external-gold-expansion-v2-batch4","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","metrics":{"frozen_case_count":48,"provider_valid_cases":48,"infrastructure_failures":0,"strict_correct":46,"strict_accuracy":0.958333,"wilson_95":{"lower":0.860243,"upper":0.988498,"lower_percent":86.0243,"upper_percent":98.8498},"semantic_coverage":1.0,"observed_semantic_miss_count":2,"strict_score":"46/48"},"measured_at":"2026-09-10T18:47:01.455700Z","latest_measured_at":"2026-09-10T18:48:10.224038Z","subject_metrics":[{"canonical_subject_id":"cloudflare-docs-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"datagouv-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"deepwiki-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":0.833333,"wilson_95":{"lower":0.436497,"upper":0.969947,"lower_percent":43.6497,"upper_percent":96.9947},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"5/6"}},{"canonical_subject_id":"exa-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"microsoft-learn","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"microsoft-release-communications","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"national-parks-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6"}},{"canonical_subject_id":"nixos-mcp","metrics":{"frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":0.833333,"wilson_95":{"lower":0.436497,"upper":0.969947,"lower_percent":43.6497,"upper_percent":96.9947},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"5/6"}}],"semantic_misses":[{"case_id":"b4-dw-005","subject_id":"deepwiki-mcp","expected_tool":"read_wiki_contents","selected_tool":"read_wiki_structure","boundary_family":"CONTENTS_VS_QUESTION","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch4/measurement-v2-batch4/semantic-miss-adjudication-v2-batch4.json"},{"case_id":"b4-nx-001","subject_id":"nixos-mcp","expected_tool":"home_manager_search","selected_tool":"home_manager_info","boundary_family":"HOME_MANAGER_VS_DARWIN_SEARCH","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch4/measurement-v2-batch4/semantic-miss-adjudication-v2-batch4.json"}],"evidence_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"},"subject_identity_metadata":{"nixos-mcp":{"canonical_subject_id":"nixos-mcp","provider_repository":"Accenture/mcp-bench NixOS server; source snapshot in the frozen corpus","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned MCP-Bench source; no remote contact in Batch 4.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/nixos-tools-list.json","source_identity":"nix-official-manuals-2026-09-10","source_url":"https://nix.dev/manual/nix/2.28/command-ref/new-cli/nix3-flake","source_version":"Nix 2.28.8 reference manual and current Home Manager/nix-darwin manuals accessed 2026-09-10","selection_rationale":"The frozen tool set contains neighboring NixOS, Home Manager, nix-darwin, and flake/version operations with cross-platform and granularity distinctions.","schema_snapshot_sha256":"b905b8aed0530aa0078dd080ef1e3e9de0dec005cf06a3c9e74f3910ac131185","tool_count":18,"tool_names":["nixos_search","nixos_info","nixos_channels","nixos_stats","home_manager_search","home_manager_info","home_manager_stats","home_manager_list_options","home_manager_options_by_prefix","darwin_search","darwin_info","darwin_stats","darwin_list_options","darwin_options_by_prefix","nixos_flakes_stats","nixos_flakes_search","nixhub_package_versions","nixhub_find_version"],"subject_order":1,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"national-parks-mcp":{"canonical_subject_id":"national-parks-mcp","provider_repository":"Accenture/mcp-bench national parks server; source snapshot in the frozen corpus","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned MCP-Bench source; no remote contact in Batch 4.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/national-parks-tools-list.json","source_identity":"nps-api-docs-2017-08-15","source_url":"https://www.nps.gov/subjects/developer/api-documentation.htm","source_version":"NPS API documentation last updated 2017-08-15; accessed 2026-09-10","selection_rationale":"Six public-lands operations provide independent park search, detail, alert, visitor-center, campground, and event boundaries.","schema_snapshot_sha256":"4ed812f3622705efbcba74689502fa0de6f2bd80bb5249384a4ca53a2b950de9","tool_count":6,"tool_names":["findParks","getParkDetails","getAlerts","getVisitorCenters","getCampgrounds","getEvents"],"subject_order":2,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"datagouv-mcp":{"subject_order":3,"canonical_subject_id":"datagouv-mcp","risk_group":"HIGH","provider_repository":"data.gouv.fr; https://github.com/datagouv/datagouv-mcp","mcp_url":"https://mcp.data.gouv.fr/mcp","source_schema":"schema-snapshots-batch2/datagouv-mcp/schema-v2-batch2.json","source_identity":"datagouv-readme-5a37d6f-2026-09-10","source_url":"https://github.com/datagouv/datagouv-mcp/blob/5a37d6f0f1da5504fd12918883498c26d6c28320/README.md","source_version":"repository commit 5a37d6f0f1da5504fd12918883498c26d6c28320; checked 2026-09-10","selection_rationale":"Largest qualified tool set with repeated catalog, metadata, resource, and query neighbors; useful high-risk boundary spread.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/datagouv-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"50617b0c4cfb612d275d131eaa63bbe1d778a27f4ed923ef46daf260cbed29d4","tool_count":10,"tool_names":["search_datasets","search_organizations","search_dataservices","get_dataservice_info","get_dataservice_openapi_spec","query_resource_data","get_dataset_info","list_dataset_resources","get_resource_info","get_metrics"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"deepwiki-mcp":{"subject_order":4,"canonical_subject_id":"deepwiki-mcp","risk_group":"HIGH","provider_repository":"Cognition/Devin; https://github.com/CognitionAI/deepwiki","mcp_url":"https://mcp.deepwiki.com/mcp","source_schema":"schema-snapshots-batch2/deepwiki-mcp/schema-v2-batch2.json","source_identity":"deepwiki-aws-guide-2026-09-10","source_url":"https://docs.devin.ai/work-with-devin/deepwiki-mcp","source_version":"public guide and repository evidence captured 2026-09-10; repository commit 10f554ce266db71f3679a3c37eb140490384a81b","selection_rationale":"Structure, contents, and synthesized-answer tools share repository identity and exercise the observed question/structure boundary.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/deepwiki-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"bb02856d9514fd8f03237739275812ceba29916159607e92a5e29380876df479","tool_count":3,"tool_names":["ask_question","read_wiki_contents","read_wiki_structure"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"microsoft-release-communications":{"subject_order":5,"canonical_subject_id":"microsoft-release-communications","risk_group":"LOW","provider_repository":"Microsoft; https://github.com/MicrosoftDocs/microsoft-365-docs","mcp_url":"https://www.microsoft.com/releasecommunications/mcp","source_schema":"schema-snapshots-batch2/microsoft-release-communications/schema-v2-batch2.json","source_identity":"mrc-official-examples-2026-09-10","source_url":"https://learn.microsoft.com/en-us/microsoft-365/admin/manage/mrc-mcp?view=o365-worldwide","source_version":"documentation last updated 2026-08-25; source commit 03b138b88a789e34d5930f0a814d30ced515db4d","selection_rationale":"Azure/M365 and collection/specific-item pairs form explicit low-risk controls with distinct required identifiers or product families.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/microsoft-release-communications/schema-v2-batch3.json","schema_snapshot_sha256":"94ed8a0076091d64801956b0215cab38725da7c33aa42f5d4c023c9394ad87b4","tool_count":4,"tool_names":["get_m365_roadmap_by_id","get_azure_update_by_id","get_recent_m365_roadmaps","get_recent_azure_updates"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"cloudflare-docs-mcp":{"subject_order":6,"canonical_subject_id":"cloudflare-docs-mcp","risk_group":"MEDIUM","provider_repository":"Cloudflare managed Documentation server; https://developers.cloudflare.com/agents/model-context-protocol/cloudflare/servers-for-cloudflare/","mcp_url":"https://docs.mcp.cloudflare.com/mcp","source_schema":"../selection-evidence-expansion-v1/subjects/cloudflare-docs-mcp/schema-v1.json","source_identity":"cloudflare-docs-official-2026-09-10","source_url":"https://developers.cloudflare.com/agents/model-context-protocol/cloudflare/servers-for-cloudflare/","source_version":"official pages last updated 2026-06-03 and 2026-07-28; checked 2026-09-10","selection_rationale":"A dedicated migration guide sits beside broad documentation search, providing a medium-risk control with explicit scope distinction.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/cloudflare-docs-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"5a02f1abdfa9ac3467c065d85fce654cc0debd79defebd3d4ed6ca198e1fad9c","tool_count":2,"tool_names":["search_cloudflare_documentation","migrate_pages_to_workers_guide"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"microsoft-learn":{"subject_order":7,"canonical_subject_id":"microsoft-learn","risk_group":"RESERVE","provider_repository":"MicrosoftDocs; https://github.com/MicrosoftDocs/mcp","mcp_url":"https://learn.microsoft.com/api/mcp","source_schema":"schema-snapshots-batch2/microsoft-learn/schema-v2-batch2.json","source_identity":"microsoft-learn-official-2026-09-10","source_url":"https://learn.microsoft.com/en-us/training/support/mcp-developer-reference","source_version":"official page last updated 2025-12-03; source commit f8ffde185dfd232dbf5d187c22ce299eadc3d583","selection_rationale":"Qualified reserve with meaningful docs/code/fetch boundaries, held out to preserve a deterministic reserve and avoid overconcentrating the batch.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/microsoft-learn/schema-v2-batch3.json","schema_snapshot_sha256":"dda9b6cfc3a8a93a2f5109e0772456016e586fb8bb40cfea42a607bedac46a1a","tool_count":3,"tool_names":["microsoft_docs_search","microsoft_code_sample_search","microsoft_docs_fetch"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"exa-mcp":{"subject_order":8,"canonical_subject_id":"exa-mcp","risk_group":"LOW","provider_repository":"Exa Labs; https://github.com/exa-labs/exa-mcp-server","mcp_url":"https://mcp.exa.ai/mcp","source_schema":"../selection-evidence-expansion-v1/subjects/exa-mcp/schema-v1.json","source_identity":"exa-official-reference-2026-09-10","source_url":"https://exa.ai/docs/reference/exa-mcp","source_version":"official reference crawled 2026-09-10; documentation index reports https://exa.ai/docs/llms.txt","selection_rationale":"Search and known-URL fetch are related retrieval operations but required input shapes provide comparatively explicit control separation.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/exa-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"d98db3a5825b1d97d2611369e8bfa3c63bfff19a834d14a686bd099c9a13e252","tool_count":2,"tool_names":["web_search_exa","web_fetch_exa"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"}},"source_files":[{"path":"validation/external-gold-expansion-v2/batch4/measurement-v2-batch4/measurement-artifact-v2-batch4.json","sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9"},{"path":"validation/external-gold-expansion-v2/batch4/case-freeze-v2-batch4.json","sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d"},{"path":"validation/external-gold-expansion-v2/batch4/final-subject-set-v2-batch4.json","sha256":"3dcfc1b819a14c568e012b1c72f80969624d04d092c44398bcf08d62cf7e5497"},{"path":"validation/external-gold-expansion-v2/batch4/measurement-v2-batch4/semantic-miss-adjudication-v2-batch4.json","sha256":"f12dafa8590b6f44938de43a9f070f83d0c304d868d97cb7131a6dae9e58bcdf"},{"path":"validation/external-gold-expansion-v2/batch4/measurement-v2-batch4/raw-predictions-v2-batch4.jsonl","sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89"}]},{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","metrics":{"frozen_case_count":77,"provider_valid_cases":77,"infrastructure_failures":0,"strict_correct":74,"strict_accuracy":0.961039,"wilson_95":{"lower":0.8916,"upper":0.986662,"lower_percent":89.16,"upper_percent":98.6662},"semantic_coverage":1.0,"observed_semantic_miss_count":3,"strict_score":"74/77"},"measured_at":"2026-09-10T19:13:49.842728Z","latest_measured_at":"2026-09-10T19:15:38.318428Z","subject_metrics":[{"canonical_subject_id":"cloudflare-docs-mcp","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}},{"canonical_subject_id":"datagouv-mcp","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}},{"canonical_subject_id":"deepwiki-mcp","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":4,"strict_accuracy":0.571429,"wilson_95":{"lower":0.250458,"upper":0.84178,"lower_percent":25.0458,"upper_percent":84.178},"semantic_coverage":1.0,"observed_semantic_miss_count":3,"strict_score":"4/7"}},{"canonical_subject_id":"exa-mcp","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}},{"canonical_subject_id":"google-maps-mcp","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}},{"canonical_subject_id":"metropolitan-museum-mcp","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}},{"canonical_subject_id":"microsoft-learn","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}},{"canonical_subject_id":"microsoft-release-communications","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}},{"canonical_subject_id":"national-parks-mcp","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}},{"canonical_subject_id":"nixos-mcp","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}},{"canonical_subject_id":"wikipedia-mcp-reference","metrics":{"frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7"}}],"semantic_misses":[{"case_id":"b5-dw-003","subject_id":"deepwiki-mcp","expected_tool":"read_wiki_contents","selected_tool":"read_wiki_structure","boundary_family":"CONTENTS_VS_QUESTION","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch5/measurement-v2-batch5/semantic-miss-adjudication-v2-batch5.json"},{"case_id":"b5-dw-004","subject_id":"deepwiki-mcp","expected_tool":"ask_question","selected_tool":"read_wiki_contents","boundary_family":"QUESTION_VS_CONTENTS","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch5/measurement-v2-batch5/semantic-miss-adjudication-v2-batch5.json"},{"case_id":"b5-dw-006","subject_id":"deepwiki-mcp","expected_tool":"read_wiki_contents","selected_tool":"read_wiki_structure","boundary_family":"CONTENTS_VS_STRUCTURE","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch5/measurement-v2-batch5/semantic-miss-adjudication-v2-batch5.json"}],"evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"},"subject_identity_metadata":{"datagouv-mcp":{"subject_order":1,"canonical_subject_id":"datagouv-mcp","risk_group":"HIGH","provider_repository":"data.gouv.fr; https://github.com/datagouv/datagouv-mcp","mcp_url":"https://mcp.data.gouv.fr/mcp","source_schema":"schema-snapshots-batch2/datagouv-mcp/schema-v2-batch2.json","source_identity":"datagouv-readme-5a37d6f-2026-09-10","source_url":"https://github.com/datagouv/datagouv-mcp/blob/5a37d6f0f1da5504fd12918883498c26d6c28320/README.md","source_version":"repository commit 5a37d6f0f1da5504fd12918883498c26d6c28320; checked 2026-09-10","selection_rationale":"Largest qualified tool set with repeated catalog, metadata, resource, and query neighbors; useful high-risk boundary spread.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/datagouv-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"50617b0c4cfb612d275d131eaa63bbe1d778a27f4ed923ef46daf260cbed29d4","tool_count":10,"tool_names":["search_datasets","search_organizations","search_dataservices","get_dataservice_info","get_dataservice_openapi_spec","query_resource_data","get_dataset_info","list_dataset_resources","get_resource_info","get_metrics"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"deepwiki-mcp":{"subject_order":2,"canonical_subject_id":"deepwiki-mcp","risk_group":"HIGH","provider_repository":"Cognition/Devin; https://github.com/CognitionAI/deepwiki","mcp_url":"https://mcp.deepwiki.com/mcp","source_schema":"schema-snapshots-batch2/deepwiki-mcp/schema-v2-batch2.json","source_identity":"deepwiki-aws-guide-2026-09-10","source_url":"https://docs.devin.ai/work-with-devin/deepwiki-mcp","source_version":"public guide and repository evidence captured 2026-09-10; repository commit 10f554ce266db71f3679a3c37eb140490384a81b","selection_rationale":"Structure, contents, and synthesized-answer tools share repository identity and exercise the observed question/structure boundary.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/deepwiki-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"bb02856d9514fd8f03237739275812ceba29916159607e92a5e29380876df479","tool_count":3,"tool_names":["ask_question","read_wiki_contents","read_wiki_structure"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"microsoft-release-communications":{"subject_order":3,"canonical_subject_id":"microsoft-release-communications","risk_group":"LOW","provider_repository":"Microsoft; https://github.com/MicrosoftDocs/microsoft-365-docs","mcp_url":"https://www.microsoft.com/releasecommunications/mcp","source_schema":"schema-snapshots-batch2/microsoft-release-communications/schema-v2-batch2.json","source_identity":"mrc-official-examples-2026-09-10","source_url":"https://learn.microsoft.com/en-us/microsoft-365/admin/manage/mrc-mcp?view=o365-worldwide","source_version":"documentation last updated 2026-08-25; source commit 03b138b88a789e34d5930f0a814d30ced515db4d","selection_rationale":"Azure/M365 and collection/specific-item pairs form explicit low-risk controls with distinct required identifiers or product families.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/microsoft-release-communications/schema-v2-batch3.json","schema_snapshot_sha256":"94ed8a0076091d64801956b0215cab38725da7c33aa42f5d4c023c9394ad87b4","tool_count":4,"tool_names":["get_m365_roadmap_by_id","get_azure_update_by_id","get_recent_m365_roadmaps","get_recent_azure_updates"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"microsoft-learn":{"subject_order":4,"canonical_subject_id":"microsoft-learn","risk_group":"RESERVE","provider_repository":"MicrosoftDocs; https://github.com/MicrosoftDocs/mcp","mcp_url":"https://learn.microsoft.com/api/mcp","source_schema":"schema-snapshots-batch2/microsoft-learn/schema-v2-batch2.json","source_identity":"microsoft-learn-official-2026-09-10","source_url":"https://learn.microsoft.com/en-us/training/support/mcp-developer-reference","source_version":"official page last updated 2025-12-03; source commit f8ffde185dfd232dbf5d187c22ce299eadc3d583","selection_rationale":"Qualified reserve with meaningful docs/code/fetch boundaries, held out to preserve a deterministic reserve and avoid overconcentrating the batch.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/microsoft-learn/schema-v2-batch3.json","schema_snapshot_sha256":"dda9b6cfc3a8a93a2f5109e0772456016e586fb8bb40cfea42a607bedac46a1a","tool_count":3,"tool_names":["microsoft_docs_search","microsoft_code_sample_search","microsoft_docs_fetch"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"cloudflare-docs-mcp":{"subject_order":5,"canonical_subject_id":"cloudflare-docs-mcp","risk_group":"MEDIUM","provider_repository":"Cloudflare managed Documentation server; https://developers.cloudflare.com/agents/model-context-protocol/cloudflare/servers-for-cloudflare/","mcp_url":"https://docs.mcp.cloudflare.com/mcp","source_schema":"../selection-evidence-expansion-v1/subjects/cloudflare-docs-mcp/schema-v1.json","source_identity":"cloudflare-docs-official-2026-09-10","source_url":"https://developers.cloudflare.com/agents/model-context-protocol/cloudflare/servers-for-cloudflare/","source_version":"official pages last updated 2026-06-03 and 2026-07-28; checked 2026-09-10","selection_rationale":"A dedicated migration guide sits beside broad documentation search, providing a medium-risk control with explicit scope distinction.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/cloudflare-docs-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"5a02f1abdfa9ac3467c065d85fce654cc0debd79defebd3d4ed6ca198e1fad9c","tool_count":2,"tool_names":["search_cloudflare_documentation","migrate_pages_to_workers_guide"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"exa-mcp":{"subject_order":6,"canonical_subject_id":"exa-mcp","risk_group":"LOW","provider_repository":"Exa Labs; https://github.com/exa-labs/exa-mcp-server","mcp_url":"https://mcp.exa.ai/mcp","source_schema":"../selection-evidence-expansion-v1/subjects/exa-mcp/schema-v1.json","source_identity":"exa-official-reference-2026-09-10","source_url":"https://exa.ai/docs/reference/exa-mcp","source_version":"official reference crawled 2026-09-10; documentation index reports https://exa.ai/docs/llms.txt","selection_rationale":"Search and known-URL fetch are related retrieval operations but required input shapes provide comparatively explicit control separation.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/exa-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"d98db3a5825b1d97d2611369e8bfa3c63bfff19a834d14a686bd099c9a13e252","tool_count":2,"tool_names":["web_search_exa","web_fetch_exa"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"nixos-mcp":{"canonical_subject_id":"nixos-mcp","provider_repository":"Accenture/mcp-bench NixOS server; source snapshot in the frozen corpus","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned MCP-Bench source; no remote contact in Batch 4.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/nixos-tools-list.json","source_identity":"nix-official-manuals-2026-09-10","source_url":"https://nix.dev/manual/nix/2.28/command-ref/new-cli/nix3-flake","source_version":"Nix 2.28.8 reference manual and current Home Manager/nix-darwin manuals accessed 2026-09-10","selection_rationale":"The frozen tool set contains neighboring NixOS, Home Manager, nix-darwin, and flake/version operations with cross-platform and granularity distinctions.","schema_snapshot_sha256":"b905b8aed0530aa0078dd080ef1e3e9de0dec005cf06a3c9e74f3910ac131185","tool_count":18,"tool_names":["nixos_search","nixos_info","nixos_channels","nixos_stats","home_manager_search","home_manager_info","home_manager_stats","home_manager_list_options","home_manager_options_by_prefix","darwin_search","darwin_info","darwin_stats","darwin_list_options","darwin_options_by_prefix","nixos_flakes_stats","nixos_flakes_search","nixhub_package_versions","nixhub_find_version"],"subject_order":7,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"national-parks-mcp":{"canonical_subject_id":"national-parks-mcp","provider_repository":"Accenture/mcp-bench national parks server; source snapshot in the frozen corpus","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned MCP-Bench source; no remote contact in Batch 4.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/national-parks-tools-list.json","source_identity":"nps-api-docs-2017-08-15","source_url":"https://www.nps.gov/subjects/developer/api-documentation.htm","source_version":"NPS API documentation last updated 2017-08-15; accessed 2026-09-10","selection_rationale":"Six public-lands operations provide independent park search, detail, alert, visitor-center, campground, and event boundaries.","schema_snapshot_sha256":"4ed812f3622705efbcba74689502fa0de6f2bd80bb5249384a4ca53a2b950de9","tool_count":6,"tool_names":["findParks","getParkDetails","getAlerts","getVisitorCenters","getCampgrounds","getEvents"],"subject_order":8,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"wikipedia-mcp-reference":{"canonical_subject_id":"wikipedia-mcp-reference","provider_repository":"stable corpus / wikipedia MCP reconstruction","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned stable corpus; no remote contact in Batch 5.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/wikipedia-tools-list.json","schema_snapshot_sha256":"ba0abc3b84718f0f3b73ece26ae93f12513ef37027516dd75d19cce837cdcf65","tool_count":9,"tool_names":["search_wikipedia","get_article","get_summary","summarize_article_for_query","summarize_article_section","extract_key_facts","get_related_topics","get_sections","get_links"],"source_identity":"mediawiki-api-official-2026-09-10","source_url":"https://www.mediawiki.org/wiki/API:Search","source_version":"MediaWiki API documentation accessed 2026-09-10","subject_order":9,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"google-maps-mcp":{"canonical_subject_id":"google-maps-mcp","provider_repository":"stable corpus / Google Maps MCP reconstruction","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned stable corpus; no remote contact in Batch 5.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/phase-3/google-maps-tools-list.json","schema_snapshot_sha256":"fca473a69e8a5732bd84ed01049b9ba1bd2e7e706c073e643040145cca92a3a1","tool_count":7,"tool_names":["search_nearby","get_place_details","maps_geocode","maps_reverse_geocode","maps_distance_matrix","maps_directions","maps_elevation"],"source_identity":"google-maps-platform-official-2026-09-10","source_url":"https://developers.google.com/maps/documentation","source_version":"Google Maps Platform documentation accessed 2026-09-10","subject_order":10,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"},"metropolitan-museum-mcp":{"canonical_subject_id":"metropolitan-museum-mcp","provider_repository":"stable corpus / Metropolitan Museum MCP reconstruction","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned stable corpus; no remote contact in Batch 5.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/phase-3/metropolitan-museum-tools-list.json","schema_snapshot_sha256":"0ef880de13ed23c83110318eff4dd339756a3d8f4c8384c820476c54a5cfe2b2","tool_count":3,"tool_names":["list-departments","search-museum-objects","get-museum-object"],"source_identity":"met-collection-api-official-2026-09-10","source_url":"https://metmuseum.github.io/","source_version":"Met Collection API documentation accessed 2026-09-10","subject_order":11,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1"}},"source_files":[{"path":"validation/external-gold-expansion-v2/batch5/measurement-v2-batch5/measurement-artifact-v2-batch5.json","sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb"},{"path":"validation/external-gold-expansion-v2/batch5/case-freeze-v2-batch5.json","sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc"},{"path":"validation/external-gold-expansion-v2/batch5/final-subject-set-v2-batch5.json","sha256":"2881b264136944f2df33463cc8f3f4618e3bf200ce3d4660be4358464e9ad151"},{"path":"validation/external-gold-expansion-v2/batch5/measurement-v2-batch5/semantic-miss-adjudication-v2-batch5.json","sha256":"32646e9f45f4931c98a6ca5a646761cc6b9595a7b941e6956605b25e72142f6f"},{"path":"validation/external-gold-expansion-v2/batch5/measurement-v2-batch5/raw-predictions-v2-batch5.jsonl","sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68"}]},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","metrics":{"frozen_case_count":80,"provider_valid_cases":80,"infrastructure_failures":0,"strict_correct":80,"strict_accuracy":1.0,"wilson_95":{"lower":0.954182,"upper":1.0,"lower_percent":95.4182,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"80/80"},"measured_at":"2026-09-11T00:07:39.419700Z","latest_measured_at":"2026-09-11T00:10:04.575675Z","subject_metrics":[{"canonical_subject_id":"cloudflare-docs-mcp","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}},{"canonical_subject_id":"datagouv-mcp","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}},{"canonical_subject_id":"exa-mcp","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}},{"canonical_subject_id":"google-maps-mcp","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}},{"canonical_subject_id":"metropolitan-museum-mcp","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}},{"canonical_subject_id":"microsoft-learn","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}},{"canonical_subject_id":"microsoft-release-communications","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}},{"canonical_subject_id":"national-parks-mcp","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}},{"canonical_subject_id":"nixos-mcp","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}},{"canonical_subject_id":"wikipedia-mcp-reference","metrics":{"frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8"}}],"semantic_misses":[],"evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"},"subject_identity_metadata":{"datagouv-mcp":{"subject_order":1,"canonical_subject_id":"datagouv-mcp","risk_group":"HIGH","provider_repository":"data.gouv.fr; https://github.com/datagouv/datagouv-mcp","mcp_url":"https://mcp.data.gouv.fr/mcp","source_schema":"schema-snapshots-batch2/datagouv-mcp/schema-v2-batch2.json","source_identity":"datagouv-readme-5a37d6f-2026-09-10","source_url":"https://github.com/datagouv/datagouv-mcp/blob/5a37d6f0f1da5504fd12918883498c26d6c28320/README.md","source_version":"repository commit 5a37d6f0f1da5504fd12918883498c26d6c28320; checked 2026-09-10","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/datagouv-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"50617b0c4cfb612d275d131eaa63bbe1d778a27f4ed923ef46daf260cbed29d4","tool_count":10,"tool_names":["search_datasets","search_organizations","search_dataservices","get_dataservice_info","get_dataservice_openapi_spec","query_resource_data","get_dataset_info","list_dataset_resources","get_resource_info","get_metrics"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED"},"microsoft-release-communications":{"subject_order":2,"canonical_subject_id":"microsoft-release-communications","risk_group":"LOW","provider_repository":"Microsoft; https://github.com/MicrosoftDocs/microsoft-365-docs","mcp_url":"https://www.microsoft.com/releasecommunications/mcp","source_schema":"schema-snapshots-batch2/microsoft-release-communications/schema-v2-batch2.json","source_identity":"mrc-official-examples-2026-09-10","source_url":"https://learn.microsoft.com/en-us/microsoft-365/admin/manage/mrc-mcp?view=o365-worldwide","source_version":"documentation last updated 2026-08-25; source commit 03b138b88a789e34d5930f0a814d30ced515db4d","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/microsoft-release-communications/schema-v2-batch3.json","schema_snapshot_sha256":"94ed8a0076091d64801956b0215cab38725da7c33aa42f5d4c023c9394ad87b4","tool_count":4,"tool_names":["get_m365_roadmap_by_id","get_azure_update_by_id","get_recent_m365_roadmaps","get_recent_azure_updates"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED"},"microsoft-learn":{"subject_order":3,"canonical_subject_id":"microsoft-learn","risk_group":"RESERVE","provider_repository":"MicrosoftDocs; https://github.com/MicrosoftDocs/mcp","mcp_url":"https://learn.microsoft.com/api/mcp","source_schema":"schema-snapshots-batch2/microsoft-learn/schema-v2-batch2.json","source_identity":"microsoft-learn-official-2026-09-10","source_url":"https://learn.microsoft.com/en-us/training/support/mcp-developer-reference","source_version":"official page last updated 2025-12-03; source commit f8ffde185dfd232dbf5d187c22ce299eadc3d583","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/microsoft-learn/schema-v2-batch3.json","schema_snapshot_sha256":"dda9b6cfc3a8a93a2f5109e0772456016e586fb8bb40cfea42a607bedac46a1a","tool_count":3,"tool_names":["microsoft_docs_search","microsoft_code_sample_search","microsoft_docs_fetch"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED"},"cloudflare-docs-mcp":{"subject_order":4,"canonical_subject_id":"cloudflare-docs-mcp","risk_group":"MEDIUM","provider_repository":"Cloudflare managed Documentation server; https://developers.cloudflare.com/agents/model-context-protocol/cloudflare/servers-for-cloudflare/","mcp_url":"https://docs.mcp.cloudflare.com/mcp","source_schema":"../selection-evidence-expansion-v1/subjects/cloudflare-docs-mcp/schema-v1.json","source_identity":"cloudflare-docs-official-2026-09-10","source_url":"https://developers.cloudflare.com/agents/model-context-protocol/cloudflare/servers-for-cloudflare/","source_version":"official pages last updated 2026-06-03 and 2026-07-28; checked 2026-09-10","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/cloudflare-docs-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"5a02f1abdfa9ac3467c065d85fce654cc0debd79defebd3d4ed6ca198e1fad9c","tool_count":2,"tool_names":["search_cloudflare_documentation","migrate_pages_to_workers_guide"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED"},"exa-mcp":{"subject_order":5,"canonical_subject_id":"exa-mcp","risk_group":"LOW","provider_repository":"Exa Labs; https://github.com/exa-labs/exa-mcp-server","mcp_url":"https://mcp.exa.ai/mcp","source_schema":"../selection-evidence-expansion-v1/subjects/exa-mcp/schema-v1.json","source_identity":"exa-official-reference-2026-09-10","source_url":"https://exa.ai/docs/reference/exa-mcp","source_version":"official reference crawled 2026-09-10; documentation index reports https://exa.ai/docs/llms.txt","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss.","schema_snapshot_path":"validation/external-gold-expansion-v2/schema-snapshots-batch3/exa-mcp/schema-v2-batch3.json","schema_snapshot_sha256":"d98db3a5825b1d97d2611369e8bfa3c63bfff19a834d14a686bd099c9a13e252","tool_count":2,"tool_names":["web_search_exa","web_fetch_exa"],"identity_mode":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS","scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED"},"nixos-mcp":{"canonical_subject_id":"nixos-mcp","provider_repository":"Accenture/mcp-bench NixOS server; source snapshot in the frozen corpus","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned MCP-Bench source; no remote contact in Batch 4.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/nixos-tools-list.json","source_identity":"nix-official-manuals-2026-09-10","source_url":"https://nix.dev/manual/nix/2.28/command-ref/new-cli/nix3-flake","source_version":"Nix 2.28.8 reference manual and current Home Manager/nix-darwin manuals accessed 2026-09-10","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss.","schema_snapshot_sha256":"b905b8aed0530aa0078dd080ef1e3e9de0dec005cf06a3c9e74f3910ac131185","tool_count":18,"tool_names":["nixos_search","nixos_info","nixos_channels","nixos_stats","home_manager_search","home_manager_info","home_manager_stats","home_manager_list_options","home_manager_options_by_prefix","darwin_search","darwin_info","darwin_stats","darwin_list_options","darwin_options_by_prefix","nixos_flakes_stats","nixos_flakes_search","nixhub_package_versions","nixhub_find_version"],"subject_order":6,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED"},"national-parks-mcp":{"canonical_subject_id":"national-parks-mcp","provider_repository":"Accenture/mcp-bench national parks server; source snapshot in the frozen corpus","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned MCP-Bench source; no remote contact in Batch 4.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/national-parks-tools-list.json","source_identity":"nps-api-docs-2017-08-15","source_url":"https://www.nps.gov/subjects/developer/api-documentation.htm","source_version":"NPS API documentation last updated 2017-08-15; accessed 2026-09-10","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss.","schema_snapshot_sha256":"4ed812f3622705efbcba74689502fa0de6f2bd80bb5249384a4ca53a2b950de9","tool_count":6,"tool_names":["findParks","getParkDetails","getAlerts","getVisitorCenters","getCampgrounds","getEvents"],"subject_order":7,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED"},"wikipedia-mcp-reference":{"canonical_subject_id":"wikipedia-mcp-reference","provider_repository":"stable corpus / wikipedia MCP reconstruction","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned stable corpus; no remote contact in Batch 5.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/wikipedia-tools-list.json","schema_snapshot_sha256":"ba0abc3b84718f0f3b73ece26ae93f12513ef37027516dd75d19cce837cdcf65","tool_count":9,"tool_names":["search_wikipedia","get_article","get_summary","summarize_article_for_query","summarize_article_section","extract_key_facts","get_related_topics","get_sections","get_links"],"source_identity":"mediawiki-api-official-2026-09-10","source_url":"https://www.mediawiki.org/wiki/API:Search","source_version":"MediaWiki API documentation accessed 2026-09-10","subject_order":8,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss."},"google-maps-mcp":{"canonical_subject_id":"google-maps-mcp","provider_repository":"stable corpus / Google Maps MCP reconstruction","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned stable corpus; no remote contact in Batch 5.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/phase-3/google-maps-tools-list.json","schema_snapshot_sha256":"fca473a69e8a5732bd84ed01049b9ba1bd2e7e706c073e643040145cca92a3a1","tool_count":7,"tool_names":["search_nearby","get_place_details","maps_geocode","maps_reverse_geocode","maps_distance_matrix","maps_directions","maps_elevation"],"source_identity":"google-maps-platform-official-2026-09-10","source_url":"https://developers.google.com/maps/documentation","source_version":"Google Maps Platform documentation accessed 2026-09-10","subject_order":9,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss."},"metropolitan-museum-mcp":{"canonical_subject_id":"metropolitan-museum-mcp","provider_repository":"stable corpus / Metropolitan Museum MCP reconstruction","mcp_url":"version-bound-local-reconstruction","identity_mode":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT","schema_acquisition":"Existing immutable tools/list reconstruction from the pinned stable corpus; no remote contact in Batch 5.","schema_snapshot_path":"validation/independent-holdout-v1/external-gold-v1/reconstruction/phase-3/metropolitan-museum-tools-list.json","schema_snapshot_sha256":"0ef880de13ed23c83110318eff4dd339756a3d8f4c8384c820476c54a5cfe2b2","tool_count":3,"tool_names":["list-departments","search-museum-objects","get-museum-object"],"source_identity":"met-collection-api-official-2026-09-10","source_url":"https://metmuseum.github.io/","source_version":"Met Collection API documentation accessed 2026-09-10","subject_order":10,"scanner_id":"BOUNDARY_RISK_SCAN_V1","scanner_version":"mcp-selection-lab.boundary-risk-scan.v1","selection_status":"PRIMARY_HOLDOUT_SELECTED","selection_rationale":"Qualified non-DeepWiki subject with externally auditable documentation, multiple tool boundaries, and no prior Batch 5 semantic miss."}},"source_files":[{"path":"validation/external-gold-expansion-v2/batch6/measurement-v2-batch6/measurement-artifact-v2-batch6.json","sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912"},{"path":"validation/external-gold-expansion-v2/batch6/case-freeze-v2-batch6.json","sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b"},{"path":"validation/external-gold-expansion-v2/batch6/final-subject-set-v2-batch6.json","sha256":"26355ec41096de990187e1a55d05c9478e4701754372bebfd5b07bb08acdcba6"},{"path":"validation/external-gold-expansion-v2/batch6/measurement-v2-batch6/semantic-miss-adjudication-v2-batch6.json","sha256":"354a0e94289c15bf8ad0cfaf2ba0e019fddc050979be47693515cb4f43fa77a9"},{"path":"validation/external-gold-expansion-v2/batch6/measurement-v2-batch6/raw-predictions-v2-batch6.jsonl","sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51"}]}],"subject_records":[{"canonical_subject_id":"cloudflare-docs-mcp","subject_name":"Cloudflare Docs MCP","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"MIXED","identity_variants":[{"canonical_subject_id":"cloudflare-docs-mcp","subject_name":"Cloudflare Docs MCP","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"LIVE_TOOLS_LIST"},{"canonical_subject_id":"cloudflare-docs-mcp","subject_name":"Cloudflare Docs","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"}],"number_of_independent_benchmark_batches":5,"total_measured_frozen_cases":51,"total_provider_valid_cases":51,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}},{"batch_id":"external-gold-expansion-v2-batch3","benchmark_id":"external-gold-expansion-v2-batch3","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T17:45:38.245198Z","evidence_identity":{"freeze_input_commit":"8450ed889756784a12ca4ebb8d84974d73b31778","final_freeze_receipt_commit":"a309bda7e8dd2867ccc7894dc7d42efbfc48395e","measurement_commit":"b81cc816158f7c0fbe7f8ac0a6fcf2b3f38f4c86","immutable_measurement_receipt_commit":"9d8290160849a5124d1896fc318d53470585ef41","case_freeze_sha256":"c1595743f48fe8ae6b3f70f2ea401731052b3bd7614297feabaa8172ce190ae5","model_lock_sha256":"48b5ed9c98f16cc2b3b76bc0da2e7ec8072ecd89ba1914683181b1c20074a3a8","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"c5cfe765f6b72489220cc73f689acb1c1c2c9284ffd8d093de0e597b645f6fbf","measurement_artifact_sha256":"7dcab9d4b71e7de59094c07cbca830ebf674b601ac6de024db0d3e677e7c0067","policy_evaluation_sha256":"3ea5b62044b4da6e92812831ebb00205970849accffe763a270248a24a0adef4"}},{"batch_id":"external-gold-expansion-v2-batch4","benchmark_id":"external-gold-expansion-v2-batch4","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T18:47:01.455700Z","evidence_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"}},{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S1","S2"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":51,"number_of_batches":5,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"context7","subject_name":"Context7","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_variants":[{"canonical_subject_id":"context7","subject_name":"Context7","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"LIVE_TOOLS_LIST"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":24,"total_provider_valid_cases":24,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S1"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MEASURED","summary":{"total_measured_cases":24,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"datagouv-mcp","subject_name":"data.gouv.fr","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"MIXED","identity_variants":[{"canonical_subject_id":"datagouv-mcp","subject_name":"data.gouv.fr","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_SNAPSHOT"},{"canonical_subject_id":"datagouv-mcp","subject_name":"data.gouv.fr","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT"},{"canonical_subject_id":"datagouv-mcp","subject_name":"data.gouv.fr","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"}],"number_of_independent_benchmark_batches":6,"total_measured_frozen_cases":39,"total_provider_valid_cases":39,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-expansion-v2-batch1","benchmark_id":"external-gold-expansion-v2-batch1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.6096657120978346,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T15:38:03.968297Z","evidence_identity":{"freeze_input_commit":"c614ae51750d12ce113b5e3d51429f52e529f08c","final_freeze_receipt_commit":"b691170","measurement_commit":"b9f358b1a65aba348fa1c0e625eac6da1f31b651","policy_evaluation_commit":"3e560e5767f181831c65569b893ca789cd3bdff0","case_freeze_sha256":"17777d2c3983cd444353b8d69dcc98836a1909ee6fde6f5c8cf18cf9b059427e","model_lock_sha256":"c4330165deb3d8587e224a2ca1b3d886db60801ca436710a0edace55525923d2","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ab085cedf13eef5724f4e160b0f49f8402d62bc5b10203de0f8d81e41f2a017e","measurement_artifact_sha256":"01243422aa946296f235810aaee6f065ec21d9e78a01017ee8161e5e61a8b5e8","policy_evaluation_sha256":"4e710a9c5205660cc07603d2fd7538aa063eda4c2574c29308f9f255e4706f82"}},{"batch_id":"external-gold-expansion-v2-batch2","benchmark_id":"external-gold-expansion-v2-batch2","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":0.8333333333333334,"wilson_95":{"lower":0.43649717781352987,"upper":0.9699466302516935},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"5/6","measured_at":"2026-09-10T16:49:53.346446Z","evidence_identity":{"freeze_input_commit":"61f6511c41562c24019cd441b37bc6ee07283951","final_freeze_receipt_commit":"d3c55d2f96a4e45bf2577225ea16929877f2ded1","measurement_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","policy_evaluation_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","case_freeze_sha256":"ce21fc085bd9a0185c9e37d0d795221f286397cbc477ee941e876be24f7ba1b4","model_lock_sha256":"153647f94eed0ee3d3069229545846eef864fd43fa821f2d311dc3fb5d265ba9","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"8c4364ab598241ddbe4adb16453ba2e12b0917a1d1852caedb07884d1a54570b","measurement_artifact_sha256":"dd13aec8a90b2b5490c934c9ad9650c37733c65f18d824e2ae6066202176ced9","policy_evaluation_sha256":"736f154a5d5c3b8f0d77cd58fa0644ca97ad94c7a6b46af98d394c1dc1578f9a"}},{"batch_id":"external-gold-expansion-v2-batch3","benchmark_id":"external-gold-expansion-v2-batch3","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T17:45:38.245198Z","evidence_identity":{"freeze_input_commit":"8450ed889756784a12ca4ebb8d84974d73b31778","final_freeze_receipt_commit":"a309bda7e8dd2867ccc7894dc7d42efbfc48395e","measurement_commit":"b81cc816158f7c0fbe7f8ac0a6fcf2b3f38f4c86","immutable_measurement_receipt_commit":"9d8290160849a5124d1896fc318d53470585ef41","case_freeze_sha256":"c1595743f48fe8ae6b3f70f2ea401731052b3bd7614297feabaa8172ce190ae5","model_lock_sha256":"48b5ed9c98f16cc2b3b76bc0da2e7ec8072ecd89ba1914683181b1c20074a3a8","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"c5cfe765f6b72489220cc73f689acb1c1c2c9284ffd8d093de0e597b645f6fbf","measurement_artifact_sha256":"7dcab9d4b71e7de59094c07cbca830ebf674b601ac6de024db0d3e677e7c0067","policy_evaluation_sha256":"3ea5b62044b4da6e92812831ebb00205970849accffe763a270248a24a0adef4"}},{"batch_id":"external-gold-expansion-v2-batch4","benchmark_id":"external-gold-expansion-v2-batch4","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T18:47:01.455700Z","evidence_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"}},{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":1,"observed_semantic_miss_case_ids":["datagouv-hard-004"],"observed_semantic_misses":[{"case_id":"datagouv-hard-004","subject_id":"datagouv-mcp","expected_tool":"list_dataset_resources","selected_tool":"get_dataset_info","boundary_family":"DATASET_VS_RESOURCE_LIST","adjudication":"SEMANTIC_MISS_GOLD_REMAINS_VALID","source_adjudication_artifact":"validation/external-gold-expansion-v2/measurement-v2-batch2/semantic-miss-adjudication-v2-batch2.json","benchmark_id":"external-gold-expansion-v2-batch2","batch_id":"external-gold-expansion-v2-batch2"}],"observed_miss_boundary_families":["DATASET_VS_RESOURCE_LIST"],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T15:38:03.968297Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":39,"number_of_batches":6,"strict_accuracy_range_across_batches":{"minimum":0.8333333333333334,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":0.8333333333333334}},{"canonical_subject_id":"deepwiki-mcp","subject_name":"DeepWiki MCP","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"MIXED","identity_variants":[{"canonical_subject_id":"deepwiki-mcp","subject_name":"DeepWiki MCP","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"LIVE_TOOLS_LIST"},{"canonical_subject_id":"deepwiki-mcp","subject_name":"DeepWiki","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_INHERITED_V1_AND_FROZEN_BY_COPY"},{"canonical_subject_id":"deepwiki-mcp","subject_name":"DeepWiki","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT"},{"canonical_subject_id":"deepwiki-mcp","subject_name":"DeepWiki","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"}],"number_of_independent_benchmark_batches":6,"total_measured_frozen_cases":52,"total_provider_valid_cases":52,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}},{"batch_id":"external-gold-expansion-v2-batch1","benchmark_id":"external-gold-expansion-v2-batch1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":4,"provider_valid_cases":4,"infrastructure_failures":0,"strict_correct":3,"strict_accuracy":0.75,"wilson_95":{"lower":0.30064184258240184,"upper":0.9544127391902995},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"3/4","measured_at":"2026-09-10T15:38:03.968297Z","evidence_identity":{"freeze_input_commit":"c614ae51750d12ce113b5e3d51429f52e529f08c","final_freeze_receipt_commit":"b691170","measurement_commit":"b9f358b1a65aba348fa1c0e625eac6da1f31b651","policy_evaluation_commit":"3e560e5767f181831c65569b893ca789cd3bdff0","case_freeze_sha256":"17777d2c3983cd444353b8d69dcc98836a1909ee6fde6f5c8cf18cf9b059427e","model_lock_sha256":"c4330165deb3d8587e224a2ca1b3d886db60801ca436710a0edace55525923d2","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ab085cedf13eef5724f4e160b0f49f8402d62bc5b10203de0f8d81e41f2a017e","measurement_artifact_sha256":"01243422aa946296f235810aaee6f065ec21d9e78a01017ee8161e5e61a8b5e8","policy_evaluation_sha256":"4e710a9c5205660cc07603d2fd7538aa063eda4c2574c29308f9f255e4706f82"}},{"batch_id":"external-gold-expansion-v2-batch2","benchmark_id":"external-gold-expansion-v2-batch2","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5","measured_at":"2026-09-10T16:49:53.346446Z","evidence_identity":{"freeze_input_commit":"61f6511c41562c24019cd441b37bc6ee07283951","final_freeze_receipt_commit":"d3c55d2f96a4e45bf2577225ea16929877f2ded1","measurement_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","policy_evaluation_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","case_freeze_sha256":"ce21fc085bd9a0185c9e37d0d795221f286397cbc477ee941e876be24f7ba1b4","model_lock_sha256":"153647f94eed0ee3d3069229545846eef864fd43fa821f2d311dc3fb5d265ba9","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"8c4364ab598241ddbe4adb16453ba2e12b0917a1d1852caedb07884d1a54570b","measurement_artifact_sha256":"dd13aec8a90b2b5490c934c9ad9650c37733c65f18d824e2ae6066202176ced9","policy_evaluation_sha256":"736f154a5d5c3b8f0d77cd58fa0644ca97ad94c7a6b46af98d394c1dc1578f9a"}},{"batch_id":"external-gold-expansion-v2-batch3","benchmark_id":"external-gold-expansion-v2-batch3","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T17:45:38.245198Z","evidence_identity":{"freeze_input_commit":"8450ed889756784a12ca4ebb8d84974d73b31778","final_freeze_receipt_commit":"a309bda7e8dd2867ccc7894dc7d42efbfc48395e","measurement_commit":"b81cc816158f7c0fbe7f8ac0a6fcf2b3f38f4c86","immutable_measurement_receipt_commit":"9d8290160849a5124d1896fc318d53470585ef41","case_freeze_sha256":"c1595743f48fe8ae6b3f70f2ea401731052b3bd7614297feabaa8172ce190ae5","model_lock_sha256":"48b5ed9c98f16cc2b3b76bc0da2e7ec8072ecd89ba1914683181b1c20074a3a8","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"c5cfe765f6b72489220cc73f689acb1c1c2c9284ffd8d093de0e597b645f6fbf","measurement_artifact_sha256":"7dcab9d4b71e7de59094c07cbca830ebf674b601ac6de024db0d3e677e7c0067","policy_evaluation_sha256":"3ea5b62044b4da6e92812831ebb00205970849accffe763a270248a24a0adef4"}},{"batch_id":"external-gold-expansion-v2-batch4","benchmark_id":"external-gold-expansion-v2-batch4","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":0.833333,"wilson_95":{"lower":0.436497,"upper":0.969947,"lower_percent":43.6497,"upper_percent":96.9947},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"5/6","measured_at":"2026-09-10T18:47:01.455700Z","evidence_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"}},{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":4,"strict_accuracy":0.571429,"wilson_95":{"lower":0.250458,"upper":0.84178,"lower_percent":25.0458,"upper_percent":84.178},"semantic_coverage":1.0,"observed_semantic_miss_count":3,"strict_score":"4/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}}],"observed_semantic_miss_count":5,"observed_semantic_miss_case_ids":["deepwiki-004","b4-dw-005","b5-dw-003","b5-dw-004","b5-dw-006"],"observed_semantic_misses":[{"case_id":"deepwiki-004","subject_id":"deepwiki-mcp","expected_tool":"ask_question","selected_tool":"read_wiki_structure","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"SEMANTIC_MISS","source_adjudication_artifact":"validation/external-gold-expansion-v2/measurement-v2-batch1/semantic-miss-adjudication-v2-batch1.json","benchmark_id":"external-gold-expansion-v2-batch1","batch_id":"external-gold-expansion-v2-batch1"},{"case_id":"b4-dw-005","subject_id":"deepwiki-mcp","expected_tool":"read_wiki_contents","selected_tool":"read_wiki_structure","boundary_family":"CONTENTS_VS_QUESTION","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch4/measurement-v2-batch4/semantic-miss-adjudication-v2-batch4.json","benchmark_id":"external-gold-expansion-v2-batch4","batch_id":"external-gold-expansion-v2-batch4"},{"case_id":"b5-dw-003","subject_id":"deepwiki-mcp","expected_tool":"read_wiki_contents","selected_tool":"read_wiki_structure","boundary_family":"CONTENTS_VS_QUESTION","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch5/measurement-v2-batch5/semantic-miss-adjudication-v2-batch5.json","benchmark_id":"external-gold-expansion-v2-batch5","batch_id":"external-gold-expansion-v2-batch5"},{"case_id":"b5-dw-004","subject_id":"deepwiki-mcp","expected_tool":"ask_question","selected_tool":"read_wiki_contents","boundary_family":"QUESTION_VS_CONTENTS","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch5/measurement-v2-batch5/semantic-miss-adjudication-v2-batch5.json","benchmark_id":"external-gold-expansion-v2-batch5","batch_id":"external-gold-expansion-v2-batch5"},{"case_id":"b5-dw-006","subject_id":"deepwiki-mcp","expected_tool":"read_wiki_contents","selected_tool":"read_wiki_structure","boundary_family":"CONTENTS_VS_STRUCTURE","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch5/measurement-v2-batch5/semantic-miss-adjudication-v2-batch5.json","benchmark_id":"external-gold-expansion-v2-batch5","batch_id":"external-gold-expansion-v2-batch5"}],"observed_miss_boundary_families":["CONTENTS_VS_QUESTION","CONTENTS_VS_STRUCTURE","NOT_RECORDED_IN_SOURCE_EVIDENCE","QUESTION_VS_CONTENTS"],"boundary_family_data_complete":false,"evidence_levels_represented":["S1","S2"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-10T19:13:49.842728Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-10T19:13:49.842728Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":52,"number_of_batches":6,"strict_accuracy_range_across_batches":{"minimum":0.571429,"maximum":1.0},"latest_strict_accuracy":0.571429,"lowest_observed_strict_accuracy":0.571429}},{"canonical_subject_id":"dex-paprika-mcp-source-snapshot","subject_name":"DEX Paprika MCP (source snapshot)","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"dex-paprika-mcp-source-snapshot","subject_name":"DEX Paprika MCP (source snapshot)","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":24,"total_provider_valid_cases":24,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S1"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MEASURED","summary":{"total_measured_cases":24,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"exa-mcp","subject_name":"Exa MCP","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"MIXED","identity_variants":[{"canonical_subject_id":"exa-mcp","subject_name":"Exa MCP","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"LIVE_TOOLS_LIST"},{"canonical_subject_id":"exa-mcp","subject_name":"Exa","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_INHERITED_V1_AND_FROZEN_BY_COPY"},{"canonical_subject_id":"exa-mcp","subject_name":"Exa","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"}],"number_of_independent_benchmark_batches":6,"total_measured_frozen_cases":55,"total_provider_valid_cases":55,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}},{"batch_id":"external-gold-expansion-v2-batch1","benchmark_id":"external-gold-expansion-v2-batch1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":4,"provider_valid_cases":4,"infrastructure_failures":0,"strict_correct":4,"strict_accuracy":1.0,"wilson_95":{"lower":0.5101091635454027,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"4/4","measured_at":"2026-09-10T15:38:03.968297Z","evidence_identity":{"freeze_input_commit":"c614ae51750d12ce113b5e3d51429f52e529f08c","final_freeze_receipt_commit":"b691170","measurement_commit":"b9f358b1a65aba348fa1c0e625eac6da1f31b651","policy_evaluation_commit":"3e560e5767f181831c65569b893ca789cd3bdff0","case_freeze_sha256":"17777d2c3983cd444353b8d69dcc98836a1909ee6fde6f5c8cf18cf9b059427e","model_lock_sha256":"c4330165deb3d8587e224a2ca1b3d886db60801ca436710a0edace55525923d2","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ab085cedf13eef5724f4e160b0f49f8402d62bc5b10203de0f8d81e41f2a017e","measurement_artifact_sha256":"01243422aa946296f235810aaee6f065ec21d9e78a01017ee8161e5e61a8b5e8","policy_evaluation_sha256":"4e710a9c5205660cc07603d2fd7538aa063eda4c2574c29308f9f255e4706f82"}},{"batch_id":"external-gold-expansion-v2-batch3","benchmark_id":"external-gold-expansion-v2-batch3","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T17:45:38.245198Z","evidence_identity":{"freeze_input_commit":"8450ed889756784a12ca4ebb8d84974d73b31778","final_freeze_receipt_commit":"a309bda7e8dd2867ccc7894dc7d42efbfc48395e","measurement_commit":"b81cc816158f7c0fbe7f8ac0a6fcf2b3f38f4c86","immutable_measurement_receipt_commit":"9d8290160849a5124d1896fc318d53470585ef41","case_freeze_sha256":"c1595743f48fe8ae6b3f70f2ea401731052b3bd7614297feabaa8172ce190ae5","model_lock_sha256":"48b5ed9c98f16cc2b3b76bc0da2e7ec8072ecd89ba1914683181b1c20074a3a8","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"c5cfe765f6b72489220cc73f689acb1c1c2c9284ffd8d093de0e597b645f6fbf","measurement_artifact_sha256":"7dcab9d4b71e7de59094c07cbca830ebf674b601ac6de024db0d3e677e7c0067","policy_evaluation_sha256":"3ea5b62044b4da6e92812831ebb00205970849accffe763a270248a24a0adef4"}},{"batch_id":"external-gold-expansion-v2-batch4","benchmark_id":"external-gold-expansion-v2-batch4","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T18:47:01.455700Z","evidence_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"}},{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S1","S2"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":55,"number_of_batches":6,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"external-gold-v1:Context7","subject_name":"Context7","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:Context7","subject_name":"Context7","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"external-gold-v1:DEX Paprika","subject_name":"DEX Paprika","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:DEX Paprika","subject_name":"DEX Paprika","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":1,"strict_accuracy":0.5,"wilson_95":{"lower":0.094531,"upper":0.905469},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"1/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":1,"observed_semantic_miss_case_ids":["dex_paprika_001"],"observed_semantic_misses":[{"case_id":"dex_paprika_001","subject_id":"external-gold-v1:DEX Paprika","expected_tool":"getStats","selected_tool":"getNetworks","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"GOLD_SCHEMA_CONTRADICTION","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json","benchmark_id":"external-gold-v1","batch_id":"external-gold-v1"}],"observed_miss_boundary_families":["NOT_RECORDED_IN_SOURCE_EVIDENCE"],"boundary_family_data_complete":false,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":0.5,"maximum":0.5},"latest_strict_accuracy":0.5,"lowest_observed_strict_accuracy":0.5}},{"canonical_subject_id":"external-gold-v1:Google Maps","subject_name":"Google Maps","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:Google Maps","subject_name":"Google Maps","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":0,"strict_accuracy":0.0,"wilson_95":{"lower":0.0,"upper":0.65762},"semantic_coverage":1.0,"observed_semantic_miss_count":2,"strict_score":"0/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":2,"observed_semantic_miss_case_ids":["google_maps_000","google_maps_001"],"observed_semantic_misses":[{"case_id":"google_maps_000","subject_id":"external-gold-v1:Google Maps","expected_tool":"maps_geocode","selected_tool":"search_nearby","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"NON_UNIQUE_FIRST_TOOL","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json","benchmark_id":"external-gold-v1","batch_id":"external-gold-v1"},{"case_id":"google_maps_001","subject_id":"external-gold-v1:Google Maps","expected_tool":"maps_geocode","selected_tool":"search_nearby","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"NON_UNIQUE_FIRST_TOOL","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json","benchmark_id":"external-gold-v1","batch_id":"external-gold-v1"}],"observed_miss_boundary_families":["NOT_RECORDED_IN_SOURCE_EVIDENCE"],"boundary_family_data_complete":false,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":0.0,"maximum":0.0},"latest_strict_accuracy":0.0,"lowest_observed_strict_accuracy":0.0}},{"canonical_subject_id":"external-gold-v1:Huge Icons","subject_name":"Huge Icons","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:Huge Icons","subject_name":"Huge Icons","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":1,"strict_accuracy":0.5,"wilson_95":{"lower":0.094531,"upper":0.905469},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"1/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":1,"observed_semantic_miss_case_ids":["huge_icons_000"],"observed_semantic_misses":[{"case_id":"huge_icons_000","subject_id":"external-gold-v1:Huge Icons","expected_tool":"list_icons","selected_tool":"search_icons","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"NON_UNIQUE_FIRST_TOOL","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json","benchmark_id":"external-gold-v1","batch_id":"external-gold-v1"}],"observed_miss_boundary_families":["NOT_RECORDED_IN_SOURCE_EVIDENCE"],"boundary_family_data_complete":false,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":0.5,"maximum":0.5},"latest_strict_accuracy":0.5,"lowest_observed_strict_accuracy":0.5}},{"canonical_subject_id":"external-gold-v1:Medical Calculator","subject_name":"Medical Calculator","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:Medical Calculator","subject_name":"Medical Calculator","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"external-gold-v1:Metropolitan Museum","subject_name":"Metropolitan Museum","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:Metropolitan Museum","subject_name":"Metropolitan Museum","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"external-gold-v1:National Parks","subject_name":"National Parks","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:National Parks","subject_name":"National Parks","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"external-gold-v1:NixOS","subject_name":"NixOS","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:NixOS","subject_name":"NixOS","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":1,"strict_accuracy":0.5,"wilson_95":{"lower":0.094531,"upper":0.905469},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"1/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":1,"observed_semantic_miss_case_ids":["nixos_001"],"observed_semantic_misses":[{"case_id":"nixos_001","subject_id":"external-gold-v1:NixOS","expected_tool":"nixos_search","selected_tool":"nixos_info","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"GENUINE_SEQUENCE_ERROR","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json","benchmark_id":"external-gold-v1","batch_id":"external-gold-v1"}],"observed_miss_boundary_families":["NOT_RECORDED_IN_SOURCE_EVIDENCE"],"boundary_family_data_complete":false,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":0.5,"maximum":0.5},"latest_strict_accuracy":0.5,"lowest_observed_strict_accuracy":0.5}},{"canonical_subject_id":"external-gold-v1:OKX Exchange","subject_name":"OKX Exchange","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:OKX Exchange","subject_name":"OKX Exchange","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":1,"strict_accuracy":0.5,"wilson_95":{"lower":0.094531,"upper":0.905469},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"1/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":1,"observed_semantic_miss_case_ids":["okx_exchange_000"],"observed_semantic_misses":[{"case_id":"okx_exchange_000","subject_id":"external-gold-v1:OKX Exchange","expected_tool":"get_price","selected_tool":"get_candlesticks","boundary_family":"NOT_RECORDED_IN_SOURCE_EVIDENCE","adjudication":"PROCEDURE_ORDER_DISAGREEMENT","source_adjudication_artifact":"validation/independent-holdout-v1/external-gold-v1/semantic-miss-adjudication-v1.json","benchmark_id":"external-gold-v1","batch_id":"external-gold-v1"}],"observed_miss_boundary_families":["NOT_RECORDED_IN_SOURCE_EVIDENCE"],"boundary_family_data_complete":false,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":0.5,"maximum":0.5},"latest_strict_accuracy":0.5,"lowest_observed_strict_accuracy":0.5}},{"canonical_subject_id":"external-gold-v1:Reddit","subject_name":"Reddit","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:Reddit","subject_name":"Reddit","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"external-gold-v1:Scientific Computing","subject_name":"Scientific Computing","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:Scientific Computing","subject_name":"Scientific Computing","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"external-gold-v1:Wikipedia","subject_name":"Wikipedia","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"external-gold-v1:Wikipedia","subject_name":"Wikipedia","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":2,"total_provider_valid_cases":2,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-v1","benchmark_id":"external-gold-v1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":2,"provider_valid_cases":2,"infrastructure_failures":0,"strict_correct":2,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"2/2","measured_at":"2026-09-09T22:51:26.259023Z","evidence_identity":{"case_freeze_sha256":"c28335eb68a5b82456ed11593a8cf589ca75d0cf9802ce570b0e46f5b1716d71","model_lock_sha256":"64e9fb5e6a9ab6fb2d72172168c13877c60645ac336b1b4fceacc8eb8219da11","measurement_artifact_sha256":"1f248ea7a423fc49b03aece1fa16d7d4e5a8b488985787b6f702a31cd1db7e33","measurement_manifest_sha256":"d7323f18ad5e642986bae0534d31fca1b877b2f3fa70764649051d7721003ecd","final_verification_receipt_sha256":"6e5a58b37b7f4a22f89db28ddbb64a187c377220f9593fcb555c821aa0baf176"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-09T22:51:26.259023Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"LIMITED_MEASUREMENT_EVIDENCE","summary":{"total_measured_cases":2,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"gitmcp-docs","subject_name":"GitMCP Docs","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"MIXED","identity_variants":[{"canonical_subject_id":"gitmcp-docs","subject_name":"GitMCP Docs","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"LIVE_TOOLS_LIST"},{"canonical_subject_id":"gitmcp-docs","subject_name":"GitMCP","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT"},{"canonical_subject_id":"gitmcp-docs","subject_name":"GitMCP","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"}],"number_of_independent_benchmark_batches":3,"total_measured_frozen_cases":35,"total_provider_valid_cases":35,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}},{"batch_id":"external-gold-expansion-v2-batch2","benchmark_id":"external-gold-expansion-v2-batch2","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":4,"strict_accuracy":0.8,"wilson_95":{"lower":0.37553462976252533,"upper":0.9637758913675698},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"4/5","measured_at":"2026-09-10T16:49:53.346446Z","evidence_identity":{"freeze_input_commit":"61f6511c41562c24019cd441b37bc6ee07283951","final_freeze_receipt_commit":"d3c55d2f96a4e45bf2577225ea16929877f2ded1","measurement_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","policy_evaluation_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","case_freeze_sha256":"ce21fc085bd9a0185c9e37d0d795221f286397cbc477ee941e876be24f7ba1b4","model_lock_sha256":"153647f94eed0ee3d3069229545846eef864fd43fa821f2d311dc3fb5d265ba9","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"8c4364ab598241ddbe4adb16453ba2e12b0917a1d1852caedb07884d1a54570b","measurement_artifact_sha256":"dd13aec8a90b2b5490c934c9ad9650c37733c65f18d824e2ae6066202176ced9","policy_evaluation_sha256":"736f154a5d5c3b8f0d77cd58fa0644ca97ad94c7a6b46af98d394c1dc1578f9a"}},{"batch_id":"external-gold-expansion-v2-batch3","benchmark_id":"external-gold-expansion-v2-batch3","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":4,"strict_accuracy":0.666667,"wilson_95":{"lower":0.299993,"upper":0.903229},"semantic_coverage":1.0,"observed_semantic_miss_count":2,"strict_score":"4/6","measured_at":"2026-09-10T17:45:38.245198Z","evidence_identity":{"freeze_input_commit":"8450ed889756784a12ca4ebb8d84974d73b31778","final_freeze_receipt_commit":"a309bda7e8dd2867ccc7894dc7d42efbfc48395e","measurement_commit":"b81cc816158f7c0fbe7f8ac0a6fcf2b3f38f4c86","immutable_measurement_receipt_commit":"9d8290160849a5124d1896fc318d53470585ef41","case_freeze_sha256":"c1595743f48fe8ae6b3f70f2ea401731052b3bd7614297feabaa8172ce190ae5","model_lock_sha256":"48b5ed9c98f16cc2b3b76bc0da2e7ec8072ecd89ba1914683181b1c20074a3a8","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"c5cfe765f6b72489220cc73f689acb1c1c2c9284ffd8d093de0e597b645f6fbf","measurement_artifact_sha256":"7dcab9d4b71e7de59094c07cbca830ebf674b601ac6de024db0d3e677e7c0067","policy_evaluation_sha256":"3ea5b62044b4da6e92812831ebb00205970849accffe763a270248a24a0adef4"}}],"observed_semantic_miss_count":3,"observed_semantic_miss_case_ids":["gitmcp-hard-005","b3-gm-005","b3-gm-006"],"observed_semantic_misses":[{"case_id":"gitmcp-hard-005","subject_id":"gitmcp-docs","expected_tool":"search_generic_documentation","selected_tool":"fetch_generic_documentation","boundary_family":"DOCUMENTED_BEHAVIOR_VS_IMPLEMENTATION","adjudication":"SEMANTIC_MISS_GOLD_REMAINS_VALID","source_adjudication_artifact":"validation/external-gold-expansion-v2/measurement-v2-batch2/semantic-miss-adjudication-v2-batch2.json","benchmark_id":"external-gold-expansion-v2-batch2","batch_id":"external-gold-expansion-v2-batch2"},{"case_id":"b3-gm-005","subject_id":"gitmcp-docs","expected_tool":"fetch_generic_url_content","selected_tool":"fetch_generic_documentation","boundary_family":"DOCUMENTATION_VS_REFERENCED_URL","adjudication":"GOLD_REMAINS_VALID; semantic miss preserved in strict score. The public source supports the requested neighboring operation, and the selected tool is a plausible but incorrect neighboring retrieval mode.","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch3/measurement-v2-batch3/semantic-miss-adjudication-v2-batch3.json","benchmark_id":"external-gold-expansion-v2-batch3","batch_id":"external-gold-expansion-v2-batch3"},{"case_id":"b3-gm-006","subject_id":"gitmcp-docs","expected_tool":"search_generic_documentation","selected_tool":"fetch_generic_documentation","boundary_family":"DOCUMENTED_BEHAVIOR_VS_IMPLEMENTATION","adjudication":"GOLD_REMAINS_VALID; semantic miss preserved in strict score. The public source supports the requested neighboring operation, and the selected tool is a plausible but incorrect neighboring retrieval mode.","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch3/measurement-v2-batch3/semantic-miss-adjudication-v2-batch3.json","benchmark_id":"external-gold-expansion-v2-batch3","batch_id":"external-gold-expansion-v2-batch3"}],"observed_miss_boundary_families":["DOCUMENTATION_VS_REFERENCED_URL","DOCUMENTED_BEHAVIOR_VS_IMPLEMENTATION"],"boundary_family_data_complete":true,"evidence_levels_represented":["S1","S2"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-10T17:45:38.245198Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-10T17:45:38.245198Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":35,"number_of_batches":3,"strict_accuracy_range_across_batches":{"minimum":0.666667,"maximum":1.0},"latest_strict_accuracy":0.666667,"lowest_observed_strict_accuracy":0.666667}},{"canonical_subject_id":"google-maps-mcp","subject_name":"Google Maps","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"google-maps-mcp","subject_name":"Google Maps","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":2,"total_measured_frozen_cases":15,"total_provider_valid_cases":15,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T19:13:49.842728Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":15,"number_of_batches":2,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"metropolitan-museum-mcp","subject_name":"Metropolitan Museum","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"metropolitan-museum-mcp","subject_name":"Metropolitan Museum","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":2,"total_measured_frozen_cases":15,"total_provider_valid_cases":15,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T19:13:49.842728Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":15,"number_of_batches":2,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"metropolitan-museum-mcp-source-snapshot","subject_name":"Metropolitan Museum MCP (source snapshot)","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"metropolitan-museum-mcp-source-snapshot","subject_name":"Metropolitan Museum MCP (source snapshot)","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":24,"total_provider_valid_cases":24,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S1"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MEASURED","summary":{"total_measured_cases":24,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"microsoft-learn","subject_name":"Microsoft Learn","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"MIXED","identity_variants":[{"canonical_subject_id":"microsoft-learn","subject_name":"Microsoft Learn","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_SNAPSHOT"},{"canonical_subject_id":"microsoft-learn","subject_name":"Microsoft Learn","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT"},{"canonical_subject_id":"microsoft-learn","subject_name":"Microsoft Learn","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"}],"number_of_independent_benchmark_batches":5,"total_measured_frozen_cases":31,"total_provider_valid_cases":31,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-expansion-v2-batch1","benchmark_id":"external-gold-expansion-v2-batch1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5","measured_at":"2026-09-10T15:38:03.968297Z","evidence_identity":{"freeze_input_commit":"c614ae51750d12ce113b5e3d51429f52e529f08c","final_freeze_receipt_commit":"b691170","measurement_commit":"b9f358b1a65aba348fa1c0e625eac6da1f31b651","policy_evaluation_commit":"3e560e5767f181831c65569b893ca789cd3bdff0","case_freeze_sha256":"17777d2c3983cd444353b8d69dcc98836a1909ee6fde6f5c8cf18cf9b059427e","model_lock_sha256":"c4330165deb3d8587e224a2ca1b3d886db60801ca436710a0edace55525923d2","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ab085cedf13eef5724f4e160b0f49f8402d62bc5b10203de0f8d81e41f2a017e","measurement_artifact_sha256":"01243422aa946296f235810aaee6f065ec21d9e78a01017ee8161e5e61a8b5e8","policy_evaluation_sha256":"4e710a9c5205660cc07603d2fd7538aa063eda4c2574c29308f9f255e4706f82"}},{"batch_id":"external-gold-expansion-v2-batch2","benchmark_id":"external-gold-expansion-v2-batch2","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5","measured_at":"2026-09-10T16:49:53.346446Z","evidence_identity":{"freeze_input_commit":"61f6511c41562c24019cd441b37bc6ee07283951","final_freeze_receipt_commit":"d3c55d2f96a4e45bf2577225ea16929877f2ded1","measurement_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","policy_evaluation_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","case_freeze_sha256":"ce21fc085bd9a0185c9e37d0d795221f286397cbc477ee941e876be24f7ba1b4","model_lock_sha256":"153647f94eed0ee3d3069229545846eef864fd43fa821f2d311dc3fb5d265ba9","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"8c4364ab598241ddbe4adb16453ba2e12b0917a1d1852caedb07884d1a54570b","measurement_artifact_sha256":"dd13aec8a90b2b5490c934c9ad9650c37733c65f18d824e2ae6066202176ced9","policy_evaluation_sha256":"736f154a5d5c3b8f0d77cd58fa0644ca97ad94c7a6b46af98d394c1dc1578f9a"}},{"batch_id":"external-gold-expansion-v2-batch4","benchmark_id":"external-gold-expansion-v2-batch4","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T18:47:01.455700Z","evidence_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"}},{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T15:38:03.968297Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":31,"number_of_batches":5,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"microsoft-release-communications","subject_name":"Microsoft Release Communications","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"MIXED","identity_variants":[{"canonical_subject_id":"microsoft-release-communications","subject_name":"Microsoft Release Communications","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_SNAPSHOT"},{"canonical_subject_id":"microsoft-release-communications","subject_name":"Microsoft Release Communications","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"CURRENT_LIVE_TOOLS_LIST_FROZEN_BY_EXISTING_IMMUTABLE_SNAPSHOT"},{"canonical_subject_id":"microsoft-release-communications","subject_name":"Microsoft Release Communications","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"CURRENT_OR_VERSION_BOUND_SNAPSHOT_REUSED_FROM_QUALIFIED_CORPUS"}],"number_of_independent_benchmark_batches":6,"total_measured_frozen_cases":37,"total_provider_valid_cases":37,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-expansion-v2-batch1","benchmark_id":"external-gold-expansion-v2-batch1","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5","measured_at":"2026-09-10T15:38:03.968297Z","evidence_identity":{"freeze_input_commit":"c614ae51750d12ce113b5e3d51429f52e529f08c","final_freeze_receipt_commit":"b691170","measurement_commit":"b9f358b1a65aba348fa1c0e625eac6da1f31b651","policy_evaluation_commit":"3e560e5767f181831c65569b893ca789cd3bdff0","case_freeze_sha256":"17777d2c3983cd444353b8d69dcc98836a1909ee6fde6f5c8cf18cf9b059427e","model_lock_sha256":"c4330165deb3d8587e224a2ca1b3d886db60801ca436710a0edace55525923d2","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ab085cedf13eef5724f4e160b0f49f8402d62bc5b10203de0f8d81e41f2a017e","measurement_artifact_sha256":"01243422aa946296f235810aaee6f065ec21d9e78a01017ee8161e5e61a8b5e8","policy_evaluation_sha256":"4e710a9c5205660cc07603d2fd7538aa063eda4c2574c29308f9f255e4706f82"}},{"batch_id":"external-gold-expansion-v2-batch2","benchmark_id":"external-gold-expansion-v2-batch2","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":5,"provider_valid_cases":5,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":1.0,"wilson_95":{"lower":0.5655175352168251,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"5/5","measured_at":"2026-09-10T16:49:53.346446Z","evidence_identity":{"freeze_input_commit":"61f6511c41562c24019cd441b37bc6ee07283951","final_freeze_receipt_commit":"d3c55d2f96a4e45bf2577225ea16929877f2ded1","measurement_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","policy_evaluation_commit":"f0b6cd33be044d8bad800c20b4f2684e6d8431ce","case_freeze_sha256":"ce21fc085bd9a0185c9e37d0d795221f286397cbc477ee941e876be24f7ba1b4","model_lock_sha256":"153647f94eed0ee3d3069229545846eef864fd43fa821f2d311dc3fb5d265ba9","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"8c4364ab598241ddbe4adb16453ba2e12b0917a1d1852caedb07884d1a54570b","measurement_artifact_sha256":"dd13aec8a90b2b5490c934c9ad9650c37733c65f18d824e2ae6066202176ced9","policy_evaluation_sha256":"736f154a5d5c3b8f0d77cd58fa0644ca97ad94c7a6b46af98d394c1dc1578f9a"}},{"batch_id":"external-gold-expansion-v2-batch3","benchmark_id":"external-gold-expansion-v2-batch3","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T17:45:38.245198Z","evidence_identity":{"freeze_input_commit":"8450ed889756784a12ca4ebb8d84974d73b31778","final_freeze_receipt_commit":"a309bda7e8dd2867ccc7894dc7d42efbfc48395e","measurement_commit":"b81cc816158f7c0fbe7f8ac0a6fcf2b3f38f4c86","immutable_measurement_receipt_commit":"9d8290160849a5124d1896fc318d53470585ef41","case_freeze_sha256":"c1595743f48fe8ae6b3f70f2ea401731052b3bd7614297feabaa8172ce190ae5","model_lock_sha256":"48b5ed9c98f16cc2b3b76bc0da2e7ec8072ecd89ba1914683181b1c20074a3a8","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"c5cfe765f6b72489220cc73f689acb1c1c2c9284ffd8d093de0e597b645f6fbf","measurement_artifact_sha256":"7dcab9d4b71e7de59094c07cbca830ebf674b601ac6de024db0d3e677e7c0067","policy_evaluation_sha256":"3ea5b62044b4da6e92812831ebb00205970849accffe763a270248a24a0adef4"}},{"batch_id":"external-gold-expansion-v2-batch4","benchmark_id":"external-gold-expansion-v2-batch4","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T18:47:01.455700Z","evidence_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"}},{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T15:38:03.968297Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":37,"number_of_batches":6,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"national-parks-mcp","subject_name":"National Parks","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"national-parks-mcp","subject_name":"National Parks","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":3,"total_measured_frozen_cases":21,"total_provider_valid_cases":21,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-expansion-v2-batch4","benchmark_id":"external-gold-expansion-v2-batch4","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":6,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"6/6","measured_at":"2026-09-10T18:47:01.455700Z","evidence_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"}},{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T18:47:01.455700Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":21,"number_of_batches":3,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"national-parks-mcp-source-snapshot","subject_name":"National Parks MCP (source snapshot)","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"national-parks-mcp-source-snapshot","subject_name":"National Parks MCP (source snapshot)","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":24,"total_provider_valid_cases":24,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S1"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MEASURED","summary":{"total_measured_cases":24,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"nixos-mcp","subject_name":"NixOS","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"nixos-mcp","subject_name":"NixOS","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":3,"total_measured_frozen_cases":21,"total_provider_valid_cases":21,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-expansion-v2-batch4","benchmark_id":"external-gold-expansion-v2-batch4","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":6,"provider_valid_cases":6,"infrastructure_failures":0,"strict_correct":5,"strict_accuracy":0.833333,"wilson_95":{"lower":0.436497,"upper":0.969947,"lower_percent":43.6497,"upper_percent":96.9947},"semantic_coverage":1.0,"observed_semantic_miss_count":1,"strict_score":"5/6","measured_at":"2026-09-10T18:47:01.455700Z","evidence_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"}},{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":1,"observed_semantic_miss_case_ids":["b4-nx-001"],"observed_semantic_misses":[{"case_id":"b4-nx-001","subject_id":"nixos-mcp","expected_tool":"home_manager_search","selected_tool":"home_manager_info","boundary_family":"HOME_MANAGER_VS_DARWIN_SEARCH","adjudication":"VALID_GOLD_RETAINED","source_adjudication_artifact":"validation/external-gold-expansion-v2/batch4/measurement-v2-batch4/semantic-miss-adjudication-v2-batch4.json","benchmark_id":"external-gold-expansion-v2-batch4","batch_id":"external-gold-expansion-v2-batch4"}],"observed_miss_boundary_families":["HOME_MANAGER_VS_DARWIN_SEARCH"],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T18:47:01.455700Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":21,"number_of_batches":3,"strict_accuracy_range_across_batches":{"minimum":0.833333,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":0.833333}},{"canonical_subject_id":"nixos-mcp-source-snapshot","subject_name":"NixOS MCP (source snapshot)","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"nixos-mcp-source-snapshot","subject_name":"NixOS MCP (source snapshot)","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":24,"total_provider_valid_cases":24,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S1"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MEASURED","summary":{"total_measured_cases":24,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"uk-premises-licence-mcp","subject_name":"UK Premises Licence MCP","provenance_type":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_variants":[{"canonical_subject_id":"uk-premises-licence-mcp","subject_name":"UK Premises Licence MCP","provenance_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","live_vs_version_bound":"LIVE_TOOLS_LIST_SNAPSHOT","identity_mode_source":"LIVE_TOOLS_LIST"}],"number_of_independent_benchmark_batches":1,"total_measured_frozen_cases":24,"total_provider_valid_cases":24,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"selection-evidence-expansion-v1","benchmark_id":"selection-evidence-expansion-v1","benchmark_type":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","benchmark_provenance":"CONSTRUCTED_FROZEN_SELECTION_BENCHMARK","promotion_level":"S1","promotion_status":"S1 selection-tested","frozen_case_count":24,"provider_valid_cases":24,"infrastructure_failures":0,"strict_correct":24,"strict_accuracy":1.0,"wilson_95":{"confidence":0.95,"high":1.0,"low":0.8620237953250198,"method":"wilson_score"},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"24/24","measured_at":"2026-09-10T13:23:12.077611Z","evidence_identity":{"freeze_commit_sha":"d2937086b419ba0bf77517da772540de1975c45f","case_freeze_sha256":"5d248e845fc1b52ef53585682a97a592fecd0d3c1b8c9e561c758a4222273aed","model_lock_sha256":"a8df78457b76e0dea73f66e2a75907feab3c42885db23658caada6973c5cb478","measurement_artifact_sha256":"27f281b49b50fe5218102274c57db0110f3119091ca5dd53ebe954997f4fe826","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S1"],"benchmark_types_represented":["CONSTRUCTED_FROZEN_SELECTION_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-10T13:23:12.077611Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MEASURED","summary":{"total_measured_cases":24,"number_of_batches":1,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}},{"canonical_subject_id":"wikipedia-mcp-reference","subject_name":"wikipedia-mcp-reference","provenance_type":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_variants":[{"canonical_subject_id":"wikipedia-mcp-reference","subject_name":"wikipedia-mcp-reference","provenance_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","live_vs_version_bound":"VERSION_BOUND_SNAPSHOT","identity_mode_source":"VERSION_BOUND_RECONSTRUCTED_SOURCE_SNAPSHOT"}],"number_of_independent_benchmark_batches":2,"total_measured_frozen_cases":15,"total_provider_valid_cases":15,"total_infrastructure_failures":0,"batch_metrics":[{"batch_id":"external-gold-expansion-v2-batch5","benchmark_id":"external-gold-expansion-v2-batch5","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":7,"provider_valid_cases":7,"infrastructure_failures":0,"strict_correct":7,"strict_accuracy":1.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"7/7","measured_at":"2026-09-10T19:13:49.842728Z","evidence_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"}},{"batch_id":"external-gold-expansion-v2-batch6","benchmark_id":"external-gold-expansion-v2-batch6","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","frozen_case_count":8,"provider_valid_cases":8,"infrastructure_failures":0,"strict_correct":8,"strict_accuracy":1.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0},"semantic_coverage":1.0,"observed_semantic_miss_count":0,"strict_score":"8/8","measured_at":"2026-09-11T00:07:39.419700Z","evidence_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"}}],"observed_semantic_miss_count":0,"observed_semantic_miss_case_ids":[],"observed_semantic_misses":[],"observed_miss_boundary_families":[],"boundary_family_data_complete":true,"evidence_levels_represented":["S2"],"benchmark_types_represented":["EXTERNAL_INDEPENDENT_GOLD_BENCHMARK"],"earliest_measurement_timestamp":"2026-09-10T19:13:49.842728Z","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","evidence_freshness":{"basis":"latest immutable measurement timestamp","latest_measurement_timestamp":"2026-09-11T00:07:39.419700Z","live_freshness_claimed":false,"age_evaluated_at_runtime":false},"reliability_status":"MULTI_BATCH_MEASURED","summary":{"total_measured_cases":15,"number_of_batches":2,"strict_accuracy_range_across_batches":{"minimum":1.0,"maximum":1.0},"latest_strict_accuracy":1.0,"lowest_observed_strict_accuracy":1.0}}],"counts":{"subjects_with_empirical_reliability_records":30,"subjects_with_multi_batch_evidence":12,"subjects_with_single_batch_evidence":18,"subjects_with_s2_backed_evidence":24,"subjects_with_observed_semantic_misses":9,"observed_semantic_miss_count":16},"known_limitations":["Batch metrics are intentionally not pooled into a single accuracy or reliability score.","Observed misses describe frozen historical measurements and do not predict future failures.","Version-bound snapshots do not establish current-live status.","Boundary-family fields are unavailable for some older external evidence and are disclosed as such.","Provider infrastructure failures are reported separately and are not semantic misses."]}}