{"schema_version":"mcp-selection-lab.agent-trust-lab.selection-evidence-status-v5","track":"EXTERNAL_GOLD_EXPANSION_V2","status":"MEASURED","benchmark":"External Gold Expansion V2 Batch 4","benchmark_version":"external-gold-expansion-v2-batch4","batch_characterization":"CROSS_SUBJECT_BOUNDARY_RISK_VALIDATION_BATCH","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","subject_level_promotions":[],"case_count":48,"strict_correct":46,"strict_score":"46/48","strict_accuracy":0.958333,"strict_accuracy_display":"95.8333%","wilson_95":{"lower":0.860243,"upper":0.988498,"lower_percent":86.0243,"upper_percent":98.8498},"wilson_95_display":"86.0243%–98.8498%","provider_valid_count":48,"provider_valid_coverage":1.0,"semantic_coverage":1.0,"infrastructure_failure_count":0,"risk_bucket_metrics":{"HIGH":{"cases":19,"provider_valid_cases":19,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":17,"misses":2,"strict_accuracy":0.894737,"wilson_95":{"lower":0.686059,"upper":0.970641,"lower_percent":68.6059,"upper_percent":97.0641}},"MEDIUM":{"cases":12,"provider_valid_cases":12,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":12,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.757506,"upper":1.0,"lower_percent":75.7506,"upper_percent":100.0}},"LOW":{"cases":17,"provider_valid_cases":17,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":17,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.815682,"upper":1.0,"lower_percent":81.5682,"upper_percent":100.0}}},"risk_comparison":{"high":{"cases":19,"provider_valid_cases":19,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":17,"misses":2,"strict_accuracy":0.894737,"wilson_95":{"lower":0.686059,"upper":0.970641,"lower_percent":68.6059,"upper_percent":97.0641},"miss_rate":0.10526315789473684},"non_high":{"cases":29,"provider_valid_cases":29,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":29,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.88303,"upper":1.0,"lower_percent":88.303,"upper_percent":100.0},"miss_rate":0.0},"miss_rate_difference_high_minus_non_high":0.10526315789473684,"risk_ratio":null,"interpretation":"EXPLORATORY; zero non-HIGH misses prevents a stable finite risk ratio and the number of failures is small."},"scanner_status":"PROMISING_CROSS_SUBJECT_SIGNAL_MORE_DATA_REQUIRED","scanner_validation":{"decision":"PROMISING_CROSS_SUBJECT_SIGNAL_MORE_DATA_REQUIRED","cross_subject_validated":false,"production_validity_claimed":false,"disclosure":"Batch 4 was a promising exploratory observation only; it did not validate BOUNDARY_RISK_SCAN_V1."},"failure_dispersion":{"subjects_with_miss":["deepwiki-mcp","nixos-mcp"],"subject_count_with_miss":2,"boundary_family_count_with_miss":2},"leave_one_subject_out":{"all_subjects_directionally_consistent":true,"results_artifact":"leave-one-subject-out-evaluation-v1.json"},"boundary_family_sensitivity":{"benchmark_version":"external-gold-expansion-v2-batch4","metrics":{"ALERTS_VS_PARK_DISCOVERY":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"AZURE_COLLECTION_VS_AZURE_ITEM":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"AZURE_ITEM_VS_AZURE_COLLECTION":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"AZURE_VS_M365_COLLECTION":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"CAMPGROUNDS_VS_VISITOR_CENTERS":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"CODE_SAMPLE_VS_FULL_DOC":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"CODE_VS_CONCEPTUAL_DOCS":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"CONTENTS_VS_QUESTION":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":0,"misses":1,"strict_accuracy":0.0,"wilson_95":{"lower":0.0,"upper":0.793451,"lower_percent":0.0,"upper_percent":79.3451},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"CONTENTS_VS_STRUCTURE":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"CURRENT_DISCOVERY_VS_KNOWN_URL":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"DARWIN_VS_HOME_MANAGER_SEARCH":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"DATASET_METADATA_VS_USAGE_METRICS":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"DATASET_RECORD_VS_RESOURCE_COLLECTION":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"DATASET_VS_DATASERVICE":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"DATASET_VS_RESOURCE_COLLECTION":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"DEDICATED_GUIDE_VS_GENERAL_SEARCH":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"DISCOVERY_VS_KNOWN_PAGE":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"DISCOVERY_VS_PAGE_READ":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"DOCS_VS_CODE":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"EVENTS_VS_ALERTS":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"FLAKES_VS_PACKAGES_OPTIONS":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"GENERAL_DOCS_VS_MIGRATION_GUIDE":{"cases":4,"provider_valid_cases":4,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":4,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.510109,"upper":1.0,"lower_percent":51.0109,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"HOME_MANAGER_OPTION_VS_DARWIN_OPTION":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"HOME_MANAGER_VS_DARWIN_SEARCH":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":0,"misses":1,"strict_accuracy":0.0,"wilson_95":{"lower":0.0,"upper":0.793451,"lower_percent":0.0,"upper_percent":79.3451},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"KNOWN_PAGE_VS_DISCOVERY":{"cases":2,"provider_valid_cases":2,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":2,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.34238,"upper":1.0,"lower_percent":34.238,"upper_percent":100.0},"subject_count":2,"interpretation":"DESCRIPTIVE_ONLY"},"KNOWN_PAGE_VS_SEARCH":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"KNOWN_URL_VS_OPEN_SEARCH":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"KNOWN_URL_VS_SEARCH":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"M365_ITEM_VS_AZURE_ITEM":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"M365_ITEM_VS_M365_COLLECTION":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"M365_VS_AZURE_COLLECTION":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"MIGRATION_GUIDE_VS_GENERAL_SEARCH":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"NIXOS_SEARCH_VS_INFO":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"OPTION_CATEGORIES_VS_PREFIX_ENTRIES":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"PARK_DETAIL_VS_DISCOVERY":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"PARK_DISCOVERY_VS_ALERTS":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"QUESTION_VS_CONTENTS":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"QUESTION_VS_STRUCTURE":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"RESOURCE_METADATA_VS_QUERY":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"SERVICE_METADATA_VS_OPENAPI":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"STRUCTURE_VS_CONTENTS":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"STRUCTURE_VS_QUESTION":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"VISITOR_CENTERS_VS_CAMPGROUNDS":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"},"WEB_DISCOVERY_VS_KNOWN_PAGE":{"cases":1,"provider_valid_cases":1,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":1,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.206549,"upper":1.0,"lower_percent":20.6549,"upper_percent":100.0},"subject_count":1,"interpretation":"DESCRIPTIVE_ONLY"}},"recurring_exact_boundary_labels_across_subjects":{"KNOWN_PAGE_VS_DISCOVERY":["exa-mcp","microsoft-learn"]},"cross_subject_recurrence_note":"Only exact frozen boundary labels recurring across subjects are listed; no labels were merged after measurement."},"semantic_miss_count":2,"semantic_misses":[{"case_id":"b4-dw-005","subject_id":"deepwiki-mcp","frozen_prompt":"Read the repository documentation pages covering its release process and return their contents verbatim enough for me to inspect the procedure.","expected_tool":"read_wiki_contents","selected_tool":"read_wiki_structure","plausible_competing_tools":["ask_question"],"selection_boundary":"CONTENTS_VS_QUESTION","difficulty":"H2","risk_bucket":"HIGH","risk_score":51.6667,"risk_factors":["DESCRIPTION_LEXICAL_OVERLAP","SHARED_REQUIRED_INPUTS","MISSING_OUTPUT_CONTRAST","INCOMPLETE_BOUNDARY_LANGUAGE"],"external_source_id":"deepwiki-aws-guide-2026-09-10","source_url":"https://docs.devin.ai/work-with-devin/deepwiki-mcp","source_version":"public guide and repository evidence captured 2026-09-10","source_excerpt_reference":"DeepWiki MCP guide; page contents request","gold_label_changed":false,"original_prediction_preserved":true,"gold_validity":"VALID_GOLD_RETAINED","adjudication_reason":"The user wants source documentation contents, not an answer generated from them."},{"case_id":"b4-nx-001","subject_id":"nixos-mcp","frozen_prompt":"Find the Home Manager option documentation for enabling the Git program module. I am configuring a user home, not a macOS system.","expected_tool":"home_manager_search","selected_tool":"home_manager_info","plausible_competing_tools":["darwin_search"],"selection_boundary":"HOME_MANAGER_VS_DARWIN_SEARCH","difficulty":"H3","risk_bucket":"HIGH","risk_score":70.0,"risk_factors":["DESCRIPTION_LEXICAL_OVERLAP","IDENTICAL_REQUIRED_INPUTS_OR_BOTH_NULL","OVERLAPPING_ACTION_VERBS","INCOMPLETE_BOUNDARY_LANGUAGE"],"external_source_id":"home-manager-manual-2026-09-10","source_url":"https://nix-community.github.io/home-manager/options.html","source_version":"Home Manager options manual accessed 2026-09-10","source_excerpt_reference":"Home Manager's published option index distinguishes user-home modules from nix-darwin system options.","gold_label_changed":false,"original_prediction_preserved":true,"gold_validity":"VALID_GOLD_RETAINED","adjudication_reason":"The requested platform and configuration domain are Home Manager, not nix-darwin."}],"taxonomy_validation":{"RETRIEVAL_GRANULARITY_CONFUSION":{"status":"PARTIALLY_SUPPORTED","batch4_evidence":["b4-dw-005 contents-versus-structure miss","b4-nx-001 search-versus-detail miss"],"confidence":"LOW_TO_MEDIUM"},"DEFAULT_GENERAL_TOOL_PREFERENCE":{"status":"SUPPORTED_SINGLE_SUBJECT_ONLY","batch4_evidence":[],"confidence":"LOW"},"QUESTION_VS_STRUCTURE_CONFUSION":{"status":"INSUFFICIENT_EVIDENCE","batch4_evidence":["b4-dw-005 is contents-versus-structure rather than a synthesized-question case"],"confidence":"LOW"},"DESCRIPTION_ASYMMETRY":{"status":"PARTIALLY_SUPPORTED","batch4_evidence":["Both misses occurred in scanner-high neighboring pairs whose distinctions are primarily descriptive; this is metadata evidence, not causal proof."],"confidence":"LOW"},"SCHEMA_SEMANTIC_GAP":{"status":"PARTIALLY_SUPPORTED","batch4_evidence":["Both misses were valid externally clear requests represented by neighboring schema operations; only two failures limit inference."],"confidence":"LOW_TO_MEDIUM"}},"subjects":[{"canonical_id":"nixos-mcp","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":5,"misses":1,"strict_accuracy":0.833333,"wilson_95":{"lower":0.436497,"upper":0.969947,"lower_percent":43.6497,"upper_percent":96.9947},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"national-parks-mcp","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"datagouv-mcp","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"deepwiki-mcp","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":5,"misses":1,"strict_accuracy":0.833333,"wilson_95":{"lower":0.436497,"upper":0.969947,"lower_percent":43.6497,"upper_percent":96.9947},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"microsoft-release-communications","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"cloudflare-docs-mcp","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"microsoft-learn","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"exa-mcp","cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"}],"frozen_measurement_identity":{"freeze_input_commit":"003985017fbd8c1e207fb146329b97b38582e606","final_freeze_receipt_commit":"224aeab80db539383018d77cc7adb849976f1db5","measurement_artifact_commit":"9b10dc0c99c4eed15a3d370743e8f46439302bae","immutable_measurement_receipt_sha256":"7d6d0a1ac4b1d501aa4c5d0044b4a4e50bb319b272bd33eb783aec6499d594c6","case_freeze_sha256":"a9513c1cb9b800b75d822d75fa8c6e47041e551d86a75699f1858b6c3785204d","model_lock_sha256":"ffc1ec351c1a62c3692cdaedba0c5a6ec2c6285923eb065e0cd31953f978e3a1","scanner_freeze_sha256":"e31caf3131b4e03f54ff411a386bb72ac70c35fbe9027e3383b7026019c22ae1","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"869812a69814f58e73a1c6c3f94a28b6dabc2cdb729285d3a532dd528ad36c89","measurement_artifact_sha256":"7d9a5f57e0897f2ca8b2734ac0f38dc5afd099b2edf2495dbfa25e41f088cee9","policy_evaluation_sha256":"0e9b11360db41731ce139584060b3f1417ce50b31dcf86ef88180e391e56e0d5"},"cross_batch_separation":{"expansion_v1_untouched":true,"external_gold_v1_untouched":true,"batch1_untouched":true,"batch2_untouched":true,"batch3_untouched":true,"batch5_untouched":true,"batch6_untouched":true,"no_combined_headline_score":true},"strict_score_authoritative":true,"adjusted_accuracy_used":false,"target_mcp_business_tools_called":false,"isolated_real_model_executor_exposed":false,"scanner_production_validity_claimed":false,"interpretation":"Batch 4 is benchmark-level S2 evidence with a promising exploratory scanner observation. It is not a validated predictor or production risk score."}