{"schema_version":"mcp-selection-lab.agent-trust-lab.selection-evidence-status-v5","track":"EXTERNAL_GOLD_EXPANSION_V2","status":"MEASURED","benchmark":"External Gold Expansion V2 Batch 5","benchmark_version":"external-gold-expansion-v2-batch5","batch_characterization":"PREDICTIVE_VALIDATION_POWER_EXPANSION_BATCH","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","subject_level_promotions":[],"case_count":77,"strict_correct":74,"strict_score":"74/77","strict_accuracy":0.961039,"strict_accuracy_display":"96.1039%","wilson_95":{"lower":0.8916,"upper":0.986662,"lower_percent":89.16,"upper_percent":98.6662},"provider_valid_count":77,"provider_valid_coverage":1.0,"semantic_coverage":1.0,"infrastructure_failure_count":0,"risk_bucket_metrics":{"HIGH":{"cases":32,"provider_valid_cases":32,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":29,"misses":3,"strict_accuracy":0.90625,"miss_rate":0.09375,"wilson_95":{"lower":0.757818,"upper":0.967598,"lower_percent":75.7818,"upper_percent":96.7598}},"MEDIUM":{"cases":21,"provider_valid_cases":21,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":21,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.845361,"upper":1.0,"lower_percent":84.5361,"upper_percent":100.0}},"LOW":{"cases":24,"provider_valid_cases":24,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":24,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.862024,"upper":1.0,"lower_percent":86.2024,"upper_percent":100.0}}},"risk_comparison":{"high":{"cases":32,"provider_valid_cases":32,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":29,"misses":3,"strict_accuracy":0.90625,"miss_rate":0.09375,"wilson_95":{"lower":0.757818,"upper":0.967598,"lower_percent":75.7818,"upper_percent":96.7598}},"low":{"cases":24,"provider_valid_cases":24,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":24,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.862024,"upper":1.0,"lower_percent":86.2024,"upper_percent":100.0}},"non_high":{"cases":45,"provider_valid_cases":45,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":45,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.921348,"upper":1.0,"lower_percent":92.1348,"upper_percent":100.0}},"high_minus_low_miss_rate":0.09375,"high_vs_low_risk_ratio":null,"high_vs_low_odds_ratio":null,"continuity_correction":{"method":"Haldane-Anscombe add 0.5 to each 2x2 cell","corrected_risk_ratio":5.30303,"corrected_odds_ratio":5.813559},"interpretation":"Exploratory only: LOW has zero misses, so uncorrected RR/OR are not finite and the corrected estimates are unstable with the observed failure count."},"scanner_status":"PROMISING_CROSS_SUBJECT_SIGNAL_MORE_DATA_REQUIRED","scanner_validation":{"decision":"PROMISING_CROSS_SUBJECT_SIGNAL_MORE_DATA_REQUIRED","criteria":{"results":{"C1_HIGH_EXCEEDS_LOW":true,"C2_FAILURE_SUBJECT_DISPERSION":false,"C3_LEAVE_ONE_SUBJECT_OUT":true,"C4_BOUNDARY_FAMILY_NONDOMINANCE":true,"C5_EVIDENCE_INTEGRITY":true,"C6_PROVIDER_COMPLETENESS":true,"C7_SCANNER_IMMUTABILITY":true},"failed_criteria":["C2_FAILURE_SUBJECT_DISPERSION"],"criterion_pass_count":6,"criterion_count":7},"cross_subject_validated":false,"production_validity_claimed":false,"disclosure":"All three Batch 5 semantic misses occurred in DeepWiki; pre-frozen cross-subject dispersion criterion C2 failed."},"failure_dispersion":{"miss_count":3,"subjects_with_miss":["deepwiki-mcp"],"subject_count_with_miss":1,"boundary_families_with_miss":["CONTENTS_VS_QUESTION","CONTENTS_VS_STRUCTURE","QUESTION_VS_CONTENTS"],"boundary_family_count_with_miss":3,"concentration_note":"All observed misses are from one subject; no cross-subject failure dispersion was observed."},"leave_one_subject_out":{"results":{"datagouv-mcp":{"omitted_subject":"datagouv-mcp","high_cases":27,"high_misses":3,"high_miss_rate":0.1111111111111111,"non_high_cases":43,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true},"deepwiki-mcp":{"omitted_subject":"deepwiki-mcp","high_cases":27,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":43,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":false,"informative_failure_signal_remains":false},"microsoft-release-communications":{"omitted_subject":"microsoft-release-communications","high_cases":28,"high_misses":3,"high_miss_rate":0.10714285714285714,"non_high_cases":42,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true},"microsoft-learn":{"omitted_subject":"microsoft-learn","high_cases":32,"high_misses":3,"high_miss_rate":0.09375,"non_high_cases":38,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true},"cloudflare-docs-mcp":{"omitted_subject":"cloudflare-docs-mcp","high_cases":32,"high_misses":3,"high_miss_rate":0.09375,"non_high_cases":38,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true},"exa-mcp":{"omitted_subject":"exa-mcp","high_cases":32,"high_misses":3,"high_miss_rate":0.09375,"non_high_cases":38,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true},"nixos-mcp":{"omitted_subject":"nixos-mcp","high_cases":27,"high_misses":3,"high_miss_rate":0.1111111111111111,"non_high_cases":43,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true},"national-parks-mcp":{"omitted_subject":"national-parks-mcp","high_cases":26,"high_misses":3,"high_miss_rate":0.11538461538461539,"non_high_cases":44,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true},"wikipedia-mcp-reference":{"omitted_subject":"wikipedia-mcp-reference","high_cases":27,"high_misses":3,"high_miss_rate":0.1111111111111111,"non_high_cases":43,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true},"google-maps-mcp":{"omitted_subject":"google-maps-mcp","high_cases":30,"high_misses":3,"high_miss_rate":0.1,"non_high_cases":40,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true},"metropolitan-museum-mcp":{"omitted_subject":"metropolitan-museum-mcp","high_cases":32,"high_misses":3,"high_miss_rate":0.09375,"non_high_cases":38,"non_high_misses":0,"non_high_miss_rate":0.0,"direction_high_at_least_non_high":true,"strict_high_greater":true,"informative_failure_signal_remains":true}},"directional_fraction":1.0,"evaluable_omissions":11,"direction_count":11,"all_direction_at_least_non_high":true,"strict_signal_remains_omissions":10,"criterion_pass":true,"interpretation":"Direction is non-reversed in every omission, but omitting DeepWiki removes all failures; cross-subject predictive validity is therefore not established."},"boundary_family_sensitivity":{"recurring_family_count":11,"high_miss_counts_by_family":{"CONTENTS_VS_QUESTION":1,"CONTENTS_VS_STRUCTURE":1,"QUESTION_VS_CONTENTS":1},"max_high_miss_share":0.333333,"sensitivity_direction_count":11,"sensitivity_evaluable_count":11,"sensitivity_direction_fraction":1.0,"criterion_pass":true},"semantic_miss_count":3,"semantic_misses":[{"case_id":"b5-dw-003","subject_id":"deepwiki-mcp","frozen_prompt":"Read the documented procedure for running the repository's test suite; do not summarize it or list other topics.","expected_tool":"read_wiki_contents","selected_tool":"read_wiki_structure","plausible_competing_tools":["ask_question"],"selection_boundary":"CONTENTS_VS_QUESTION","difficulty":"H3","risk_score":51.6667,"risk_bucket":"HIGH","risk_factors":["DESCRIPTION_LEXICAL_OVERLAP","SHARED_REQUIRED_INPUTS","MISSING_OUTPUT_CONTRAST","INCOMPLETE_BOUNDARY_LANGUAGE"],"external_source_id":"deepwiki-aws-guide-2026-09-10","source_url":"https://docs.devin.ai/work-with-devin/deepwiki-mcp","source_version":"public guide and repository evidence captured 2026-09-10","source_excerpt_reference":"guide section on documentation contents","gold_label_changed":false,"original_prediction_preserved":true,"gold_validity":"VALID_GOLD_RETAINED","adjudication_reason":"The user requests source page content, while synthesized Q&A would produce an explanation instead."},{"case_id":"b5-dw-004","subject_id":"deepwiki-mcp","frozen_prompt":"Explain why the repository separates build and release stages, using its documentation as context.","expected_tool":"ask_question","selected_tool":"read_wiki_contents","plausible_competing_tools":["read_wiki_contents"],"selection_boundary":"QUESTION_VS_CONTENTS","difficulty":"H3","risk_score":51.6667,"risk_bucket":"HIGH","risk_factors":["DESCRIPTION_LEXICAL_OVERLAP","SHARED_REQUIRED_INPUTS","MISSING_OUTPUT_CONTRAST","INCOMPLETE_BOUNDARY_LANGUAGE"],"external_source_id":"deepwiki-aws-guide-2026-09-10","source_url":"https://docs.devin.ai/work-with-devin/deepwiki-mcp","source_version":"public guide and repository evidence captured 2026-09-10","source_excerpt_reference":"guide section on repository Q&A","gold_label_changed":false,"original_prediction_preserved":true,"gold_validity":"VALID_GOLD_RETAINED","adjudication_reason":"The requested output is an explanation, not a page dump."},{"case_id":"b5-dw-006","subject_id":"deepwiki-mcp","frozen_prompt":"Read the documentation page describing the repository's contribution workflow in full.","expected_tool":"read_wiki_contents","selected_tool":"read_wiki_structure","plausible_competing_tools":["read_wiki_structure"],"selection_boundary":"CONTENTS_VS_STRUCTURE","difficulty":"H3","risk_score":73.5714,"risk_bucket":"HIGH","risk_factors":["DESCRIPTION_LEXICAL_OVERLAP","IDENTICAL_REQUIRED_INPUTS_OR_BOTH_NULL","MISSING_OUTPUT_CONTRAST","INCOMPLETE_BOUNDARY_LANGUAGE"],"external_source_id":"deepwiki-aws-guide-2026-09-10","source_url":"https://docs.devin.ai/work-with-devin/deepwiki-mcp","source_version":"public guide and repository evidence captured 2026-09-10","source_excerpt_reference":"guide documentation-page example","gold_label_changed":false,"original_prediction_preserved":true,"gold_validity":"VALID_GOLD_RETAINED","adjudication_reason":"The requested artifact is page content, not the list of pages."}],"taxonomy_validation":{"RETRIEVAL_GRANULARITY_CONFUSION":{"status":"SUPPORTED_SINGLE_SUBJECT_ONLY","evidence":["b5-dw-003","b5-dw-006"],"confidence":"LOW"},"DEFAULT_GENERAL_TOOL_PREFERENCE":{"status":"SUPPORTED_SINGLE_SUBJECT_ONLY","evidence":["The three misses selected neighboring DeepWiki retrieval tools, but none establishes a general fallback preference across subjects."],"confidence":"LOW"},"QUESTION_VS_STRUCTURE_CONFUSION":{"status":"SUPPORTED_SINGLE_SUBJECT_ONLY","evidence":["b5-dw-003 and b5-dw-004 are contents/question/structure boundary misses within DeepWiki."],"confidence":"LOW_TO_MEDIUM"},"DESCRIPTION_ASYMMETRY":{"status":"PARTIALLY_SUPPORTED","evidence":["All misses carry scanner factors MISSING_OUTPUT_CONTRAST and INCOMPLETE_BOUNDARY_LANGUAGE, but all occur in one schema."],"confidence":"LOW"},"SCHEMA_SEMANTIC_GAP":{"status":"SUPPORTED_SINGLE_SUBJECT_ONLY","evidence":["DeepWiki's neighboring tools share repoName and rely on output semantics; no cross-subject recurrence of failure was observed."],"confidence":"LOW"}},"subjects":[{"canonical_id":"datagouv-mcp","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"deepwiki-mcp","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":4,"misses":3,"strict_accuracy":0.571429,"miss_rate":0.428571,"wilson_95":{"lower":0.250458,"upper":0.84178,"lower_percent":25.0458,"upper_percent":84.178},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"microsoft-release-communications","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"microsoft-learn","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"cloudflare-docs-mcp","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"exa-mcp","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"nixos-mcp","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"national-parks-mcp","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"wikipedia-mcp-reference","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"google-maps-mcp","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"metropolitan-museum-mcp","cases":7,"provider_valid_cases":7,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":7,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.64567,"upper":1.0,"lower_percent":64.567,"upper_percent":100.0},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"}],"frozen_measurement_identity":{"freeze_input_commit":"935e16c4cc073d72b1c64c48206196feaf28c170","final_freeze_receipt_commit":"4c11daed10b52027a8a31130703335e4835873ec","measurement_artifact_commit":"87a78aba4797f79f3c99c22e202eddd1da70f034","immutable_measurement_receipt_commit":"b064dea6a0ea488826d11fc0fc88c5b4de3d5c26","case_freeze_sha256":"40847d041af701492505676ae027f824248a94dba8dad19c4c764baec224a2bc","model_lock_sha256":"60c5653e461180e44e9621a54bb77f3f474e6889ddc29d1ff30d3cb9398013fa","scanner_freeze_sha256":"80b3b0d60a5ec2e3b5019d2c45838dd06517453ce64c9a32ed0dbda6be718795","validation_criteria_sha256":"0cd5821683b9f85ae37140bfc73d25064012d69aabbbbb4363e56b058cb240a6","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"612c995839a9ad1a32d87320055ada3201c511b626a0f3e8778a8bbe5201bc68","measurement_artifact_sha256":"9b5f0c7f94783f2b25d28d8ecdaebe3a5a11b9db00dd84f677f95ac008f32eeb","policy_evaluation_sha256":"2ef31f2a44a722fb862425dc11755b118c3ffb712672471d10d4cd0bbcee9af1"},"cross_batch_separation":{"expansion_v1_untouched":true,"external_gold_v1_untouched":true,"batch1_untouched":true,"batch2_untouched":true,"batch3_untouched":true,"batch4_untouched":true,"no_combined_headline_score":true},"strict_score_authoritative":true,"adjusted_accuracy_used":false,"target_mcp_business_tools_called":false,"isolated_real_model_executor_exposed":false,"scanner_production_validity_claimed":false,"interpretation":"This is benchmark-level S2 evidence only. Batch 5 provides a promising but incomplete cross-subject signal; it does not validate BOUNDARY_RISK_SCAN_V1 for experimental use because C2 failed."}