{"schema_version":"mcp-selection-lab.agent-trust-lab.selection-evidence-status-v6","track":"EXTERNAL_GOLD_EXPANSION_V2","status":"MEASURED","benchmark":"External Gold Expansion V2 Batch 6","benchmark_version":"external-gold-expansion-v2-batch6","batch_characterization":"NEW_SUBJECT_DISPERSION_HOLDOUT","benchmark_type":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","benchmark_provenance":"EXTERNAL_INDEPENDENT_GOLD_BENCHMARK","promotion_level":"S2","promotion_status":"S2 external-selection-tested","subject_level_promotions":[],"case_count":80,"strict_correct":80,"strict_score":"80/80","strict_accuracy":1.0,"strict_accuracy_display":"100%","wilson_95":{"lower":0.954182,"upper":1.0,"lower_percent":95.4182,"upper_percent":100.0},"wilson_95_display":"95.4182%–100%","provider_valid_count":80,"provider_valid_coverage":1.0,"semantic_coverage":1.0,"infrastructure_failure_count":0,"risk_bucket_metrics":{"HIGH":{"cases":32,"provider_valid_cases":32,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":32,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.892821,"upper":1.0,"lower_percent":89.2821,"upper_percent":100.0}},"MEDIUM":{"cases":20,"provider_valid_cases":20,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":20,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.838875,"upper":1.0,"lower_percent":83.8875,"upper_percent":100.0}},"LOW":{"cases":28,"provider_valid_cases":28,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":28,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.879357,"upper":1.0,"lower_percent":87.9357,"upper_percent":100.0}}},"risk_comparison":{"high":{"cases":32,"provider_valid_cases":32,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":32,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.892821,"upper":1.0,"lower_percent":89.2821,"upper_percent":100.0}},"low":{"cases":28,"provider_valid_cases":28,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":28,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.879357,"upper":1.0,"lower_percent":87.9357,"upper_percent":100.0}},"non_high":{"cases":48,"provider_valid_cases":48,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":48,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.9259,"upper":1.0,"lower_percent":92.59,"upper_percent":100.0}},"high_minus_low_miss_rate":0.0,"high_vs_low_risk_ratio":null,"high_vs_low_odds_ratio":null,"continuity_correction":{"method":"Haldane-Anscombe add 0.5 to each 2x2 cell","corrected_risk_ratio":0.878788,"corrected_odds_ratio":0.876923},"interpretation":"No semantic misses were observed in HIGH or LOW; uncorrected ratios are undefined and corrected estimates are unstable and non-validating."},"scanner_status":"WEAK_PREDICTIVE_SIGNAL","scanner_validation":{"decision":"WEAK_PREDICTIVE_SIGNAL","criteria":{"results":{"C1_HIGH_EXCEEDS_LOW":false,"C2_NON_DEEPWIKI_SUBJECT_DISPERSION":false,"C3_HIGH_MISS_CONCENTRATION":false,"C4_LEAVE_ONE_SUBJECT_OUT":true,"C5_BOUNDARY_FAMILY_ROBUSTNESS":true,"C6_NO_SINGLE_SUBJECT_DOMINATION":true,"C7_GOLD_INTEGRITY":true,"C8_PROVIDER_COMPLETENESS":true,"C9_SCANNER_IMMUTABILITY":true},"pass_count":6,"criterion_count":9,"failed_criteria":["C1_HIGH_EXCEEDS_LOW","C2_NON_DEEPWIKI_SUBJECT_DISPERSION","C3_HIGH_MISS_CONCENTRATION"]},"cross_subject_validated":false,"production_validity_claimed":false,"disclosure":"DeepWiki was excluded prospectively; no semantic failures occurred in the holdout, HIGH-risk cases did not underperform, and pre-frozen C1/C2/C3 failed. BOUNDARY_RISK_SCAN_V1 was therefore not validated."},"primary_holdout_excluded_subjects":["deepwiki-mcp"],"failure_dispersion":{"miss_count":0,"subject_count_with_miss":0,"subjects_with_miss":[],"largest_subject_miss_count":0,"largest_subject_miss_share":null,"boundary_family_count_with_miss":0,"boundary_families_with_miss":[],"interpretation":"No provider-valid semantic misses occurred in the primary non-DeepWiki holdout; cross-subject predictive failure dispersion is not demonstrated."},"subject_adjusted_analysis":{"method":"Within-subject HIGH versus non-HIGH comparison; all primary subjects with both partitions are retained.","evaluable_subject_count":6,"results":[{"subject_id":"datagouv-mcp","high_cases":6,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":2,"non_high_misses":0,"non_high_miss_rate":0.0,"high_at_least_non_high":true,"strictly_higher":false},{"subject_id":"microsoft-release-communications","high_cases":4,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":4,"non_high_misses":0,"non_high_miss_rate":0.0,"high_at_least_non_high":true,"strictly_higher":false},{"subject_id":"nixos-mcp","high_cases":6,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":2,"non_high_misses":0,"non_high_miss_rate":0.0,"high_at_least_non_high":true,"strictly_higher":false},{"subject_id":"national-parks-mcp","high_cases":6,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":2,"non_high_misses":0,"non_high_miss_rate":0.0,"high_at_least_non_high":true,"strictly_higher":false},{"subject_id":"wikipedia-mcp-reference","high_cases":6,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":2,"non_high_misses":0,"non_high_miss_rate":0.0,"high_at_least_non_high":true,"strictly_higher":false},{"subject_id":"google-maps-mcp","high_cases":4,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":4,"non_high_misses":0,"non_high_miss_rate":0.0,"high_at_least_non_high":true,"strictly_higher":false}],"high_at_least_non_high_count":6,"strictly_higher_count":0,"directional_fraction":1.0,"interpretation":"Descriptive subject-adjusted comparison only; all observed partitions have zero semantic misses, so no independent failure signal is established."},"leave_one_subject_out":{"results":{"datagouv-mcp":{"omitted_subject":"datagouv-mcp","high_cases":26,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":46,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false},"microsoft-release-communications":{"omitted_subject":"microsoft-release-communications","high_cases":28,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":44,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false},"microsoft-learn":{"omitted_subject":"microsoft-learn","high_cases":32,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":40,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false},"cloudflare-docs-mcp":{"omitted_subject":"cloudflare-docs-mcp","high_cases":32,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":40,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false},"exa-mcp":{"omitted_subject":"exa-mcp","high_cases":32,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":40,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false},"nixos-mcp":{"omitted_subject":"nixos-mcp","high_cases":26,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":46,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false},"national-parks-mcp":{"omitted_subject":"national-parks-mcp","high_cases":26,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":46,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false},"wikipedia-mcp-reference":{"omitted_subject":"wikipedia-mcp-reference","high_cases":26,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":46,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false},"google-maps-mcp":{"omitted_subject":"google-maps-mcp","high_cases":28,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":44,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false},"metropolitan-museum-mcp":{"omitted_subject":"metropolitan-museum-mcp","high_cases":32,"high_misses":0,"high_miss_rate":0.0,"non_high_cases":40,"non_high_misses":0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false,"informative_failure_signal_remains":false}},"evaluable_omissions":10,"direction_count":10,"directional_fraction":1.0,"criterion_pass":true,"interpretation":"Direction is non-reversed but entirely uninformative because no semantic failures occurred."},"boundary_family_sensitivity":{"results":{"ADDRESS_TO_COORDINATES_VS_COORDINATES_TO_ADDRESS":{"excluded_family":"ADDRESS_TO_COORDINATES_VS_COORDINATES_TO_ADDRESS","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"ALERTS_VS_EVENTS":{"excluded_family":"ALERTS_VS_EVENTS","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"ALERTS_VS_PARK_DISCOVERY":{"excluded_family":"ALERTS_VS_PARK_DISCOVERY","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"ARTICLE_DISCOVERY_VS_FULL_CONTENT":{"excluded_family":"ARTICLE_DISCOVERY_VS_FULL_CONTENT","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"AZURE_COLLECTION_VS_ITEM":{"excluded_family":"AZURE_COLLECTION_VS_ITEM","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"AZURE_ITEM_VS_COLLECTION":{"excluded_family":"AZURE_ITEM_VS_COLLECTION","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"AZURE_ITEM_VS_M365_ITEM":{"excluded_family":"AZURE_ITEM_VS_M365_ITEM","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"AZURE_VS_M365_COLLECTION":{"excluded_family":"AZURE_VS_M365_COLLECTION","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"CAMPGROUNDS_VS_VISITOR_CENTERS":{"excluded_family":"CAMPGROUNDS_VS_VISITOR_CENTERS","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"CODE_SAMPLE_VS_FULL_DOC":{"excluded_family":"CODE_SAMPLE_VS_FULL_DOC","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"CODE_VS_CONCEPTUAL_DOCS":{"excluded_family":"CODE_VS_CONCEPTUAL_DOCS","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"COORDINATES_TO_ADDRESS_VS_ADDRESS_TO_COORDINATES":{"excluded_family":"COORDINATES_TO_ADDRESS_VS_ADDRESS_TO_COORDINATES","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"CURRENT_DISCOVERY_VS_KNOWN_URL":{"excluded_family":"CURRENT_DISCOVERY_VS_KNOWN_URL","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DARWIN_OPTION_VS_HOME_MANAGER_OPTION":{"excluded_family":"DARWIN_OPTION_VS_HOME_MANAGER_OPTION","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DARWIN_VS_HOME_MANAGER_SEARCH":{"excluded_family":"DARWIN_VS_HOME_MANAGER_SEARCH","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DATASERVICE_VS_DATASET":{"excluded_family":"DATASERVICE_VS_DATASET","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DATASET_METADATA_VS_USAGE_METRICS":{"excluded_family":"DATASET_METADATA_VS_USAGE_METRICS","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DATASET_VS_DATASERVICE":{"excluded_family":"DATASET_VS_DATASERVICE","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DEDICATED_GUIDE_VS_GENERAL_SEARCH":{"excluded_family":"DEDICATED_GUIDE_VS_GENERAL_SEARCH","remaining_cases":77,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DEPARTMENT_INDEX_VS_OBJECT_DETAIL":{"excluded_family":"DEPARTMENT_INDEX_VS_OBJECT_DETAIL","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DEPARTMENT_INDEX_VS_OBJECT_SEARCH":{"excluded_family":"DEPARTMENT_INDEX_VS_OBJECT_SEARCH","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DISCOVERY_VS_KNOWN_PAGE":{"excluded_family":"DISCOVERY_VS_KNOWN_PAGE","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DISCOVERY_VS_PAGE_READ":{"excluded_family":"DISCOVERY_VS_PAGE_READ","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DISCOVERY_VS_PARK_DETAIL":{"excluded_family":"DISCOVERY_VS_PARK_DETAIL","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"DOCS_VS_CODE":{"excluded_family":"DOCS_VS_CODE","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"EVENTS_VS_ALERTS":{"excluded_family":"EVENTS_VS_ALERTS","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"FLAKE_SEARCH_VS_HOME_MANAGER_SEARCH":{"excluded_family":"FLAKE_SEARCH_VS_HOME_MANAGER_SEARCH","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"FULL_ARTICLE_VS_SECTIONS":{"excluded_family":"FULL_ARTICLE_VS_SECTIONS","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"FULL_ARTICLE_VS_SUMMARY":{"excluded_family":"FULL_ARTICLE_VS_SUMMARY","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"GENERAL_DOCS_VS_MIGRATION_GUIDE":{"excluded_family":"GENERAL_DOCS_VS_MIGRATION_GUIDE","remaining_cases":75,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"HOME_MANAGER_OPTION_VS_DARWIN_OPTION":{"excluded_family":"HOME_MANAGER_OPTION_VS_DARWIN_OPTION","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"HOME_MANAGER_SEARCH_VS_FLAKE_SEARCH":{"excluded_family":"HOME_MANAGER_SEARCH_VS_FLAKE_SEARCH","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"HOME_MANAGER_VS_DARWIN_SEARCH":{"excluded_family":"HOME_MANAGER_VS_DARWIN_SEARCH","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"KEY_FACTS_VS_GENERAL_SUMMARY":{"excluded_family":"KEY_FACTS_VS_GENERAL_SUMMARY","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"KNOWN_PAGE_VS_DISCOVERY":{"excluded_family":"KNOWN_PAGE_VS_DISCOVERY","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"KNOWN_PAGE_VS_SEARCH":{"excluded_family":"KNOWN_PAGE_VS_SEARCH","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"KNOWN_URL_VS_OPEN_SEARCH":{"excluded_family":"KNOWN_URL_VS_OPEN_SEARCH","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"KNOWN_URL_VS_SEARCH":{"excluded_family":"KNOWN_URL_VS_SEARCH","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"M365_COLLECTION_VS_ITEM":{"excluded_family":"M365_COLLECTION_VS_ITEM","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"M365_ITEM_VS_AZURE_ITEM":{"excluded_family":"M365_ITEM_VS_AZURE_ITEM","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"M365_ITEM_VS_COLLECTION":{"excluded_family":"M365_ITEM_VS_COLLECTION","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"M365_VS_AZURE_COLLECTION":{"excluded_family":"M365_VS_AZURE_COLLECTION","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"NIXOS_INFO_VS_SEARCH":{"excluded_family":"NIXOS_INFO_VS_SEARCH","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"NIXOS_SEARCH_VS_INFO":{"excluded_family":"NIXOS_SEARCH_VS_INFO","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"OBJECT_DETAIL_VS_OBJECT_SEARCH":{"excluded_family":"OBJECT_DETAIL_VS_OBJECT_SEARCH","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"OBJECT_SEARCH_VS_OBJECT_DETAIL":{"excluded_family":"OBJECT_SEARCH_VS_OBJECT_DETAIL","remaining_cases":77,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"OPENAPI_VS_SERVICE_METADATA":{"excluded_family":"OPENAPI_VS_SERVICE_METADATA","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"PARK_DETAIL_VS_DISCOVERY":{"excluded_family":"PARK_DETAIL_VS_DISCOVERY","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"PARK_DISCOVERY_VS_ALERTS":{"excluded_family":"PARK_DISCOVERY_VS_ALERTS","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"PLACE_DETAIL_VS_PLACE_DISCOVERY":{"excluded_family":"PLACE_DETAIL_VS_PLACE_DISCOVERY","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"PLACE_DISCOVERY_VS_PLACE_DETAIL":{"excluded_family":"PLACE_DISCOVERY_VS_PLACE_DETAIL","remaining_cases":78,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"RESOURCE_METADATA_VS_QUERY":{"excluded_family":"RESOURCE_METADATA_VS_QUERY","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"RESOURCE_QUERY_VS_METADATA":{"excluded_family":"RESOURCE_QUERY_VS_METADATA","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"SECTIONS_VS_FULL_ARTICLE":{"excluded_family":"SECTIONS_VS_FULL_ARTICLE","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"SERVICE_METADATA_VS_OPENAPI":{"excluded_family":"SERVICE_METADATA_VS_OPENAPI","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"SUMMARY_VS_FULL_ARTICLE":{"excluded_family":"SUMMARY_VS_FULL_ARTICLE","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"SUMMARY_VS_SECTIONS":{"excluded_family":"SUMMARY_VS_SECTIONS","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"USAGE_METRICS_VS_DATASET_METADATA":{"excluded_family":"USAGE_METRICS_VS_DATASET_METADATA","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"VISITOR_CENTERS_VS_CAMPGROUNDS":{"excluded_family":"VISITOR_CENTERS_VS_CAMPGROUNDS","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false},"WEB_DISCOVERY_VS_KNOWN_PAGE":{"excluded_family":"WEB_DISCOVERY_VS_KNOWN_PAGE","remaining_cases":79,"high_miss_rate":0.0,"non_high_miss_rate":0.0,"evaluable":true,"direction_high_at_least_non_high":true,"strict_direction":false}},"evaluable_count":60,"direction_count":60,"directional_fraction":1.0},"semantic_miss_count":0,"semantic_misses":[],"taxonomy_validation":{"RETRIEVAL_GRANULARITY_CONFUSION":{"status":"INSUFFICIENT_EVIDENCE","confidence":"LOW","evidence":["Batch 6 primary holdout excluded DeepWiki and observed zero semantic misses; no new cross-subject recurrence supports this signal."]},"DEFAULT_GENERAL_TOOL_PREFERENCE":{"status":"INSUFFICIENT_EVIDENCE","confidence":"LOW","evidence":["Batch 6 primary holdout excluded DeepWiki and observed zero semantic misses; no new cross-subject recurrence supports this signal."]},"QUESTION_VS_STRUCTURE_CONFUSION":{"status":"INSUFFICIENT_EVIDENCE","confidence":"LOW","evidence":["Batch 6 primary holdout excluded DeepWiki and observed zero semantic misses; no new cross-subject recurrence supports this signal."]},"DESCRIPTION_ASYMMETRY":{"status":"INSUFFICIENT_EVIDENCE","confidence":"LOW","evidence":["Batch 6 primary holdout excluded DeepWiki and observed zero semantic misses; no new cross-subject recurrence supports this signal."]},"SCHEMA_SEMANTIC_GAP":{"status":"INSUFFICIENT_EVIDENCE","confidence":"LOW","evidence":["Batch 6 primary holdout excluded DeepWiki and observed zero semantic misses; no new cross-subject recurrence supports this signal."]}},"subjects":[{"canonical_id":"datagouv-mcp","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"HIGH":6,"LOW":2},"HIGH":{"cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0}},"MEDIUM":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"LOW":{"cases":2,"provider_valid_cases":2,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":2,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.34238,"upper":1.0,"lower_percent":34.238,"upper_percent":100.0}},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"microsoft-release-communications","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"HIGH":4,"LOW":4},"HIGH":{"cases":4,"provider_valid_cases":4,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":4,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.510109,"upper":1.0,"lower_percent":51.0109,"upper_percent":100.0}},"MEDIUM":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"LOW":{"cases":4,"provider_valid_cases":4,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":4,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.510109,"upper":1.0,"lower_percent":51.0109,"upper_percent":100.0}},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"microsoft-learn","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"LOW":8},"HIGH":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"MEDIUM":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"LOW":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"cloudflare-docs-mcp","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"LOW":8},"HIGH":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"MEDIUM":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"LOW":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"exa-mcp","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"MEDIUM":8},"HIGH":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"MEDIUM":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"LOW":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"nixos-mcp","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"HIGH":6,"MEDIUM":2},"HIGH":{"cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0}},"MEDIUM":{"cases":2,"provider_valid_cases":2,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":2,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.34238,"upper":1.0,"lower_percent":34.238,"upper_percent":100.0}},"LOW":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"national-parks-mcp","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"HIGH":6,"LOW":2},"HIGH":{"cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0}},"MEDIUM":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"LOW":{"cases":2,"provider_valid_cases":2,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":2,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.34238,"upper":1.0,"lower_percent":34.238,"upper_percent":100.0}},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"wikipedia-mcp-reference","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"HIGH":6,"MEDIUM":2},"HIGH":{"cases":6,"provider_valid_cases":6,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":6,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.609666,"upper":1.0,"lower_percent":60.9666,"upper_percent":100.0}},"MEDIUM":{"cases":2,"provider_valid_cases":2,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":2,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.34238,"upper":1.0,"lower_percent":34.238,"upper_percent":100.0}},"LOW":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"google-maps-mcp","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"HIGH":4,"LOW":4},"HIGH":{"cases":4,"provider_valid_cases":4,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":4,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.510109,"upper":1.0,"lower_percent":51.0109,"upper_percent":100.0}},"MEDIUM":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"LOW":{"cases":4,"provider_valid_cases":4,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":4,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.510109,"upper":1.0,"lower_percent":51.0109,"upper_percent":100.0}},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"},{"canonical_id":"metropolitan-museum-mcp","overall":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"risk_bucket_distribution":{"MEDIUM":8},"HIGH":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"MEDIUM":{"cases":8,"provider_valid_cases":8,"infrastructure_failures":0,"semantic_coverage":1.0,"correct":8,"misses":0,"strict_accuracy":1.0,"miss_rate":0.0,"wilson_95":{"lower":0.675592,"upper":1.0,"lower_percent":67.5592,"upper_percent":100.0}},"LOW":{"cases":0,"provider_valid_cases":0,"infrastructure_failures":0,"semantic_coverage":null,"correct":0,"misses":0,"strict_accuracy":null,"miss_rate":null,"wilson_95":null},"badge":null,"promotion_status":"DESCRIPTIVE_ONLY_NO_SUBJECT_LEVEL_POLICY_PROMOTION"}],"frozen_measurement_identity":{"freeze_input_commit":"95d1a1017ab4b0fde8dfb3dbab3140997867a07b","final_freeze_receipt_commit":"6b1957bf7d556dbf023ac24cf22e50bcd64b0136","measurement_artifact_commit":"8854ae3a9e69abfc09cd63fb0daaf9fe5b34fc63","immutable_measurement_receipt_binding_commit":"df7c3c0a04b91e4d113e445f1ed4d78a212fe6dd","case_freeze_sha256":"4e7500bb9cc1bb4c2b1bbdc757814a7eb2dd5eac48f131665328c1d886f6960b","model_lock_sha256":"c020241636261d34903e5155cb6177402ce658c0f6693f3efaf75bd3946829ee","scanner_freeze_sha256":"b21d5f9693cecf1d29554e4064de814c9b3d64a2c41ea9b446430cd0e68a0a67","validation_criteria_sha256":"6c081fe020106093c25f85b40eb2960faee1bd88337ca3688f231573f06b36b5","promotion_policy_sha256":"6714ca504b2c5679c0f1ba0c7da2e5ee52f61b70b0e814ecaf9071f080f9f61d","raw_predictions_sha256":"ed1eac7db87bf2be0dbc9f94acf79a2f72cc9a5dc9dbb33e5935a0a5c8971c51","measurement_artifact_sha256":"e662d13721d11587fb102fe4699b16906c566d1342286a1c946424653aacd912","policy_evaluation_sha256":"94bbf82474a26461f303406b30a671ef776c31cadbd229e627e114c538acffd1","scanner_validation_result_sha256":"c182332ae5ec6829b68054f1575066c35bf89b8d088b65322d5798e1d8042192"},"cross_batch_separation":{"expansion_v1_untouched":true,"external_gold_v1_untouched":true,"batch1_untouched":true,"batch2_untouched":true,"batch3_untouched":true,"batch4_untouched":true,"batch5_untouched":true,"no_combined_headline_score":true},"strict_score_authoritative":true,"adjusted_accuracy_used":false,"target_mcp_business_tools_called":false,"isolated_real_model_executor_exposed":false,"scanner_production_validity_claimed":false,"interpretation":"This is benchmark-level S2 evidence and measured historical selection performance from a frozen holdout. It does not validate BOUNDARY_RISK_SCAN_V1, predict future errors, or provide a reliable pre-measurement failure forecast."}