{"id":"https://openalex.org/W3199126386","doi":"https://doi.org/10.1109/iccv48922.2021.01053","title":"Low-Shot Validation: Active Importance Sampling for Estimating Classifier Performance on Rare Categories","display_name":"Low-Shot Validation: Active Importance Sampling for Estimating Classifier Performance on Rare Categories","publication_year":2021,"publication_date":"2021-10-01","ids":{"openalex":"https://openalex.org/W3199126386","doi":"https://doi.org/10.1109/iccv48922.2021.01053","mag":"3199126386"},"language":"en","primary_location":{"is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccv48922.2021.01053","pdf_url":null,"source":{"id":"https://openalex.org/S4363607764","display_name":"2021 IEEE/CVF International Conference on Computer Vision (ICCV)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false},"type":"article","type_crossref":"proceedings-article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2109.05720","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027115836","display_name":"Fait Poms","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fait Poms","raw_affiliation_strings":["Stanford University"],"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069897950","display_name":"Vishnu Sarukkai","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vishnu Sarukkai","raw_affiliation_strings":["Stanford University"],"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006412542","display_name":"Ravi Teja Mullapudi","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ravi Teja Mullapudi","raw_affiliation_strings":["Carnegie Mellon University"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070318397","display_name":"Nimit S. Sohoni","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nimit S. Sohoni","raw_affiliation_strings":["Stanford University"],"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109394409","display_name":"William R. Mark","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"William R. Mark","raw_affiliation_strings":["Google"],"affiliations":[{"raw_affiliation_string":"Google","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004353237","display_name":"Deva Ramanan","orcid":"https://orcid.org/0009-0008-9180-8983"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Deva Ramanan","raw_affiliation_strings":["Argo AI","Carnegie Mellon University"],"affiliations":[{"raw_affiliation_string":"Argo AI","institution_ids":[]},{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5037444018","display_name":"Kayvon Fatahalian","orcid":"https://orcid.org/0000-0001-8754-0429"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kayvon Fatahalian","raw_affiliation_strings":["Stanford University"],"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]}],"institution_assertions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.539,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.766735,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":79,"max":81},"biblio":{"volume":"10","issue":null,"first_page":"10685","last_page":"10694"},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9994,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9994,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.998,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.53297806},{"id":"https://openalex.org/keywords/binary-classification","display_name":"Binary classification","score":0.49174908}],"concepts":[{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.7575942},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7215771},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.6590866},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.65808237},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.61958927},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.59627473},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5681559},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.53297806},{"id":"https://openalex.org/C66905080","wikidata":"https://www.wikidata.org/wiki/Q17005494","display_name":"Binary classification","level":3,"score":0.49174908},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.49076375},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.43517822},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4055352},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.33549517},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.17643327},{"id":"https://openalex.org/C12267149","wikidata":"https://www.wikidata.org/wiki/Q282453","display_name":"Support vector machine","level":2,"score":0.14214665},{"id":"https://openalex.org/C94915269","wikidata":"https://www.wikidata.org/wiki/Q1834857","display_name":"Detector","level":2,"score":0.09048313},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccv48922.2021.01053","pdf_url":null,"source":{"id":"https://openalex.org/S4363607764","display_name":"2021 IEEE/CVF International Conference on Computer Vision (ICCV)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false},{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2109.05720","pdf_url":"https://arxiv.org/pdf/2109.05720","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2109.05720","pdf_url":"https://arxiv.org/pdf/2109.05720","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[],"grants":[{"funder":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation","award_id":null}],"datasets":[],"versions":[],"referenced_works_count":39,"referenced_works":["https://openalex.org/W1618905105","https://openalex.org/W1997865285","https://openalex.org/W2011034758","https://openalex.org/W2012942264","https://openalex.org/W2015731569","https://openalex.org/W2043779491","https://openalex.org/W2098824882","https://openalex.org/W2116917894","https://openalex.org/W2117539524","https://openalex.org/W2128511958","https://openalex.org/W2570764145","https://openalex.org/W2604763608","https://openalex.org/W2626967530","https://openalex.org/W2762929692","https://openalex.org/W2769041395","https://openalex.org/W2774171063","https://openalex.org/W2774918944","https://openalex.org/W2783389297","https://openalex.org/W2794363191","https://openalex.org/W2797977484","https://openalex.org/W2804869193","https://openalex.org/W2944828972","https://openalex.org/W2949071206","https://openalex.org/W2951786554","https://openalex.org/W2955260427","https://openalex.org/W2959716049","https://openalex.org/W2963902936","https://openalex.org/W3006924828","https://openalex.org/W3035060554","https://openalex.org/W3036224891","https://openalex.org/W3036982689","https://openalex.org/W3041053621","https://openalex.org/W3082701951","https://openalex.org/W3100859887","https://openalex.org/W3101098106","https://openalex.org/W3105347387","https://openalex.org/W3171229070","https://openalex.org/W4206723194","https://openalex.org/W4294646197"],"related_works":["https://openalex.org/W4382934300","https://openalex.org/W4285388059","https://openalex.org/W3122602933","https://openalex.org/W2950038056","https://openalex.org/W2595172197","https://openalex.org/W2289285490","https://openalex.org/W2127970246","https://openalex.org/W2121061354","https://openalex.org/W1657880117","https://openalex.org/W1544940847"],"abstract_inverted_index":{"For":[0],"machine":[1],"learning":[2],"models":[3,114],"trained":[4],"with":[5,128,145],"limited":[6],"labeled":[7],"training":[8],"data,":[9],"validation":[10,26],"stands":[11],"to":[12,17,43,99,130],"become":[13],"the":[14,31,64,79,122],"main":[15],"bottleneck":[16],"reducing":[18],"overall":[19],"annotation":[20],"costs.":[21],"We":[22],"propose":[23],"a":[24,97,106,146],"statistical":[25],"algorithm":[27],"that":[28,53,86],"accurately":[29],"estimates":[30,61,124],"F-score":[32],"of":[33,78,81,125,148],"binary":[34],"classifiers":[35],"for":[36],"rare":[37],"categories,":[38],"where":[39],"finding":[40],"relevant":[41],"examples":[42],"evaluate":[44],"on":[45,115],"is":[46,52,89],"particularly":[47],"challenging.":[48],"Our":[49],"key":[50],"insight":[51],"simultaneous":[54],"calibration":[55],"and":[56,84,117],"importance":[57],"sampling":[58],"enables":[59],"accurate":[60,75,91],"even":[62],"in":[63],"low-sample":[65,108],"regime":[66],"(<":[67],"300":[68],"samples).":[69],"Critically,":[70],"we":[71,139],"also":[72],"derive":[73],"an":[74],"single-trial":[76],"estimator":[77,88],"variance":[80,147],"our":[82,119],"method":[83,120],"demonstrate":[85],"this":[87],"empirically":[90],"at":[92],"low":[93],"sample":[94],"counts,":[95],"enabling":[96],"practitioner":[98],"know":[100],"how":[101],"well":[102],"they":[103],"can":[104,140],"trust":[105],"given":[107],"estimate.":[109],"When":[110],"validating":[111],"state-of-the-art":[112],"semi-supervised":[113],"ImageNet":[116],"iNatural-ist2017,":[118],"achieves":[121],"same":[123],"model":[126,142],"performance":[127],"up":[129],"10\u00d7":[131],"fewer":[132],"labels":[133],"than":[134],"competing":[135],"approaches.":[136],"In":[137],"particular,":[138],"estimate":[141],"F1":[143],"scores":[144],"0.005":[149],"using":[150],"as":[151,153],"few":[152],"100":[154],"labels.":[155]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3199126386","counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2}],"updated_date":"2024-12-06T00:42:23.317281","created_date":"2021-09-27"}