{"id":"https://openalex.org/W4380357773","doi":"https://doi.org/10.1145/3593013.3594019","title":"Representation in AI Evaluations","display_name":"Representation in AI Evaluations","publication_year":2023,"publication_date":"2023-06-12","ids":{"openalex":"https://openalex.org/W4380357773","doi":"https://doi.org/10.1145/3593013.3594019"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.1145/3593013.3594019","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3593013.3594019","source":{"id":"https://openalex.org/S4363608463","display_name":"2022 ACM Conference on Fairness, Accountability, and Transparency","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"type":"article","type_crossref":"proceedings-article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3593013.3594019","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5031719232","display_name":"A. S. Bergman","orcid":"https://orcid.org/0000-0002-4331-1357"},"institutions":[],"countries":["US"],"is_corresponding":false,"raw_author_name":"A. Stevie Bergman","raw_affiliation_strings":["DeepMind, USA"],"affiliations":[{"raw_affiliation_string":"DeepMind, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020758501","display_name":"Lisa Anne Hendricks","orcid":"https://orcid.org/0000-0001-9340-5143"},"institutions":[{"id":"https://openalex.org/I4210090411","display_name":"DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Lisa Anne Hendricks","raw_affiliation_strings":["DeepMind, United Kingdom"],"affiliations":[{"raw_affiliation_string":"DeepMind, United Kingdom","institution_ids":["https://openalex.org/I4210090411"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080160770","display_name":"Maribeth Rauh","orcid":null},"institutions":[{"id":"https://openalex.org/I4210090411","display_name":"DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Maribeth Rauh","raw_affiliation_strings":["DeepMind, United Kingdom"],"affiliations":[{"raw_affiliation_string":"DeepMind, United Kingdom","institution_ids":["https://openalex.org/I4210090411"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063588445","display_name":"Boxi Wu","orcid":"https://orcid.org/0009-0007-0641-6065"},"institutions":[{"id":"https://openalex.org/I4210090411","display_name":"DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Boxi Wu","raw_affiliation_strings":["DeepMind, United Kingdom"],"affiliations":[{"raw_affiliation_string":"DeepMind, United Kingdom","institution_ids":["https://openalex.org/I4210090411"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011791808","display_name":"William S. Agnew","orcid":"https://orcid.org/0000-0002-1362-554X"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"William Agnew","raw_affiliation_strings":["University of Washington, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Washington, United Kingdom","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043562314","display_name":"Markus Kunesch","orcid":"https://orcid.org/0000-0003-3818-7897"},"institutions":[{"id":"https://openalex.org/I4210090411","display_name":"DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Markus Kunesch","raw_affiliation_strings":["DeepMind, United Kingdom"],"affiliations":[{"raw_affiliation_string":"DeepMind, United Kingdom","institution_ids":["https://openalex.org/I4210090411"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092146268","display_name":"Isabella Duan","orcid":"https://orcid.org/0009-0001-9766-0886"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Isabella Duan","raw_affiliation_strings":["University of Chicago, USA"],"affiliations":[{"raw_affiliation_string":"University of Chicago, USA","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032814084","display_name":"Iason Gabriel","orcid":"https://orcid.org/0000-0002-7552-4576"},"institutions":[{"id":"https://openalex.org/I4210090411","display_name":"DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Iason Gabriel","raw_affiliation_strings":["DeepMind, United Kingdom"],"affiliations":[{"raw_affiliation_string":"DeepMind, United Kingdom","institution_ids":["https://openalex.org/I4210090411"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5048272174","display_name":"William Isaac","orcid":"https://orcid.org/0000-0002-1297-5409"},"institutions":[{"id":"https://openalex.org/I4210090411","display_name":"DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"William Isaac","raw_affiliation_strings":["DeepMind, United Kingdom"],"affiliations":[{"raw_affiliation_string":"DeepMind, United Kingdom","institution_ids":["https://openalex.org/I4210090411"]}]}],"institution_assertions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.687,"has_fulltext":true,"fulltext_origin":"pdf","cited_by_count":6,"citation_normalized_percentile":{"value":0.870098,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":92,"max":93},"biblio":{"volume":null,"issue":null,"first_page":"519","last_page":"533"},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9981,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9981,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9952,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9489,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/representation","display_name":"Representation","score":0.5110179}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.70102954},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5110179},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.47695103},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.32416248},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"is_oa":true,"landing_page_url":"https://doi.org/10.1145/3593013.3594019","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3593013.3594019","source":{"id":"https://openalex.org/S4363608463","display_name":"2022 ACM Conference on Fairness, Accountability, and Transparency","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.1145/3593013.3594019","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3593013.3594019","source":{"id":"https://openalex.org/S4363608463","display_name":"2022 ACM Conference on Fairness, Accountability, and Transparency","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"sustainable_development_goals":[],"grants":[],"datasets":[],"versions":[],"referenced_works_count":45,"referenced_works":["https://openalex.org/W1792712577","https://openalex.org/W1993953087","https://openalex.org/W2108598243","https://openalex.org/W2153066044","https://openalex.org/W2194566175","https://openalex.org/W2560674852","https://openalex.org/W2807975560","https://openalex.org/W2897154134","https://openalex.org/W2913593702","https://openalex.org/W2962059918","https://openalex.org/W2990751682","https://openalex.org/W3010093963","https://openalex.org/W3013571594","https://openalex.org/W3035507081","https://openalex.org/W3037831233","https://openalex.org/W3084611909","https://openalex.org/W3098075008","https://openalex.org/W3101662419","https://openalex.org/W3118813946","https://openalex.org/W3119719070","https://openalex.org/W3133893548","https://openalex.org/W3134889244","https://openalex.org/W3135371071","https://openalex.org/W3135514117","https://openalex.org/W3159003756","https://openalex.org/W3172415559","https://openalex.org/W3177219738","https://openalex.org/W3181414820","https://openalex.org/W3185376810","https://openalex.org/W3202415077","https://openalex.org/W3204130547","https://openalex.org/W3212368439","https://openalex.org/W4220820301","https://openalex.org/W4221138850","https://openalex.org/W4233063040","https://openalex.org/W4249922884","https://openalex.org/W4283166422","https://openalex.org/W4283170908","https://openalex.org/W4287073009","https://openalex.org/W4287239856","https://openalex.org/W4287332702","https://openalex.org/W4302071234","https://openalex.org/W4308789076","https://openalex.org/W4388161807","https://openalex.org/W4390006064"],"related_works":["https://openalex.org/W4391913857","https://openalex.org/W3204019825","https://openalex.org/W2748952813","https://openalex.org/W2478288626","https://openalex.org/W2390279801","https://openalex.org/W2382290278","https://openalex.org/W2376932109","https://openalex.org/W2358668433","https://openalex.org/W2350741829","https://openalex.org/W2001405890"],"abstract_inverted_index":{"Calls":[0],"for":[1,42,143,153],"representation":[2,77,88,158,166],"in":[3,89,159,167],"artificial":[4],"intelligence":[5],"(AI)":[6],"and":[7,24,32,61,122,135,146,172],"machine":[8],"learning":[9],"(ML)":[10],"are":[11],"widespread,":[12],"with":[13,71],"\"representation\"":[14],"or":[15,101],"\"representativeness\"":[16],"generally":[17],"understood":[18],"to":[19,34,46,73,92,96],"be":[20,47],"both":[21],"an":[22,29,43,98],"instrumentally":[23,125,164],"intrinsically":[25],"beneficial":[26],"quality":[27],"of":[28,51,76,87,106,124,132,140],"AI":[30,44,53,90,99],"system,":[31,65],"central":[33],"fairness":[35],"concerns.":[36],"But":[37],"what":[38,74],"does":[39],"it":[40],"mean":[41],"system":[45],"\"representative\"?":[48],"Each":[49],"element":[50],"the":[52,64,85,104,120,130,151],"lifecycle":[54],"is":[55,78,112],"geared":[56],"towards":[57,103,169],"its":[58,68],"own":[59,69],"goals":[60],"effect":[62],"on":[63,157],"therefore":[66],"requiring":[67],"analyses":[70],"regard":[72],"kind":[75],"best.":[79],"In":[80],"this":[81],"work":[82,149],"we":[83],"untangle":[84],"benefits":[86],"evaluations":[91,168],"develop":[93],"a":[94,114,154],"framework":[95],"guide":[97],"practitioner":[100],"auditor":[102],"creation":[105],"representative":[107,126],"ML":[108],"evaluations.":[109],"Representation,":[110],"however,":[111],"not":[113],"panacea.":[115],"We":[116],"further":[117],"lay":[118],"out":[119],"limitations":[121],"tensions":[123],"datasets,":[127],"such":[128],"as":[129],"necessity":[131],"data":[133],"existence":[134],"access,":[136],"surveillance":[137],"vs":[138],"expectations":[139],"privacy,":[141],"implications":[142],"foundation":[144],"models":[145],"power.":[147],"This":[148],"sets":[150],"stage":[152],"research":[155],"agenda":[156],"AI,":[160],"which":[161],"extends":[162],"beyond":[163],"valuable":[165],"refocusing":[170],"on,":[171],"empowering,":[173],"impacted":[174],"communities.":[175]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4380357773","counts_by_year":[{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":2}],"updated_date":"2025-01-20T06:38:17.338690","created_date":"2023-06-13"}