{"id":"https://openalex.org/W3081286828","doi":"https://doi.org/10.1145/3394486.3403199","title":"Evaluating Fairness Using Permutation Tests","display_name":"Evaluating Fairness Using Permutation Tests","publication_year":2020,"publication_date":"2020-08-20","ids":{"openalex":"https://openalex.org/W3081286828","doi":"https://doi.org/10.1145/3394486.3403199","mag":"3081286828"},"language":"en","primary_location":{"is_oa":false,"landing_page_url":"https://doi.org/10.1145/3394486.3403199","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false},"type":"article","type_crossref":"proceedings-article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2007.05124","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5068232760","display_name":"Cyrus DiCiccio","orcid":"https://orcid.org/0009-0006-2803-858X"},"institutions":[{"id":"https://openalex.org/I1316064682","display_name":"LinkedIn (United States)","ror":"https://ror.org/02fyxhe35","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I1316064682"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cyrus DiCiccio","raw_affiliation_strings":["LinkedIn Corporation, Mountain View, CA, USA"],"affiliations":[{"raw_affiliation_string":"LinkedIn Corporation, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1316064682"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103486798","display_name":"Sriram Vasudevan","orcid":null},"institutions":[{"id":"https://openalex.org/I1316064682","display_name":"LinkedIn (United States)","ror":"https://ror.org/02fyxhe35","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I1316064682"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sriram Vasudevan","raw_affiliation_strings":["LinkedIn Corporation, Mountain View, CA, USA"],"affiliations":[{"raw_affiliation_string":"LinkedIn Corporation, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1316064682"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078166693","display_name":"Kinjal Basu","orcid":"https://orcid.org/0000-0002-4091-0119"},"institutions":[{"id":"https://openalex.org/I1316064682","display_name":"LinkedIn (United States)","ror":"https://ror.org/02fyxhe35","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I1316064682"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kinjal Basu","raw_affiliation_strings":["LinkedIn Corporation, Mountain View, CA, USA"],"affiliations":[{"raw_affiliation_string":"LinkedIn Corporation, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1316064682"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002843568","display_name":"Krishnaram Kenthapadi","orcid":"https://orcid.org/0000-0003-1237-087X"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Krishnaram Kenthapadi","raw_affiliation_strings":["Amazon AWS AI, Mountain View, CA, USA"],"affiliations":[{"raw_affiliation_string":"Amazon AWS AI, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101736089","display_name":"Deepak Agarwal","orcid":"https://orcid.org/0000-0003-2590-2206"},"institutions":[{"id":"https://openalex.org/I1316064682","display_name":"LinkedIn (United States)","ror":"https://ror.org/02fyxhe35","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I1316064682"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Deepak Agarwal","raw_affiliation_strings":["LinkedIn Corporation, Mountain View, CA, USA"],"affiliations":[{"raw_affiliation_string":"LinkedIn Corporation, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1316064682"]}]}],"institution_assertions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.565,"has_fulltext":false,"cited_by_count":21,"citation_normalized_percentile":{"value":0.998221,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":92,"max":93},"biblio":{"volume":null,"issue":null,"first_page":"1467","last_page":"1477"},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethical Implications of Artificial Intelligence","score":0.9995,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethical Implications of Artificial Intelligence","score":0.9995,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence","score":0.9945,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Deep Learning Models","score":0.9941,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/federated-learning","display_name":"Federated Learning","score":0.512594},{"id":"https://openalex.org/keywords/fairness","display_name":"Fairness","score":0.511062}],"concepts":[{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.7624401},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.72218484},{"id":"https://openalex.org/C21308566","wikidata":"https://www.wikidata.org/wiki/Q7169365","display_name":"Permutation (music)","level":2,"score":0.6845273},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.56944585},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.477651},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.47044057},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.44909132},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.403187},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32665542},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.13887265},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.10215399},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C24890656","wikidata":"https://www.wikidata.org/wiki/Q82811","display_name":"Acoustics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"is_oa":false,"landing_page_url":"https://doi.org/10.1145/3394486.3403199","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false},{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2007.05124","pdf_url":"https://arxiv.org/pdf/2007.05124","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2007.05124","pdf_url":"https://arxiv.org/pdf/2007.05124","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, justice, and strong institutions","score":0.41}],"grants":[],"datasets":[],"versions":[],"referenced_works_count":14,"referenced_works":["https://openalex.org/W1115930093","https://openalex.org/W1559060276","https://openalex.org/W1593806347","https://openalex.org/W2020018978","https://openalex.org/W2100960835","https://openalex.org/W2162445884","https://openalex.org/W2507358938","https://openalex.org/W2730550703","https://openalex.org/W2950018712","https://openalex.org/W3013778941","https://openalex.org/W307744830","https://openalex.org/W3106076062","https://openalex.org/W4210309498","https://openalex.org/W4289751798"],"related_works":["https://openalex.org/W2390968135","https://openalex.org/W2388563748","https://openalex.org/W2382997850","https://openalex.org/W2382213751","https://openalex.org/W2375179084","https://openalex.org/W2366646518","https://openalex.org/W2354715126","https://openalex.org/W2351750670","https://openalex.org/W2043222232","https://openalex.org/W1597848696"],"abstract_inverted_index":{"Machine":[0],"learning":[1,76],"models":[2,25],"are":[3,36,108],"central":[4],"to":[5,29,32,54,58,103,117,126,134,143,156],"people's":[6],"lives":[7],"and":[8,42],"impact":[9],"society":[10],"in":[11,39,138,162],"ways":[12],"as":[13,15,151,153],"fundamental":[14],"determining":[16],"how":[17,158],"people":[18],"access":[19],"information.":[20],"The":[21],"gravity":[22],"of":[23,70,112,120],"these":[24],"imparts":[26],"a":[27,40,47,74,84,90,94,128,147],"responsibility":[28],"model":[30,48,77,95,121],"developers":[31],"ensure":[33],"that":[34,88,93,114,131],"they":[35,141],"treating":[37],"users":[38],"fair":[41,97],"equitable":[43],"manner.":[44],"Before":[45],"deploying":[46],"into":[49],"production,":[50],"it":[51],"is":[52,96,125],"crucial":[53],"examine":[55],"the":[56,68],"extent":[57],"which":[59],"its":[60],"predictions":[61],"demonstrate":[62],"biases.":[63],"This":[64],"paper":[65],"deals":[66],"with":[67,101],"detection":[69],"bias":[71],"exhibited":[72],"by":[73],"machine":[75],"through":[78],"statistical":[79],"hypothesis":[80,91],"testing.":[81],"We":[82,145],"propose":[83],"permutation":[85],"testing":[86,149],"methodology":[87],"performs":[89],"test":[92],"across":[98],"two":[99],"groups":[100],"respect":[102],"any":[104,139],"given":[105],"metric.":[106],"There":[107],"increasingly":[109],"many":[110],"notions":[111],"fairness":[113],"can":[115],"speak":[116],"different":[118],"aspects":[119],"fairness.":[122],"Our":[123],"aim":[124],"provide":[127,146],"flexible":[129],"framework":[130],"empowers":[132],"practitioners":[133],"identify":[135],"significant":[136],"biases":[137],"metric":[140],"wish":[142],"study.":[144],"formal":[148],"mechanism":[150],"well":[152],"extensive":[154],"experiments":[155],"show":[157],"this":[159],"method":[160],"works":[161],"practice.":[163]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3081286828","counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":5},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":3}],"updated_date":"2024-11-23T16:02:36.246595","created_date":"2020-09-01"}