{"id":"https://openalex.org/W2933315718","doi":"https://doi.org/10.1145/3314419","title":"Fixing Mislabeling by Human Annotators Leveraging Conflict Resolution and Prior Knowledge","display_name":"Fixing Mislabeling by Human Annotators Leveraging Conflict Resolution and Prior Knowledge","publication_year":2019,"publication_date":"2019-03-29","ids":{"openalex":"https://openalex.org/W2933315718","doi":"https://doi.org/10.1145/3314419","mag":"2933315718"},"language":"en","primary_location":{"is_oa":false,"landing_page_url":"https://doi.org/10.1145/3314419","pdf_url":null,"source":{"id":"https://openalex.org/S4210219751","display_name":"Proceedings of the ACM on Interactive Mobile Wearable and Ubiquitous Technologies","issn_l":"2474-9567","issn":["2474-9567"],"is_oa":false,"is_in_doaj":false,"is_indexed_in_scopus":true,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false},"type":"article","type_crossref":"journal-article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5106225936","display_name":"Mattia Zeni","orcid":"https://orcid.org/0000-0002-6767-9519"},"institutions":[{"id":"https://openalex.org/I193223587","display_name":"University of Trento","ror":"https://ror.org/05trd4x28","country_code":"IT","type":"funder","lineage":["https://openalex.org/I193223587"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Mattia Zeni","raw_affiliation_strings":["Department of Information Engineering and Computer Science, University of Trento, Trento, TN, IT"],"affiliations":[{"raw_affiliation_string":"Department of Information Engineering and Computer Science, University of Trento, Trento, TN, IT","institution_ids":["https://openalex.org/I193223587"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060121586","display_name":"Wanyi Zhang","orcid":"https://orcid.org/0000-0002-3952-0826"},"institutions":[{"id":"https://openalex.org/I193223587","display_name":"University of Trento","ror":"https://ror.org/05trd4x28","country_code":"IT","type":"funder","lineage":["https://openalex.org/I193223587"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Wanyi Zhang","raw_affiliation_strings":["Department of Information Engineering and Computer Science, University of Trento, Trento, TN, IT"],"affiliations":[{"raw_affiliation_string":"Department of Information Engineering and Computer Science, University of Trento, Trento, TN, IT","institution_ids":["https://openalex.org/I193223587"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011492998","display_name":"Enrico Bignotti","orcid":null},"institutions":[{"id":"https://openalex.org/I193223587","display_name":"University of Trento","ror":"https://ror.org/05trd4x28","country_code":"IT","type":"funder","lineage":["https://openalex.org/I193223587"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Enrico Bignotti","raw_affiliation_strings":["Department of Information Engineering and Computer Science, University of Trento, Trento, TN, IT"],"affiliations":[{"raw_affiliation_string":"Department of Information Engineering and Computer Science, University of Trento, Trento, TN, IT","institution_ids":["https://openalex.org/I193223587"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066187890","display_name":"Andrea Passerini","orcid":"https://orcid.org/0000-0002-2765-5395"},"institutions":[{"id":"https://openalex.org/I193223587","display_name":"University of Trento","ror":"https://ror.org/05trd4x28","country_code":"IT","type":"funder","lineage":["https://openalex.org/I193223587"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Andrea Passerini","raw_affiliation_strings":["Department of Information Engineering and Computer Science, University of Trento, Trento, TN, IT"],"affiliations":[{"raw_affiliation_string":"Department of Information Engineering and Computer Science, University of Trento, Trento, TN, IT","institution_ids":["https://openalex.org/I193223587"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001227032","display_name":"Fausto Giunchiglia","orcid":"https://orcid.org/0000-0002-5903-6150"},"institutions":[{"id":"https://openalex.org/I194450716","display_name":"Jilin University","ror":"https://ror.org/00js3aw79","country_code":"CN","type":"funder","lineage":["https://openalex.org/I194450716"]},{"id":"https://openalex.org/I193223587","display_name":"University of Trento","ror":"https://ror.org/05trd4x28","country_code":"IT","type":"funder","lineage":["https://openalex.org/I193223587"]}],"countries":["CN","IT"],"is_corresponding":false,"raw_author_name":"Fausto Giunchiglia","raw_affiliation_strings":["Department of Information Engineering and Computer Science, University of Trento, Trento, TN, IT and College of Computer Science and Technology, Jilin University, Changchun, China, Changchun, JL, CN"],"affiliations":[{"raw_affiliation_string":"Department of Information Engineering and Computer Science, University of Trento, Trento, TN, IT and College of Computer Science and Technology, Jilin University, Changchun, China, Changchun, JL, CN","institution_ids":["https://openalex.org/I194450716","https://openalex.org/I193223587"]}]}],"institution_assertions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":6.118,"has_fulltext":true,"fulltext_origin":"ngrams","cited_by_count":36,"citation_normalized_percentile":{"value":0.941581,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":"3","issue":"1","first_page":"1","last_page":"23"},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9987,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9881,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/coreference","display_name":"Coreference","score":0.66257817}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.83175695},{"id":"https://openalex.org/C28076734","wikidata":"https://www.wikidata.org/wiki/Q63087","display_name":"Coreference","level":3,"score":0.66257817},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.6578025},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5653395},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5245276},{"id":"https://openalex.org/C2780224610","wikidata":"https://www.wikidata.org/wiki/Q1530061","display_name":"Credibility","level":2,"score":0.46298376},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.45445326},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.4431789},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.43935618},{"id":"https://openalex.org/C138268822","wikidata":"https://www.wikidata.org/wiki/Q1051925","display_name":"Resolution (logic)","level":2,"score":0.425595},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4239494},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.10720286},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"is_oa":false,"landing_page_url":"https://doi.org/10.1145/3314419","pdf_url":null,"source":{"id":"https://openalex.org/S4210219751","display_name":"Proceedings of the ACM on Interactive Mobile Wearable and Ubiquitous Technologies","issn_l":"2474-9567","issn":["2474-9567"],"is_oa":false,"is_in_doaj":false,"is_indexed_in_scopus":true,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false}],"best_oa_location":null,"sustainable_development_goals":[],"grants":[],"datasets":[],"versions":[],"referenced_works_count":52,"referenced_works":["https://openalex.org/W1176919","https://openalex.org/W1495236921","https://openalex.org/W1566608954","https://openalex.org/W1570435088","https://openalex.org/W1597195591","https://openalex.org/W1605688901","https://openalex.org/W1670346950","https://openalex.org/W1673310716","https://openalex.org/W1873057782","https://openalex.org/W1977970897","https://openalex.org/W1980191529","https://openalex.org/W1994550352","https://openalex.org/W1994788295","https://openalex.org/W2018076238","https://openalex.org/W2022466128","https://openalex.org/W2031218379","https://openalex.org/W2055665526","https://openalex.org/W2088499414","https://openalex.org/W2107397716","https://openalex.org/W2110644663","https://openalex.org/W2137404138","https://openalex.org/W2141854027","https://openalex.org/W2143554828","https://openalex.org/W2152222281","https://openalex.org/W2156567116","https://openalex.org/W2158797600","https://openalex.org/W2160585474","https://openalex.org/W2167917621","https://openalex.org/W2201744460","https://openalex.org/W2283611363","https://openalex.org/W2293768274","https://openalex.org/W2391561458","https://openalex.org/W2407344505","https://openalex.org/W2562836854","https://openalex.org/W2593133890","https://openalex.org/W2611266700","https://openalex.org/W2740960106","https://openalex.org/W2753518399","https://openalex.org/W2781484749","https://openalex.org/W2897422894","https://openalex.org/W2897426351","https://openalex.org/W2903158431","https://openalex.org/W2911964244","https://openalex.org/W2951304829","https://openalex.org/W2959138495","https://openalex.org/W2963044698","https://openalex.org/W3003902971","https://openalex.org/W3102825217","https://openalex.org/W3110899665","https://openalex.org/W4292022450","https://openalex.org/W4302126364","https://openalex.org/W572548289"],"related_works":["https://openalex.org/W4320505349","https://openalex.org/W3212677975","https://openalex.org/W3174839683","https://openalex.org/W3029917925","https://openalex.org/W2999272952","https://openalex.org/W2969230458","https://openalex.org/W2379250586","https://openalex.org/W2205554523","https://openalex.org/W2148467945","https://openalex.org/W1858063944"],"abstract_inverted_index":{"According":[0],"to":[1,66,79,106,117,127,153],"the":[2,5,20,31,44,99,108,114,155,164,169,173,180,188],"\"human":[3],"in":[4,19,30,70],"loop\"":[6],"paradigm,":[7],"machine":[8,100],"learning":[9,196],"algorithms":[10],"can":[11,57],"improve":[12],"when":[13,130],"leveraging":[14],"on":[15],"human":[16],"intelligence,":[17],"usually":[18],"form":[21],"of":[22,33,61,73,110,125,141,166,172,179,182,192],"labels":[23],"or":[24,40],"annotation":[25,167],"from":[26],"domain":[27],"experts.":[28],"However,":[29],"case":[32],"research":[34],"areas":[35],"such":[36],"as":[37],"ubiquitous":[38],"computing":[39],"lifelong":[41],"learning,":[42],"where":[43,75,98],"annotator":[45],"is":[46,51,92],"not":[47],"an":[48,96],"expert":[49],"and":[50,113,147,162,187],"continuously":[52],"asked":[53,78],"for":[54,123],"feedback,":[55],"humans":[56],"provide":[58,80],"significant":[59],"fractions":[60],"incorrect":[62],"labels.":[63],"We":[64],"propose":[65],"address":[67],"this":[68],"issue":[69],"a":[71,86,119,131,139,149,183,193],"series":[72,140],"experiments":[74],"students":[76],"are":[77,159],"information":[81],"about":[82],"their":[83],"behavior":[84],"via":[85],"dedicated":[87],"mobile":[88],"application.":[89],"Their":[90],"trustworthiness":[91],"tested":[93],"by":[94],"employing":[95],"architecture":[97],"uses":[101],"all":[102],"its":[103,111],"available":[104],"knowledge":[105],"check":[107],"correctness":[109],"own":[112],"user":[115],"labeling":[116],"build":[118],"uniform":[120,184],"confidence":[121,146],"measure":[122],"both":[124],"them":[126],"be":[128],"used":[129],"contradiction":[132],"arises.":[133],"The":[134,157],"overarching":[135],"system":[136],"runs":[137],"through":[138],"modes":[142],"with":[143],"progressively":[144],"higher":[145],"features":[148],"conflict":[150],"resolution":[151],"component":[152],"settle":[154],"inconsistencies.":[156],"results":[158],"very":[160],"promising":[161],"show":[163],"pervasiveness":[165],"mistakes,":[168],"extreme":[170],"diversity":[171],"users'":[174],"behaviors":[175],"which":[176],"provides":[177],"evidence":[178],"impracticality":[181],"fits-it-all":[185],"solution,":[186],"substantially":[189],"improved":[190],"performance":[191],"skeptical":[194],"supervised":[195],"strategy.":[197]},"abstract_inverted_index_v3":null,"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2933315718","counts_by_year":[{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":11},{"year":2020,"cited_by_count":10},{"year":2019,"cited_by_count":1}],"updated_date":"2025-03-20T17:02:44.672294","created_date":"2019-04-11"}