{"id":"https://openalex.org/W3041011740","doi":"https://doi.org/10.1016/j.ijhcs.2022.102772","title":"Modeling and mitigating human annotation errors to design efficient stream processing systems with human-in-the-loop machine learning","display_name":"Modeling and mitigating human annotation errors to design efficient stream processing systems with human-in-the-loop machine learning","publication_year":2022,"publication_date":"2022-01-10","ids":{"openalex":"https://openalex.org/W3041011740","doi":"https://doi.org/10.1016/j.ijhcs.2022.102772","mag":"3041011740"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.ijhcs.2022.102772","pdf_url":"https://www.sciencedirect.com/science/article/am/pii/S1071581922000015","source":{"id":"https://openalex.org/S4210190811","display_name":"International Journal of Human-Computer Studies","issn_l":"1071-5819","issn":["1071-5819","1095-9300"],"is_oa":false,"is_in_doaj":false,"is_indexed_in_scopus":true,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"acceptedVersion","is_accepted":true,"is_published":false},"type":"article","type_crossref":"journal-article","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.sciencedirect.com/science/article/am/pii/S1071581922000015","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5020350278","display_name":"Rahul Pandey","orcid":"https://orcid.org/0000-0003-2748-0502"},"institutions":[{"id":"https://openalex.org/I162714631","display_name":"George Mason University","ror":"https://ror.org/02jqj7156","country_code":"US","type":"funder","lineage":["https://openalex.org/I162714631"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Rahul Pandey","raw_affiliation_strings":["George Mason University, 4400 University Dr, Fairfax, VA, USA"],"affiliations":[{"raw_affiliation_string":"George Mason University, 4400 University Dr, Fairfax, VA, USA","institution_ids":["https://openalex.org/I162714631"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012073624","display_name":"Hemant Purohit","orcid":"https://orcid.org/0000-0002-4573-8450"},"institutions":[{"id":"https://openalex.org/I162714631","display_name":"George Mason University","ror":"https://ror.org/02jqj7156","country_code":"US","type":"funder","lineage":["https://openalex.org/I162714631"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hemant Purohit","raw_affiliation_strings":["George Mason University, 4400 University Dr, Fairfax, VA, USA"],"affiliations":[{"raw_affiliation_string":"George Mason University, 4400 University Dr, Fairfax, VA, USA","institution_ids":["https://openalex.org/I162714631"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084282503","display_name":"Carlos Castillo","orcid":"https://orcid.org/0000-0003-4544-0416"},"institutions":[{"id":"https://openalex.org/I170486558","display_name":"Pompeu Fabra University","ror":"https://ror.org/04n0g0b29","country_code":"ES","type":"funder","lineage":["https://openalex.org/I170486558"]},{"id":"https://openalex.org/I11932220","display_name":"Instituci\u00f3 Catalana de Recerca i Estudis Avan\u00e7ats","ror":"https://ror.org/0371hy230","country_code":"ES","type":"nonprofit","lineage":["https://openalex.org/I11932220"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Carlos Castillo","raw_affiliation_strings":["ICREA, Pg. Llu\u00eds Companys 23, Barcelona, Spain","Universitat Pompeu Fabra, Plaza de la Merc, Barcelona, 10-12, Spain"],"affiliations":[{"raw_affiliation_string":"Universitat Pompeu Fabra, Plaza de la Merc, Barcelona, 10-12, Spain","institution_ids":["https://openalex.org/I170486558"]},{"raw_affiliation_string":"ICREA, Pg. Llu\u00eds Companys 23, Barcelona, Spain","institution_ids":["https://openalex.org/I11932220"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5108053995","display_name":"Valerie L. Shalin","orcid":"https://orcid.org/0000-0001-8135-2793"},"institutions":[{"id":"https://openalex.org/I19648265","display_name":"Wright State University","ror":"https://ror.org/04qk6pt94","country_code":"US","type":"funder","lineage":["https://openalex.org/I19648265"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Valerie L. Shalin","raw_affiliation_strings":["Wright State University, 3640 Colonel Glenn Hwy, Dayton, OH, USA"],"affiliations":[{"raw_affiliation_string":"Wright State University, 3640 Colonel Glenn Hwy, Dayton, OH, USA","institution_ids":["https://openalex.org/I19648265"]}]}],"institution_assertions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5020350278"],"corresponding_institution_ids":["https://openalex.org/I162714631"],"apc_list":{"value":3640,"currency":"USD","value_usd":3640},"apc_paid":null,"fwci":2.827,"has_fulltext":true,"fulltext_origin":"pdf","cited_by_count":21,"citation_normalized_percentile":{"value":0.814918,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":"160","issue":null,"first_page":"102772","last_page":"102772"},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9999,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9999,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.991,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9906,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.71496844},{"id":"https://openalex.org/keywords/human-in-the-loop","display_name":"Human-in-the-loop","score":0.66993254}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.83769375},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.71496844},{"id":"https://openalex.org/C2780626000","wikidata":"https://www.wikidata.org/wiki/Q5936775","display_name":"Human-in-the-loop","level":2,"score":0.66993254},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.6583109},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6113305},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6008292},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.5429068},{"id":"https://openalex.org/C68387754","wikidata":"https://www.wikidata.org/wiki/Q7271585","display_name":"Schedule","level":2,"score":0.43631196},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4304861},{"id":"https://openalex.org/C169806903","wikidata":"https://www.wikidata.org/wiki/Q5937752","display_name":"Human error","level":2,"score":0.42168438},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.11140019},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.ijhcs.2022.102772","pdf_url":"https://www.sciencedirect.com/science/article/am/pii/S1071581922000015","source":{"id":"https://openalex.org/S4210190811","display_name":"International Journal of Human-Computer Studies","issn_l":"1071-5819","issn":["1071-5819","1095-9300"],"is_oa":false,"is_in_doaj":false,"is_indexed_in_scopus":true,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"acceptedVersion","is_accepted":true,"is_published":false},{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2007.03177","pdf_url":"https://arxiv.org/pdf/2007.03177","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":false,"landing_page_url":"https://api.datacite.org/dois/10.48550/arxiv.2007.03177","pdf_url":null,"source":{"id":"https://openalex.org/S4393179698","display_name":"DataCite API","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I4210145204","host_organization_name":"DataCite","host_organization_lineage":["https://openalex.org/I4210145204"],"host_organization_lineage_names":["DataCite"],"type":"metadata"},"license":null,"license_id":null,"version":null}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.ijhcs.2022.102772","pdf_url":"https://www.sciencedirect.com/science/article/am/pii/S1071581922000015","source":{"id":"https://openalex.org/S4210190811","display_name":"International Journal of Human-Computer Studies","issn_l":"1071-5819","issn":["1071-5819","1095-9300"],"is_oa":false,"is_in_doaj":false,"is_indexed_in_scopus":true,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"acceptedVersion","is_accepted":true,"is_published":false},"sustainable_development_goals":[{"score":0.72,"id":"https://metadata.un.org/sdg/13","display_name":"Climate action"}],"grants":[{"funder":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation","award_id":"LCF/PR/PR16/51110009"},{"funder":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation","award_id":"1815459"},{"funder":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation","award_id":"IIS-1657379"},{"funder":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation","award_id":"100010434"},{"funder":"https://openalex.org/F4320312058","funder_display_name":"\u201cla Caixa\u201d Foundation","award_id":null},{"funder":"https://openalex.org/F4320320300","funder_display_name":"European Commission","award_id":null},{"funder":"https://openalex.org/F4320330256","funder_display_name":"Fundaci\u00f3n Bancaria Caixa d'Estalvis i Pensions de Barcelona","award_id":null},{"funder":"https://openalex.org/F4320334701","funder_display_name":"Joint Research Centre","award_id":null}],"datasets":[],"versions":["https://openalex.org/W3041011740"],"referenced_works_count":74,"referenced_works":["https://openalex.org/W118683668","https://openalex.org/W1486811565","https://openalex.org/W1505086392","https://openalex.org/W1540652867","https://openalex.org/W1543599171","https://openalex.org/W1604115992","https://openalex.org/W1608614953","https://openalex.org/W1675227191","https://openalex.org/W1747560487","https://openalex.org/W1889619166","https://openalex.org/W1907380269","https://openalex.org/W1965191725","https://openalex.org/W1982039810","https://openalex.org/W1985095406","https://openalex.org/W1993430291","https://openalex.org/W1998654733","https://openalex.org/W2008615354","https://openalex.org/W2022775778","https://openalex.org/W2030007850","https://openalex.org/W2049059074","https://openalex.org/W2057763140","https://openalex.org/W2088974767","https://openalex.org/W2099302642","https://openalex.org/W2099419573","https://openalex.org/W2104969481","https://openalex.org/W2108525416","https://openalex.org/W2119885577","https://openalex.org/W2121038136","https://openalex.org/W2121721477","https://openalex.org/W2132277248","https://openalex.org/W2133088989","https://openalex.org/W2133469585","https://openalex.org/W2134385409","https://openalex.org/W2146436037","https://openalex.org/W2149868499","https://openalex.org/W2151098288","https://openalex.org/W2154819126","https://openalex.org/W2161829306","https://openalex.org/W2166781513","https://openalex.org/W2168656979","https://openalex.org/W2170178584","https://openalex.org/W2250539671","https://openalex.org/W2262972580","https://openalex.org/W2493475353","https://openalex.org/W2510788490","https://openalex.org/W2580371235","https://openalex.org/W2769009638","https://openalex.org/W2789731619","https://openalex.org/W2798683079","https://openalex.org/W2810030148","https://openalex.org/W2890750208","https://openalex.org/W2895715183","https://openalex.org/W2898274636","https://openalex.org/W2911280963","https://openalex.org/W2971035958","https://openalex.org/W3011865677","https://openalex.org/W3098037727","https://openalex.org/W3103400475","https://openalex.org/W3135154479","https://openalex.org/W4230399697","https://openalex.org/W4239868767","https://openalex.org/W4240152324","https://openalex.org/W4241994393","https://openalex.org/W4242878432","https://openalex.org/W4242958149","https://openalex.org/W4245652108","https://openalex.org/W4247431252","https://openalex.org/W4254502570","https://openalex.org/W4255466416","https://openalex.org/W4256250027","https://openalex.org/W4289725992","https://openalex.org/W4299377466","https://openalex.org/W4365799834","https://openalex.org/W629598914"],"related_works":["https://openalex.org/W4384486036","https://openalex.org/W4318823662","https://openalex.org/W4286908577","https://openalex.org/W3207526114","https://openalex.org/W3032998312","https://openalex.org/W2886410948","https://openalex.org/W2337920774","https://openalex.org/W2025875869","https://openalex.org/W1503094549","https://openalex.org/W135177976"],"abstract_inverted_index":null,"abstract_inverted_index_v3":null,"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3041011740","counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":9},{"year":2022,"cited_by_count":3}],"updated_date":"2025-03-02T10:52:06.799891","created_date":"2020-07-16"}