{"id":"https://openalex.org/W4385014863","doi":"https://doi.org/10.48550/arxiv.2307.10168","title":"LLMs as Workers in Human-Computational Algorithms? Replicating Crowdsourcing Pipelines with LLMs","display_name":"LLMs as Workers in Human-Computational Algorithms? Replicating Crowdsourcing Pipelines with LLMs","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4385014863","doi":"https://doi.org/10.48550/arxiv.2307.10168"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2307.10168","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false},"type":"preprint","type_crossref":"posted-content","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/abs/2307.10168","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5004225142","display_name":"Tongshuang Wu","orcid":"https://orcid.org/0000-0003-1630-0588"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Tongshuang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051842323","display_name":"Haiyi Zhu","orcid":"https://orcid.org/0000-0001-7271-9100"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Haiyi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092513653","display_name":"Maya Albayrak","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Albayrak, Maya","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114068490","display_name":"Alexis Axon","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Axon, Alexis","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013007353","display_name":"Amanda Bertsch","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bertsch, Amanda","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111192886","display_name":"Wenxing Deng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Deng, Wenxing","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101317643","display_name":"Ziqi Ding","orcid":"https://orcid.org/0009-0002-9971-0899"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ding, Ziqi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092513654","display_name":"Bill Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Bill","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108855136","display_name":"Sireesh Gururaja","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gururaja, Sireesh","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112964294","display_name":"Tzu-Sheng Kuo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kuo, Tzu-Sheng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006849693","display_name":"Jenny T. Liang","orcid":"https://orcid.org/0000-0001-6722-9959"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liang, Jenny T.","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061907283","display_name":"Ryan Wen Liu","orcid":"https://orcid.org/0000-0002-1591-5583"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Ryan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092513648","display_name":"Ihita Mandal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mandal, Ihita","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000331140","display_name":"Jeremiah Milbauer","orcid":"https://orcid.org/0000-0003-2809-534X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Milbauer, Jeremiah","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112733575","display_name":"Xiaolin Ni","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ni, Xiaolin","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092513649","display_name":"Namrata Padmanabhan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Padmanabhan, Namrata","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092513650","display_name":"Subhashini Ramkumar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ramkumar, Subhashini","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092513651","display_name":"Alexis Sudjianto","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sudjianto, Alexis","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112964295","display_name":"Jordan Taylor","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Taylor, Jordan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102912419","display_name":"Ying-Jui Tseng","orcid":"https://orcid.org/0009-0006-1801-6061"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tseng, Ying-Jui","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092513652","display_name":"Patricia Vaidos","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vaidos, Patricia","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009461496","display_name":"Zhijin Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Zhijin","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102536970","display_name":"Wei Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Wei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5101966381","display_name":"Chenyang Yang","orcid":"https://orcid.org/0000-0001-5016-7296"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Chenyang","raw_affiliation_strings":[],"affiliations":[]}],"institution_assertions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.997653,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":90},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9997,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9997,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12784","display_name":"Modular Robots and Swarm Intelligence","score":0.9928,"subfield":{"id":"https://openalex.org/subfields/2210","display_name":"Mechanical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9776,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.85256565}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.85256565},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3831122},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.34960645},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.098579675}],"mesh":[],"locations_count":3,"locations":[{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2307.10168","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.10168","pdf_url":"http://arxiv.org/pdf/2307.10168","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":false,"landing_page_url":"https://api.datacite.org/dois/10.48550/arxiv.2307.10168","pdf_url":null,"source":{"id":"https://openalex.org/S4393179698","display_name":"DataCite API","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210145204","host_organization_name":"DataCite","host_organization_lineage":["https://openalex.org/I4210145204"],"host_organization_lineage_names":["DataCite"],"type":"metadata"},"license":null,"license_id":null,"version":null}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2307.10168","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[],"grants":[],"datasets":[],"versions":[],"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W4384486036","https://openalex.org/W4286908577","https://openalex.org/W3032998312","https://openalex.org/W2886410948","https://openalex.org/W2748952813","https://openalex.org/W2337920774","https://openalex.org/W2025875869","https://openalex.org/W1503094549","https://openalex.org/W135177976"],"abstract_inverted_index":{"LLMs":[0,33,44,113,136],"have":[1],"shown":[2],"promise":[3],"in":[4,8,51,150],"replicating":[5,122],"human-like":[6],"behavior":[7],"crowdsourcing":[9,38,123],"tasks":[10,139,160],"that":[11,42,121],"were":[12],"previously":[13],"thought":[14],"to":[15,18,94,129,164],"be":[16],"exclusive":[17],"human":[19,89],"abilities.":[20],"However,":[21],"current":[22],"efforts":[23],"focus":[24],"mainly":[25],"on":[26,88,137,144],"simple":[27],"atomic":[28],"tasks.":[29],"We":[30,40,86],"explore":[31],"whether":[32],"can":[34,45,155],"replicate":[35],"more":[36],"complex":[37,151],"pipelines.":[39],"find":[41],"modern":[43],"simulate":[46],"some":[47],"of":[48,59,68,99,109,135,158],"crowdworkers'":[49],"abilities":[50],"these":[52,84],"\"human":[53],"computation":[54],"algorithms,\"":[55],"but":[56],"the":[57,71,78,97,107,132,159],"level":[58],"success":[60],"is":[61],"variable":[62],"and":[63,77,90,105,112,146],"influenced":[64],"by":[65],"requesters'":[66],"understanding":[67],"LLM":[69],"capabilities,":[70],"specific":[72],"skills":[73],"required":[74],"for":[75,82,103],"sub-tasks,":[76],"optimal":[79],"interaction":[80],"modality":[81],"performing":[83],"sub-tasks.":[85],"reflect":[87],"LLMs'":[91,148],"different":[92,138],"sensitivities":[93],"instructions,":[95],"stress":[96],"importance":[98],"enabling":[100],"human-facing":[101],"safeguards":[102],"LLMs,":[104],"discuss":[106],"potential":[108,149],"training":[110],"humans":[111],"with":[114],"complementary":[115],"skill":[116],"sets.":[117],"Crucially,":[118],"we":[119],"show":[120],"pipelines":[124],"offers":[125],"a":[126],"valuable":[127],"platform":[128],"investigate":[130],"(1)":[131],"relative":[133],"strengths":[134],"(by":[140],"cross-comparing":[141],"their":[142],"performances":[143],"sub-tasks)":[145],"(2)":[147],"tasks,":[152],"where":[153],"they":[154],"complete":[156],"part":[157],"while":[161],"leaving":[162],"others":[163],"humans.":[165]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4385014863","counts_by_year":[{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":1}],"updated_date":"2025-01-04T15:57:53.230107","created_date":"2023-07-22"}