{"id":"https://openalex.org/W3156470785","doi":"https://doi.org/10.18653/v1/2022.acl-long.556","title":"Fantastically Ordered Prompts and Where to Find Them: Overcoming Few-Shot Prompt Order Sensitivity","display_name":"Fantastically Ordered Prompts and Where to Find Them: Overcoming Few-Shot Prompt Order Sensitivity","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W3156470785","doi":"https://doi.org/10.18653/v1/2022.acl-long.556","mag":"3156470785"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2022.acl-long.556","pdf_url":"https://aclanthology.org/2022.acl-long.556.pdf","source":{"id":"https://openalex.org/S4363608652","display_name":"Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"type":"article","type_crossref":"proceedings-article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2022.acl-long.556.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026907508","display_name":"Yao Lu","orcid":"https://orcid.org/0000-0002-9873-8753"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Yao Lu","raw_affiliation_strings":["University College London"],"affiliations":[{"raw_affiliation_string":"University College London","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018864006","display_name":"Max Bartolo","orcid":"https://orcid.org/0009-0007-3301-7895"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Max Bartolo","raw_affiliation_strings":["University College London"],"affiliations":[{"raw_affiliation_string":"University College London","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015493447","display_name":"Alastair Moore","orcid":"https://orcid.org/0000-0002-3726-6473"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alastair Moore","raw_affiliation_strings":["Mishcon de Reya LLP"],"affiliations":[{"raw_affiliation_string":"Mishcon de Reya LLP","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101404695","display_name":"Sebastian Riedel","orcid":"https://orcid.org/0000-0002-3655-2486"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Sebastian Riedel","raw_affiliation_strings":["University College London"],"affiliations":[{"raw_affiliation_string":"University College London","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5049427071","display_name":"Pontus Stenetorp","orcid":null},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Pontus Stenetorp","raw_affiliation_strings":["University College London"],"affiliations":[{"raw_affiliation_string":"University College London","institution_ids":["https://openalex.org/I45129253"]}]}],"institution_assertions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":35.016,"has_fulltext":true,"fulltext_origin":"pdf","cited_by_count":234,"citation_normalized_percentile":{"value":0.999875,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9999,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9999,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.999,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9733,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.74321514},{"id":"https://openalex.org/C21308566","wikidata":"https://www.wikidata.org/wiki/Q7169365","display_name":"Permutation (music)","level":2,"score":0.6711423},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6002213},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.585827},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.5845864},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.54326844},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5292458},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.4637606},{"id":"https://openalex.org/C21200559","wikidata":"https://www.wikidata.org/wiki/Q7451068","display_name":"Sensitivity (control systems)","level":2,"score":0.41205344},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4013945},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.38887796},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.07389784},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C24326235","wikidata":"https://www.wikidata.org/wiki/Q126095","display_name":"Electronic engineering","level":1,"score":0.0},{"id":"https://openalex.org/C24890656","wikidata":"https://www.wikidata.org/wiki/Q82811","display_name":"Acoustics","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2022.acl-long.556","pdf_url":"https://aclanthology.org/2022.acl-long.556.pdf","source":{"id":"https://openalex.org/S4363608652","display_name":"Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},{"is_oa":true,"landing_page_url":"https://discovery.ucl.ac.uk/id/eprint/10154329/","pdf_url":"https://discovery.ucl.ac.uk/10154329/1/2022.acl-long.556.pdf","source":{"id":"https://openalex.org/S4306400024","display_name":"UCL Discovery (University College London)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I45129253","host_organization_name":"University College London","host_organization_lineage":["https://openalex.org/I45129253"],"host_organization_lineage_names":["University College London"],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":true,"landing_page_url":"https://discovery.ucl.ac.uk/id/eprint/10154329/1/2022.acl-long.556.pdf","pdf_url":"https://discovery.ucl.ac.uk/id/eprint/10154329/1/2022.acl-long.556.pdf","source":{"id":"https://openalex.org/S4306400024","display_name":"UCL Discovery (University College London)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I45129253","host_organization_name":"University College London","host_organization_lineage":["https://openalex.org/I45129253"],"host_organization_lineage_names":["University College London"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2104.08786","pdf_url":"http://arxiv.org/pdf/2104.08786","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2022.acl-long.556","pdf_url":"https://aclanthology.org/2022.acl-long.556.pdf","source":{"id":"https://openalex.org/S4363608652","display_name":"Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality education","score":0.73}],"grants":[],"datasets":[],"versions":[],"referenced_works_count":36,"referenced_works":["https://openalex.org/W2014902591","https://openalex.org/W2028175314","https://openalex.org/W2114524997","https://openalex.org/W2130158090","https://openalex.org/W2131494463","https://openalex.org/W2160660844","https://openalex.org/W2163455955","https://openalex.org/W2170240176","https://openalex.org/W2251939518","https://openalex.org/W2612675303","https://openalex.org/W2896457183","https://openalex.org/W2949380545","https://openalex.org/W2952186591","https://openalex.org/W2962739339","https://openalex.org/W2963012544","https://openalex.org/W2963341956","https://openalex.org/W2963768805","https://openalex.org/W2964091467","https://openalex.org/W2965373594","https://openalex.org/W2970161131","https://openalex.org/W2970476646","https://openalex.org/W2970597249","https://openalex.org/W3021533447","https://openalex.org/W3026404337","https://openalex.org/W3033176962","https://openalex.org/W3044438666","https://openalex.org/W3082274269","https://openalex.org/W3098267758","https://openalex.org/W3132736064","https://openalex.org/W3164972323","https://openalex.org/W3172642864","https://openalex.org/W3172943453","https://openalex.org/W3173777717","https://openalex.org/W4288089799","https://openalex.org/W4292779060","https://openalex.org/W4308900200"],"related_works":["https://openalex.org/W4306674287","https://openalex.org/W4306321456","https://openalex.org/W4288804510","https://openalex.org/W4286629047","https://openalex.org/W4224009465","https://openalex.org/W2961085424","https://openalex.org/W2359001871","https://openalex.org/W1989705153","https://openalex.org/W1772447446","https://openalex.org/W1652742547"],"abstract_inverted_index":{"When":[0],"primed":[1],"with":[2],"only":[3],"a":[4,85,92,108,165],"handful":[5],"of":[6,88,137,151],"training":[7],"samples,":[8,89],"very":[9],"large,":[10,26],"pretrained":[11,27],"language":[12,28,138],"models":[13,139,171],"such":[14],"as":[15,125],"GPT-3":[16],"have":[17],"shown":[18],"competitive":[19],"results":[20],"when":[21],"compared":[22],"to":[23,84,102,111,140],"fully-supervised,":[24],"fine-tuned,":[25],"models.":[29],"We":[30,60],"demonstrate":[31],"that":[32,91],"the":[33,37,43,76,121,134,152],"order":[34],"in":[35,64],"which":[36,113],"samples":[38],"are":[39,55,115],"provided":[40],"can":[41],"make":[42],"difference":[44],"between":[45],"near":[46],"state-of-the-art":[47],"and":[48,57,90,146],"random":[49],"guess":[50],"performance:":[51],"essentially":[52],"some":[53,58],"permutations":[54,114,154],"\u201cfantastic\u201d":[56],"not.":[59],"analyse":[61],"this":[62,117,156],"phenomenon":[63],"detail,":[65],"establishing":[66],"that:":[67],"it":[68,80,126],"is":[69,81,99],"present":[70],"across":[71,172],"model":[72,98],"sizes":[73],"(even":[74],"for":[75,96,169],"largest":[77],"current":[78],"models),":[79],"not":[82,100],"related":[83],"specific":[86],"subset":[87],"given":[93],"good":[94],"permutation":[95],"one":[97,105],"transferable":[101],"another.":[103],"While":[104],"could":[106],"use":[107,133],"development":[109,144],"set":[110,145],"determine":[112],"performant,":[116],"would":[118],"deviate":[119],"from":[120],"true":[122],"few-shot":[123],"setting":[124],"requires":[127],"additional":[128],"annotated":[129],"data.":[130],"Instead,":[131],"we":[132,158],"generative":[135],"nature":[136],"construct":[141],"an":[142],"artificial":[143],"based":[147],"on":[148,155],"entropy":[149],"statistics":[150],"candidate":[153],"set,":[157],"identify":[159],"performant":[160],"prompts.":[161],"Our":[162],"method":[163],"yields":[164],"13%":[166],"relative":[167],"improvement":[168],"GPT-family":[170],"eleven":[173],"different":[174],"established":[175],"text":[176],"classification":[177],"tasks.":[178]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3156470785","counts_by_year":[{"year":2024,"cited_by_count":81},{"year":2023,"cited_by_count":120},{"year":2022,"cited_by_count":21},{"year":2021,"cited_by_count":12}],"updated_date":"2025-01-09T02:20:26.654071","created_date":"2021-04-26"}