{"id":"https://openalex.org/W4378942694","doi":"https://doi.org/10.48550/arxiv.2305.18654","title":"Faith and Fate: Limits of Transformers on Compositionality","display_name":"Faith and Fate: Limits of Transformers on Compositionality","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4378942694","doi":"https://doi.org/10.48550/arxiv.2305.18654"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2305.18654","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false},"type":"preprint","type_crossref":"posted-content","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/abs/2305.18654","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5049618494","display_name":"Nouha Dziri","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dziri, Nouha","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102930940","display_name":"Ximing Lu","orcid":"https://orcid.org/0000-0001-6671-4573"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Ximing","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039472047","display_name":"Melanie Sclar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sclar, Melanie","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034858125","display_name":"Xiang Lorraine Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Xiang Lorraine","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108840339","display_name":"Liwei Jian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jian, Liwei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013041264","display_name":"Bill Yuchen Lin","orcid":"https://orcid.org/0000-0002-1149-0186"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Bill Yuchen","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112759123","display_name":"Peter West","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"West, Peter","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044250030","display_name":"Chandra Bhagavatula","orcid":"https://orcid.org/0000-0001-6264-0378"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bhagavatula, Chandra","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024879161","display_name":"Ronan Le Bras","orcid":"https://orcid.org/0000-0003-2439-6938"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bras, Ronan Le","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080544237","display_name":"Jena D. Hwang","orcid":"https://orcid.org/0000-0003-3801-294X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hwang, Jena D.","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102487773","display_name":"Soumya Sanyal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sanyal, Soumya","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019030424","display_name":"Sean Welleck","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Welleck, Sean","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009408707","display_name":"Xiang Ren","orcid":"https://orcid.org/0000-0001-8655-663X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ren, Xiang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012068726","display_name":"Allyson Ettinger","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ettinger, Allyson","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010613767","display_name":"Za\u00efd Harchaoui","orcid":"https://orcid.org/0000-0003-1186-1343"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Harchaoui, Zaid","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5102992157","display_name":"Yejin Choi","orcid":"https://orcid.org/0000-0003-3032-5378"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Choi, Yejin","raw_affiliation_strings":[],"affiliations":[]}],"institution_assertions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":38,"citation_normalized_percentile":{"value":0.999953,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9895,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9882,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6016708},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.59960765},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.43806413},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.389625},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33671415},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.25097752},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2305.18654","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.18654","pdf_url":"http://arxiv.org/pdf/2305.18654","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":false,"landing_page_url":"https://api.datacite.org/dois/10.48550/arxiv.2305.18654","pdf_url":null,"source":{"id":"https://openalex.org/S4393179698","display_name":"DataCite API","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I4210145204","host_organization_name":"DataCite","host_organization_lineage":["https://openalex.org/I4210145204"],"host_organization_lineage_names":["DataCite"],"type":"metadata"},"license":null,"license_id":null,"version":null}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2305.18654","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[{"score":0.76,"id":"https://metadata.un.org/sdg/4","display_name":"Quality education"}],"grants":[],"datasets":[],"versions":[],"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391913857","https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2477549100","https://openalex.org/W2390279801","https://openalex.org/W2382290278","https://openalex.org/W2376932109","https://openalex.org/W2358668433","https://openalex.org/W2350741829","https://openalex.org/W2001405890"],"abstract_inverted_index":{"Transformer":[0],"large":[1],"language":[2],"models":[3,21,57],"(LLMs)":[4],"have":[5],"sparked":[6],"admiration":[7],"for":[8],"their":[9],"exceptional":[10],"performance":[11,158],"on":[12,25,148],"tasks":[13,62,76,94,122],"that":[14,117,153],"demand":[15],"intricate":[16],"multi-step":[17,125,150],"reasoning.":[18],"Yet,":[19],"these":[20,34,56,85],"simultaneously":[22],"show":[23],"failures":[24],"surprisingly":[26],"trivial":[27],"problems.":[28],"This":[29],"begs":[30],"the":[31,53,101],"question:":[32],"Are":[33],"errors":[35],"incidental,":[36],"or":[37],"do":[38],"they":[39],"signal":[40],"more":[41],"substantial":[42],"limitations?":[43],"In":[44],"an":[45],"attempt":[46],"to":[47,98],"demystify":[48],"transformer":[49,118],"LLMs,":[50],"we":[51,144],"investigate":[52],"limits":[54],"of":[55,103],"across":[58],"three":[59],"representative":[60],"compositional":[61,93,121,126],"--":[63],"multi-digit":[64],"multiplication,":[65],"logic":[66],"grid":[67],"puzzles,":[68],"and":[69,83,105],"a":[70,88],"classic":[71],"dynamic":[72],"programming":[73],"problem.":[74],"These":[75],"require":[77],"breaking":[78],"problems":[79,152],"down":[80,107],"into":[81,87,110,128],"sub-steps":[82],"synthesizing":[84],"steps":[86,109],"precise":[89],"answer.":[90],"We":[91],"formulate":[92],"as":[95],"computation":[96],"graphs":[97],"systematically":[99],"quantify":[100],"level":[102],"complexity,":[104],"break":[106],"reasoning":[108,127,151],"intermediate":[111],"sub-procedures.":[112],"Our":[113],"empirical":[114,142],"findings":[115],"suggest":[116],"LLMs":[119],"solve":[120],"by":[123],"reducing":[124],"linearized":[129],"subgraph":[130],"matching,":[131],"without":[132],"necessarily":[133],"developing":[134],"systematic":[135],"problem-solving":[136],"skills.":[137],"To":[138],"round":[139],"off":[140],"our":[141],"study,":[143],"provide":[145],"theoretical":[146],"arguments":[147],"abstract":[149],"highlight":[154],"how":[155],"autoregressive":[156],"generations'":[157],"can":[159],"rapidly":[160],"decay":[161],"with\\,increased\\,task\\,complexity.":[162]},"abstract_inverted_index_v3":null,"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4378942694","counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":18},{"year":2023,"cited_by_count":18}],"updated_date":"2025-03-14T08:59:15.113971","created_date":"2023-06-01"}