{"id":"https://openalex.org/W4403964227","doi":"https://doi.org/10.48550/arxiv.2410.04840","title":"Strong Model Collapse","display_name":"Strong Model Collapse","publication_year":2024,"publication_date":"2024-10-07","ids":{"openalex":"https://openalex.org/W4403964227","doi":"https://doi.org/10.48550/arxiv.2410.04840"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.04840","pdf_url":"http://arxiv.org/pdf/2410.04840","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"type":"preprint","type_crossref":"posted-content","indexed_in":["arxiv"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://arxiv.org/pdf/2410.04840","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5040199022","display_name":"Elvis Dohmatob","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dohmatob, Elvis","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064491965","display_name":"Yunzhen Feng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Feng, Yunzhen","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114482375","display_name":"Arjun Subramonian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Subramonian, Arjun","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5051933322","display_name":"Julia Kempe","orcid":"https://orcid.org/0009-0006-1984-0060"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kempe, Julia","raw_affiliation_strings":[],"affiliations":[]}],"institution_assertions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":0,"max":78},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T11270","display_name":"Complex Systems and Time Series Analysis","score":0.0652,"subfield":{"id":"https://openalex.org/subfields/2002","display_name":"Economics and Econometrics"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11270","display_name":"Complex Systems and Time Series Analysis","score":0.0652,"subfield":{"id":"https://openalex.org/subfields/2002","display_name":"Economics and Econometrics"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11829","display_name":"Mathematical Biology Tumor Growth","score":0.0634,"subfield":{"id":"https://openalex.org/subfields/2611","display_name":"Modeling and Simulation"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12639","display_name":"Global Energy and Sustainability Research","score":0.0571,"subfield":{"id":"https://openalex.org/subfields/2105","display_name":"Renewable Energy, Sustainability and the Environment"},"field":{"id":"https://openalex.org/fields/21","display_name":"Energy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[],"concepts":[{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.37068757},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.34147573}],"mesh":[],"locations_count":1,"locations":[{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.04840","pdf_url":"http://arxiv.org/pdf/2410.04840","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.04840","pdf_url":"http://arxiv.org/pdf/2410.04840","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[],"grants":[],"datasets":[],"versions":[],"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4396701345","https://openalex.org/W4396696052","https://openalex.org/W4391913857","https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2376932109","https://openalex.org/W2358668433","https://openalex.org/W2001405890"],"abstract_inverted_index":{"Within":[0],"the":[1,7,25,32,45,53,65,142,158],"scaling":[2],"laws":[3],"paradigm,":[4],"which":[5],"underpins":[6],"training":[8,46,67,78,98],"of":[9,27,31,56,64,119],"large":[10,99,152],"neural":[11,112,180],"networks":[12,113,181],"like":[13],"ChatGPT":[14],"and":[15,23,76,125,178],"Llama,":[16],"we":[17,122],"consider":[18],"a":[19,28,36,108],"supervised":[20],"regression":[21],"setting":[22],"establish":[24],"existance":[26],"strong":[29],"form":[30],"model":[33,73,89,105,133],"collapse":[34],"phenomenon,":[35],"critical":[37],"performance":[38],"degradation":[39],"due":[40],"to":[41,72],"synthetic":[42,57],"data":[43,58],"in":[44,97],"corpus.":[47],"Our":[48,167],"results":[49],"show":[50,127],"that":[51,128],"even":[52],"smallest":[54],"fraction":[55],"(e.g.,":[59],"as":[60,62],"little":[61],"1\\%":[63],"total":[66],"dataset)":[68],"can":[69,131,146],"still":[70],"lead":[71],"collapse:":[74],"larger":[75,77,129,154],"sets":[79],"do":[80,162],"not":[81,163],"enhance":[82],"performance.":[83],"We":[84],"further":[85],"investigate":[86],"whether":[87],"increasing":[88],"size,":[90,121],"an":[91],"approach":[92],"aligned":[93],"with":[94],"current":[95],"trends":[96],"language":[100,176],"models,":[101],"exacerbates":[102],"or":[103],"mitigates":[104],"collapse.":[106,134],"In":[107],"simplified":[109],"regime":[110],"where":[111],"are":[114,170],"approximated":[115],"via":[116],"random":[117],"projections":[118],"tunable":[120],"both":[123],"theoretically":[124],"empirically":[126,171],"models":[130,155,177],"amplify":[132],"Interestingly,":[135],"our":[136],"theory":[137],"also":[138],"indicates":[139],"that,":[140],"beyond":[141],"interpolation":[143],"threshold":[144],"(which":[145],"be":[147],"extremely":[148],"high":[149],"for":[150,182],"very":[151],"datasets),":[153],"may":[156],"mitigate":[157],"collapse,":[159],"although":[160],"they":[161],"entirely":[164],"prevent":[165],"it.":[166],"theoretical":[168],"findings":[169],"verified":[172],"through":[173],"experiments":[174],"on":[175],"feed-forward":[179],"images.":[183]},"abstract_inverted_index_v3":null,"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4403964227","counts_by_year":[],"updated_date":"2025-04-02T16:04:06.568090","created_date":"2024-11-01"}