{"id":"https://openalex.org/W4387322997","doi":"https://doi.org/10.48550/arxiv.2310.01382","title":"Compressing LLMs: The Truth is Rarely Pure and Never Simple","display_name":"Compressing LLMs: The Truth is Rarely Pure and Never Simple","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4387322997","doi":"https://doi.org/10.48550/arxiv.2310.01382"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2310.01382","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false},"type":"preprint","type_crossref":"posted-content","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/abs/2310.01382","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5034136464","display_name":"Ajay Jaiswal","orcid":"https://orcid.org/0000-0001-9032-2916"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jaiswal, Ajay","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066666034","display_name":"Zhe Gan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gan, Zhe","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017048711","display_name":"Xianzhi Du","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Du, Xianzhi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100385159","display_name":"Bowen Zhang","orcid":"https://orcid.org/0000-0003-0892-2053"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Bowen","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048522863","display_name":"Zhangyang Wang","orcid":"https://orcid.org/0000-0002-2050-5693"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zhangyang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5112656212","display_name":"Yinfei Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Yinfei","raw_affiliation_strings":[],"affiliations":[]}],"institution_assertions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.933861,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":65,"max":76},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T14012","display_name":"Comparative and International Law Studies","score":0.9496,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T14012","display_name":"Comparative and International Law Studies","score":0.9496,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12374","display_name":"European and International Contract Law","score":0.9264,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14102","display_name":"Corporate Governance and Law","score":0.9039,"subfield":{"id":"https://openalex.org/subfields/1408","display_name":"Strategy and Management"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[],"concepts":[{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.80901945},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.32625782},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.24743757}],"mesh":[],"locations_count":3,"locations":[{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2310.01382","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.01382","pdf_url":"http://arxiv.org/pdf/2310.01382","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":false,"landing_page_url":"https://api.datacite.org/dois/10.48550/arxiv.2310.01382","pdf_url":null,"source":{"id":"https://openalex.org/S4393179698","display_name":"DataCite API","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I4210145204","host_organization_name":"DataCite","host_organization_lineage":["https://openalex.org/I4210145204"],"host_organization_lineage_names":["DataCite"],"type":"metadata"},"license":null,"license_id":null,"version":null}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2310.01382","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[{"score":0.61,"id":"https://metadata.un.org/sdg/4","display_name":"Quality education"}],"grants":[],"datasets":[],"versions":[],"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4300101996","https://openalex.org/W4253593777","https://openalex.org/W2951497643","https://openalex.org/W2382521049","https://openalex.org/W2338854850","https://openalex.org/W2184239527","https://openalex.org/W2165950148","https://openalex.org/W2144385241","https://openalex.org/W2142393343","https://openalex.org/W1585007175"],"abstract_inverted_index":{"Despite":[0],"their":[1,127,138],"remarkable":[2],"achievements,":[3],"modern":[4],"Large":[5],"Language":[6],"Models":[7],"(LLMs)":[8],"face":[9],"exorbitant":[10],"computational":[11],"and":[12,24,28,38,77,92,130,146,168,196],"memory":[13],"footprints.":[14],"Recently,":[15],"several":[16],"works":[17],"have":[18,123],"shown":[19],"significant":[20,124,158],"success":[21],"in":[22,137,173],"training-free":[23],"data-free":[25],"compression":[26,69,84,152,231],"(pruning":[27],"quantization)":[29],"of":[30,53,81,110,149,228],"LLMs":[31,186],"that":[32],"achieve":[33],"50":[34],"-":[35],"60%":[36],"sparsity":[37,164,172,191],"reduce":[39],"the":[40,56,79,116,226],"bit":[41],"width":[42],"to":[43,114,133,204],"3":[44],"or":[45],"4":[46],"bits":[47],"per":[48],"weight,":[49],"with":[50,126],"negligible":[51],"degradation":[52],"perplexity":[54,96,131],"over":[55],"uncompressed":[57],"baseline.":[58],"As":[59],"recent":[60],"research":[61],"efforts":[62],"are":[63,179,192,236],"focused":[64],"on":[65,88],"developing":[66],"increasingly":[67],"sophisticated":[68],"methods,":[70,85],"our":[71,222],"work":[72],"takes":[73],"a":[74,89,108],"step":[75],"back":[76],"re-evaluates":[78],"effectiveness":[80],"existing":[82],"SoTA":[83,151],"which":[86,122],"rely":[87],"fairly":[90],"simple":[91],"widely":[93],"questioned":[94],"metric,":[95],"(even":[97],"for":[98,119,170,210],"dense":[99,128],"LLMs).":[100],"We":[101,220],"introduce":[102],"Knowledge-Intensive":[103],"Compressed":[104],"LLM":[105,230],"BenchmarK":[106],"(LLM-KICK),":[107],"collection":[109],"carefully":[111],"curated":[112],"tasks":[113],"redefine":[115],"evaluation":[117],"protocol":[118],"compressed":[120,207],"LLMs,":[121],"alignment":[125],"counterparts":[129],"fail":[132,169],"capture":[134],"subtle":[135],"change":[136],"true":[139],"capabilities.":[140],"LLM-KICK":[141,201],"unveils":[142],"many":[143],"favorable":[144],"merits":[145],"unfortunate":[147],"plights":[148],"current":[150,176],"methods:":[153],"all":[154],"pruning":[155],"methods":[156,178],"suffer":[157],"performance":[159],"degradation,":[160],"sometimes":[161],"at":[162,188,238],"trivial":[163],"ratios":[165],"(e.g.,":[166],"25-30%),":[167],"N:M":[171],"knowledge-intensive":[174],"tasks;":[175],"quantization":[177],"more":[180],"successful":[181],"than":[182],"pruning;":[183],"yet,":[184],"pruned":[185],"even":[187],"$\\geq":[189],"50$%":[190],"robust":[193],"in-context":[194,215,217],"retrieval":[195],"summarization":[197],"systems;":[198],"among":[199],"others.":[200],"is":[202],"designed":[203],"holistically":[205],"access":[206],"LLMs'":[208],"ability":[209],"language":[211],"understanding,":[212],"reasoning,":[213],"generation,":[214],"retrieval,":[216],"summarization,":[218],"etc.":[219],"hope":[221],"study":[223],"can":[224],"foster":[225],"development":[227],"better":[229],"methods.":[232],"The":[233],"reproduced":[234],"codes":[235],"available":[237],"https://github.com/VITA-Group/llm-kick.":[239]},"abstract_inverted_index_v3":null,"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4387322997","counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2025-04-14T06:18:11.115600","created_date":"2023-10-04"}