{"id":"https://openalex.org/W4399597324","doi":"https://doi.org/10.48550/arxiv.2406.06567","title":"DHA: Learning Decoupled-Head Attention from Transformer Checkpoints via\n Adaptive Heads Fusion","display_name":"DHA: Learning Decoupled-Head Attention from Transformer Checkpoints via\n Adaptive Heads Fusion","publication_year":2024,"publication_date":"2024-06-03","ids":{"openalex":"https://openalex.org/W4399597324","doi":"https://doi.org/10.48550/arxiv.2406.06567"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.06567","pdf_url":"http://arxiv.org/pdf/2406.06567","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"type":"preprint","type_crossref":"posted-content","indexed_in":["arxiv"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://arxiv.org/pdf/2406.06567","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5005755293","display_name":"Yilong Chen","orcid":"https://orcid.org/0009-0002-0732-3323"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Yilong","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014957302","display_name":"Linhao Zhang","orcid":"https://orcid.org/0000-0002-8484-1733"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Linhao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023311055","display_name":"Junyuan Shang","orcid":"https://orcid.org/0000-0003-4301-750X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shang, Junyuan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100389521","display_name":"Zhenyu Zhang","orcid":"https://orcid.org/0000-0002-2309-3145"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Zhenyu","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103214505","display_name":"Tingwen Liu","orcid":"https://orcid.org/0000-0003-0487-0751"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Tingwen","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058880150","display_name":"Shuohuan Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Shuohuan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5100772257","display_name":"Yu Sun","orcid":"https://orcid.org/0000-0001-7225-9677"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Yu","raw_affiliation_strings":[],"affiliations":[]}],"institution_assertions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":0,"max":82},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.7871,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.7871,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13050","display_name":"Oil and Gas Production Techniques","score":0.7714,"subfield":{"id":"https://openalex.org/subfields/2212","display_name":"Ocean Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11206","display_name":"Model Reduction and Neural Networks","score":0.7687,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[],"concepts":[{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6225716},{"id":"https://openalex.org/C158525013","wikidata":"https://www.wikidata.org/wiki/Q2593739","display_name":"Fusion","level":2,"score":0.5545027},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5364232},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4413546},{"id":"https://openalex.org/C2780312720","wikidata":"https://www.wikidata.org/wiki/Q5689100","display_name":"Head (geology)","level":2,"score":0.42700022},{"id":"https://openalex.org/C171146098","wikidata":"https://www.wikidata.org/wiki/Q124192","display_name":"Automotive engineering","level":1,"score":0.35014814},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.23706657},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.21038109},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.13979143},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.08362427},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.06567","pdf_url":"http://arxiv.org/pdf/2406.06567","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.06567","pdf_url":"http://arxiv.org/pdf/2406.06567","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[],"grants":[],"datasets":[],"versions":[],"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W3097502728","https://openalex.org/W3009949491","https://openalex.org/W2967030268","https://openalex.org/W2613352840","https://openalex.org/W2530546662","https://openalex.org/W2379547295","https://openalex.org/W2185253430","https://openalex.org/W2113206756","https://openalex.org/W2099421762","https://openalex.org/W1927475415"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,136],"(LLMs)":[3],"with":[4],"billions":[5],"of":[6,63,99,118,129,141,158,167,172,188],"parameters":[7,40,121],"demonstrate":[8],"impressive":[9],"performance.":[10,58],"However,":[11],"the":[12,61,97,108,112,126,130,159],"widely":[13],"used":[14],"Multi-Head":[15],"Attention":[16,70,178],"(MHA)":[17],"in":[18],"LLMs":[19],"incurs":[20],"substantial":[21,52],"computational":[22],"and":[23,81,93,196],"memory":[24],"costs":[25,55],"during":[26],"inference.":[27],"While":[28],"some":[29],"efforts":[30],"have":[31],"optimized":[32],"attention":[33,64],"mechanisms":[34],"by":[35,96,123,137],"pruning":[36],"heads":[37,80,83],"or":[38,50],"sharing":[39,77],"among":[41],"heads,":[42,102],"these":[43],"methods":[44],"often":[45],"lead":[46],"to":[47,56,105,164,176],"performance":[48,92,168,190],"degradation":[49],"necessitate":[51],"continued":[53],"pre-training":[54,162,194,202],"restore":[57],"Based":[59],"on":[60],"analysis":[62],"redundancy,":[65],"we":[66,103],"design":[67],"a":[68,88,155,182,186],"Decoupled-Head":[69],"(DHA)":[71],"mechanism.":[72],"DHA":[73,113,135,152,180],"adaptively":[74],"configures":[75],"group":[76],"for":[78],"key":[79],"value":[82],"across":[84],"various":[85,139],"layers,":[86],"achieving":[87],"better":[89],"balance":[90],"between":[91],"efficiency.":[94],"Inspired":[95],"observation":[98],"clustering":[100],"similar":[101,119],"propose":[104],"progressively":[106],"transform":[107],"MHA":[109,131,142],"checkpoint":[110],"into":[111],"model":[114],"through":[115],"linear":[116],"fusion":[117],"head":[120,146],"step":[122],"step,":[124],"retaining":[125],"parametric":[127],"knowledge":[128],"checkpoint.":[132],"We":[133],"construct":[134],"transforming":[138],"scales":[140],"checkpoints":[143],"given":[144],"target":[145],"budgets.":[147],"Our":[148],"experiments":[149],"show":[150],"that":[151],"remarkably":[153],"requires":[154],"mere":[156],"0.25\\%":[157],"original":[160],"model's":[161],"budgets":[163],"achieve":[165],"97.6\\%":[166],"while":[169],"saving":[170],"75\\%":[171],"KV":[173],"cache.":[174],"Compared":[175],"Group-Query":[177],"(GQA),":[179],"achieves":[181],"5$\\times$":[183],"training":[184],"acceleration,":[185],"maximum":[187],"13.93\\%":[189],"improvement":[191,199],"under":[192,200],"0.01\\%":[193],"budget,":[195],"4\\%":[197],"relative":[198],"0.05\\%":[201],"budget.":[203]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4399597324","counts_by_year":[],"updated_date":"2025-01-22T19:17:00.214206","created_date":"2024-06-13"}