{"id":"https://openalex.org/W4399399681","doi":"https://doi.org/10.48550/arxiv.2406.00894","title":"Pretrained Hybrids with MAD Skills","display_name":"Pretrained Hybrids with MAD Skills","publication_year":2024,"publication_date":"2024-06-02","ids":{"openalex":"https://openalex.org/W4399399681","doi":"https://doi.org/10.48550/arxiv.2406.00894"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2406.00894","pdf_url":"https://arxiv.org/pdf/2406.00894","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"type":"preprint","type_crossref":"posted-content","indexed_in":["arxiv"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2406.00894","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5104060780","display_name":"Nicholas Roberts","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Roberts, Nicholas","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101373599","display_name":"Samuel Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Samuel","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114132601","display_name":"Zhiqi Gao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gao, Zhiqi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5099038265","display_name":"Satya Sai Srinath Namburi GNVV","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"GNVV, Satya Sai Srinath Namburi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083463013","display_name":"Sonia Cromp","orcid":"https://orcid.org/0009-0002-6556-3317"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cromp, Sonia","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113242103","display_name":"Chengjun Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Chengjun","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108940227","display_name":"Chengyu Duan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Duan, Chengyu","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5103088414","display_name":"Fr\u00e9d\u00e9ric Sala","orcid":"https://orcid.org/0000-0003-0379-2827"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sala, Frederic","raw_affiliation_strings":[],"affiliations":[]}],"institution_assertions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":0,"max":84},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T11195","display_name":"Simulation Techniques and Applications","score":0.0653,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11195","display_name":"Simulation Techniques and Applications","score":0.0653,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11450","display_name":"Model-Driven Software Engineering Techniques","score":0.0553,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[],"concepts":[{"id":"https://openalex.org/C48189365","wikidata":"https://www.wikidata.org/wiki/Q42621","display_name":"Hybrid","level":2,"score":0.7097479},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4268199},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.38414815},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.3365163},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.32613522},{"id":"https://openalex.org/C144027150","wikidata":"https://www.wikidata.org/wiki/Q48803","display_name":"Horticulture","level":1,"score":0.1343689}],"mesh":[],"locations_count":1,"locations":[{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2406.00894","pdf_url":"https://arxiv.org/pdf/2406.00894","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2406.00894","pdf_url":"https://arxiv.org/pdf/2406.00894","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[],"grants":[],"datasets":[],"versions":[],"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2788277189","https://openalex.org/W2748952813","https://openalex.org/W2130076355","https://openalex.org/W2119695867","https://openalex.org/W2082860237","https://openalex.org/W2046158694","https://openalex.org/W2013243191","https://openalex.org/W1993764875","https://openalex.org/W1990804418"],"abstract_inverted_index":{"While":[0],"Transformers":[1],"underpin":[2],"modern":[3],"large":[4],"language":[5],"models":[6,83,118],"(LMs),":[7],"there":[8],"is":[9,46],"a":[10,34,67],"growing":[11],"list":[12],"of":[13,41,77,145],"alternative":[14],"architectures":[15],"with":[16],"new":[17,57],"capabilities,":[18],"promises,":[19],"and":[20,56,129,152,178,184],"tradeoffs.":[21],"This":[22],"makes":[23],"choosing":[24],"the":[25,39,75,126,143,153],"right":[26],"LM":[27,137],"architecture":[28,121],"challenging.":[29],"Recently-proposed":[30],"$\\textit{hybrid":[31],"architectures}$":[32,79],"seek":[33],"best-of-all-worlds":[35],"approach":[36,89],"that":[37,69,102,115],"reaps":[38],"benefits":[40],"all":[42],"architectures.":[43,110],"Hybrid":[44],"design":[45,76],"difficult":[47],"for":[48],"two":[49],"reasons:":[50],"it":[51],"requires":[52],"manual":[53],"expert-driven":[54],"search,":[55],"hybrids":[58,114,147,158,164],"must":[59],"be":[60],"trained":[61],"from":[62,92,108,119,148],"scratch.":[63],"We":[64,111],"propose":[65],"$\\textbf{Manticore}$,":[66],"framework":[68],"addresses":[70],"these":[71],"challenges.":[72],"Manticore":[73,163],"$\\textit{automates":[74],"hybrid":[78],"while":[80],"reusing":[81],"pretrained":[82,106,117,146,150,157,182],"to":[84,155,159],"create":[85],"$\\textit{pretrained}$":[86],"hybrids.":[87],"Our":[88],"augments":[90],"ideas":[91],"differentiable":[93],"Neural":[94],"Architecture":[95],"Search":[96],"(NAS)":[97],"by":[98],"incorporating":[99],"simple":[100],"projectors":[101],"translate":[103],"features":[104],"between":[105],"blocks":[107],"different":[109,120],"then":[112],"fine-tune":[113],"combine":[116],"families":[122],"--":[123,131],"such":[124],"as":[125],"GPT":[127],"series":[128],"Mamba":[130],"end-to-end.":[132],"With":[133],"Manticore,":[134],"we":[135],"enable":[136],"selection":[138],"without":[139],"training":[140],"multiple":[141],"models,":[142,151],"construction":[144],"existing":[149,166],"ability":[154],"$\\textit{program}$":[156],"have":[160],"certain":[161],"capabilities.":[162],"outperform":[165],"manually-designed":[167],"hybrids,":[168],"achieve":[169],"strong":[170],"performance":[171],"on":[172,181],"Long":[173],"Range":[174],"Arena":[175],"(LRA)":[176],"tasks,":[177],"can":[179],"improve":[180],"transformers":[183],"state":[185],"space":[186],"models.":[187]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4399399681","counts_by_year":[],"updated_date":"2024-12-15T08:53:36.503704","created_date":"2024-06-07"}