{"id":"https://openalex.org/W4309130070","doi":"https://doi.org/10.48550/arxiv.2211.06687","title":"Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation","display_name":"Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W4309130070","doi":"https://doi.org/10.48550/arxiv.2211.06687"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2211.06687","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false},"type":"preprint","type_crossref":"posted-content","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/abs/2211.06687","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102132394","display_name":"Yusong Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Yusong","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100451984","display_name":"Ke Chen","orcid":"https://orcid.org/0000-0001-9457-9364"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Ke","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100425864","display_name":"Tianyu Zhang","orcid":"https://orcid.org/0000-0003-1969-2855"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Tianyu","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103129849","display_name":"Yuchen Hui","orcid":"https://orcid.org/0000-0002-9659-3714"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hui, Yuchen","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017455302","display_name":"Taylor Berg-Kirkpatrick","orcid":"https://orcid.org/0000-0002-1283-4075"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Berg-Kirkpatrick, Taylor","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5042278809","display_name":"Shlomo Dubnov","orcid":"https://orcid.org/0000-0003-0222-1125"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dubnov, Shlomo","raw_affiliation_strings":[],"affiliations":[]}],"institution_assertions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.674307,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":69,"max":75},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9994,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9994,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9952,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9941,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/audio-mining","display_name":"Audio mining","score":0.6035227},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.52250797}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8409451},{"id":"https://openalex.org/C157968479","wikidata":"https://www.wikidata.org/wiki/Q3079876","display_name":"Audio mining","level":4,"score":0.6035227},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5403917},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5396642},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.5251869},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.52250797},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5088256},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.50575566},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.48602515},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.4780346},{"id":"https://openalex.org/C155635449","wikidata":"https://www.wikidata.org/wiki/Q4674699","display_name":"Acoustic model","level":3,"score":0.29271987},{"id":"https://openalex.org/C61328038","wikidata":"https://www.wikidata.org/wiki/Q3358061","display_name":"Speech processing","level":2,"score":0.21019831},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.09914154},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2211.06687","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2211.06687","pdf_url":"http://arxiv.org/pdf/2211.06687","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":false,"landing_page_url":"https://api.datacite.org/dois/10.48550/arxiv.2211.06687","pdf_url":null,"source":{"id":"https://openalex.org/S4393179698","display_name":"DataCite API","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I4210145204","host_organization_name":"DataCite","host_organization_lineage":["https://openalex.org/I4210145204"],"host_organization_lineage_names":["DataCite"],"type":"metadata"},"license":null,"license_id":null,"version":null}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2211.06687","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[{"score":0.7,"display_name":"Quality education","id":"https://metadata.un.org/sdg/4"}],"grants":[],"datasets":[],"versions":[],"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4388145910","https://openalex.org/W4312490297","https://openalex.org/W4248336175","https://openalex.org/W3009369890","https://openalex.org/W2391445434","https://openalex.org/W2381570729","https://openalex.org/W2366107444","https://openalex.org/W2062212388","https://openalex.org/W2031260042","https://openalex.org/W1976205134"],"abstract_inverted_index":{"Contrastive":[0],"learning":[1],"has":[2],"shown":[3],"remarkable":[4],"success":[5],"in":[6,131,144,158],"the":[7,74,82,88,99,139,145,159,164,171],"field":[8],"of":[9,20,48,94],"multimodal":[10],"representation":[11,28],"learning.":[12],"In":[13,135],"this":[14,39],"paper,":[15],"we":[16,41,57,102],"propose":[17],"a":[18,45,59],"pipeline":[19],"contrastive":[21,60],"language-audio":[22,61],"pretraining":[23,62],"to":[24,85,90,106,151,155,170],"develop":[25],"an":[26],"audio":[27,31,67,92,116,120,136],"by":[29,64],"combining":[30],"data":[32,54],"with":[33],"natural":[34],"language":[35],"descriptions.":[36],"To":[37],"accomplish":[38],"target,":[40],"first":[42],"release":[43],"LAION-Audio-630K,":[44],"large":[46],"collection":[47],"633,526":[49],"audio-text":[50],"pairs":[51],"from":[52],"different":[53,66],"sources.":[55],"Second,":[56],"construct":[58],"model":[63,83,89,109,127,140,166],"considering":[65],"encoders":[68],"and":[69,78,97,118,148,163],"text":[70],"encoders.":[71],"We":[72],"incorporate":[73],"feature":[75],"fusion":[76],"mechanism":[77],"keyword-to-caption":[79],"augmentation":[80],"into":[81],"design":[84],"further":[86],"enable":[87],"process":[91],"inputs":[93],"variable":[95],"lengths":[96],"enhance":[98],"performance.":[100],"Third,":[101],"perform":[103],"comprehensive":[104],"experiments":[105],"evaluate":[107],"our":[108,126],"across":[110],"three":[111],"tasks:":[112],"text-to-audio":[113,132],"retrieval,":[114],"zero-shot":[115,146],"classification,":[117],"supervised":[119],"classification.":[121],"The":[122],"results":[123,157],"demonstrate":[124],"that":[125],"achieves":[128,141],"superior":[129],"performance":[130,143,153],"retrieval":[133],"task.":[134],"classification":[137],"tasks,":[138],"state-of-the-art":[142],"setting":[147],"is":[149],"able":[150],"obtain":[152],"comparable":[154],"models'":[156],"non-zero-shot":[160],"setting.":[161],"LAION-Audio-630K":[162],"proposed":[165],"are":[167],"both":[168],"available":[169],"public.":[172]},"abstract_inverted_index_v3":null,"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4309130070","counts_by_year":[{"year":2024,"cited_by_count":2}],"updated_date":"2025-03-20T12:08:38.066526","created_date":"2022-11-23"}