{"id":"https://openalex.org/W4386071547","doi":"https://doi.org/10.1109/cvpr52729.2023.01832","title":"MaPLe: Multi-modal Prompt Learning","display_name":"MaPLe: Multi-modal Prompt Learning","publication_year":2023,"publication_date":"2023-06-01","ids":{"openalex":"https://openalex.org/W4386071547","doi":"https://doi.org/10.1109/cvpr52729.2023.01832"},"language":"en","primary_location":{"is_oa":false,"landing_page_url":"https://doi.org/10.1109/cvpr52729.2023.01832","pdf_url":null,"source":{"id":"https://openalex.org/S4363607701","display_name":"2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false},"type":"article","type_crossref":"proceedings-article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2210.03117","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5077014645","display_name":"Muhammad Uzair Khattak","orcid":null},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Muhammad Uzair Khattak","raw_affiliation_strings":["Mohamed bin Zayed University of AI"],"affiliations":[{"raw_affiliation_string":"Mohamed bin Zayed University of AI","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064948724","display_name":"Hanoona Rasheed","orcid":null},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Hanoona Rasheed","raw_affiliation_strings":["Mohamed bin Zayed University of AI"],"affiliations":[{"raw_affiliation_string":"Mohamed bin Zayed University of AI","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034370385","display_name":"Muhammad Maaz","orcid":"https://orcid.org/0000-0002-3869-631X"},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Muhammad Maaz","raw_affiliation_strings":["Mohamed bin Zayed University of AI"],"affiliations":[{"raw_affiliation_string":"Mohamed bin Zayed University of AI","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101483978","display_name":"Salman Khan","orcid":"https://orcid.org/0000-0001-8732-3395"},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]},{"id":"https://openalex.org/I118347636","display_name":"Australian National University","ror":"https://ror.org/019wvm592","country_code":"AU","type":"education","lineage":["https://openalex.org/I118347636"]}],"countries":["AE","AU"],"is_corresponding":false,"raw_author_name":"Salman Khan","raw_affiliation_strings":["Australian National University","Mohamed bin Zayed University of AI"],"affiliations":[{"raw_affiliation_string":"Mohamed bin Zayed University of AI","institution_ids":["https://openalex.org/I4210113480"]},{"raw_affiliation_string":"Australian National University","institution_ids":["https://openalex.org/I118347636"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100760570","display_name":"Fahad Shahbaz Khan","orcid":"https://orcid.org/0000-0002-4263-3143"},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Fahad Shahbaz Khan","raw_affiliation_strings":["Mohamed bin Zayed University of AI"],"affiliations":[{"raw_affiliation_string":"Mohamed bin Zayed University of AI","institution_ids":["https://openalex.org/I4210113480"]}]}],"institution_assertions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":60.825,"has_fulltext":false,"cited_by_count":248,"citation_normalized_percentile":{"value":0.999743,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"19113","last_page":"19122"},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9999,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9999,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9994,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9682,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/representation","display_name":"Representation","score":0.45933285},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4161389}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.82649744},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.6682359},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5942625},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5701598},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.49753812},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.4871192},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.45933285},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.45660686},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.45273888},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.43588698},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.4168822},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4161389},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3951041},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.12694815},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"is_oa":false,"landing_page_url":"https://doi.org/10.1109/cvpr52729.2023.01832","pdf_url":null,"source":{"id":"https://openalex.org/S4363607701","display_name":"2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false},{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2210.03117","pdf_url":"https://arxiv.org/pdf/2210.03117","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2210.03117","pdf_url":"https://arxiv.org/pdf/2210.03117","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[{"score":0.78,"display_name":"Quality education","id":"https://metadata.un.org/sdg/4"}],"grants":[],"datasets":[],"versions":[],"referenced_works_count":48,"referenced_works":["https://openalex.org/W12634471","https://openalex.org/W1572666543","https://openalex.org/W1846799578","https://openalex.org/W1977295328","https://openalex.org/W2017814585","https://openalex.org/W2047643928","https://openalex.org/W2108598243","https://openalex.org/W2138011018","https://openalex.org/W2155904486","https://openalex.org/W24089286","https://openalex.org/W2533598788","https://openalex.org/W2947707615","https://openalex.org/W2953937638","https://openalex.org/W2964194231","https://openalex.org/W3037492894","https://openalex.org/W3094502228","https://openalex.org/W3126337491","https://openalex.org/W3166396011","https://openalex.org/W3174770825","https://openalex.org/W3177096435","https://openalex.org/W3198377975","https://openalex.org/W3204250462","https://openalex.org/W3205270560","https://openalex.org/W3205789812","https://openalex.org/W3206072662","https://openalex.org/W3207493267","https://openalex.org/W3212456749","https://openalex.org/W3213454282","https://openalex.org/W3213647938","https://openalex.org/W3215626407","https://openalex.org/W4205991051","https://openalex.org/W4224545477","https://openalex.org/W4225922988","https://openalex.org/W4226058394","https://openalex.org/W4226452378","https://openalex.org/W4229453513","https://openalex.org/W4284961860","https://openalex.org/W4296151208","https://openalex.org/W4310557340","https://openalex.org/W4312238419","https://openalex.org/W4312310776","https://openalex.org/W4312351187","https://openalex.org/W4312420092","https://openalex.org/W4312480274","https://openalex.org/W4312559104","https://openalex.org/W4312925417","https://openalex.org/W4312935996","https://openalex.org/W4390873714"],"related_works":["https://openalex.org/W43109613","https://openalex.org/W3162204513","https://openalex.org/W3083152911","https://openalex.org/W2997567050","https://openalex.org/W2371138613","https://openalex.org/W2359952343","https://openalex.org/W2239445980","https://openalex.org/W2080152487","https://openalex.org/W2075445622","https://openalex.org/W2048963458"],"abstract_inverted_index":{"Pre-trained":[0],"vision-language":[1,127],"(V-L)":[2],"models":[3,219],"such":[4],"as":[5,50],"CLIP":[6,45,56,73],"have":[7],"shown":[8],"excellent":[9],"generalization":[10,172],"ability":[11],"to":[12,19,33,54,65,86,111,129,148,155,173],"downstream":[13,58,94],"tasks.":[14,59],"However,":[15],"they":[16],"are":[17,220],"sensitive":[18],"the":[20,38,51,84,115,126,151,162,185],"choice":[21],"of":[22,30,72,164,171,198],"input":[23],"text":[24],"prompts":[25,49,128,143],"and":[26,108,117,133,179,193,203,217],"require":[27],"careful":[28],"selection":[29],"prompt":[31],"templates":[32],"perform":[34],"well.":[35],"Inspired":[36],"by":[37],"Natural":[39],"Language":[40],"Processing":[41],"(NLP)":[42],"literature,":[43],"recent":[44],"adaptation":[46],"approaches":[47],"learn":[48,141],"textual":[52],"inputs":[53],"fine-tune":[55],"for":[57,105],"We":[60,160],"note":[61],"that":[62],"using":[63],"prompting":[64],"adapt":[66],"representations":[67],"in":[68],"a":[69,93],"single":[70],"branch":[71],"(language":[74],"or":[75],"vision)":[76],"is":[77],"sub-optimal":[78],"since":[79],"it":[80],"does":[81],"not":[82],"allow":[83,156],"flexibility":[85],"dynamically":[87],"adjust":[88],"both":[89,106],"representation":[90],"spaces":[91],"on":[92,167,200,205],"task.":[95],"In":[96],"this":[97],"work,":[98],"we":[99,140],"propose":[100],"Multi-modal":[101],"Prompt":[102],"Learning":[103],"(MaPLe)":[104],"vision":[107,116],"language":[109,118],"branches":[110],"improve":[112],"alignment":[113],"between":[114,125],"representations.":[119],"Our":[120,215],"design":[121],"promotes":[122],"strong":[123],"coupling":[124],"ensure":[130],"mutual":[131],"synergy":[132],"discourages":[134],"learning":[135],"independent":[136],"uni-modal":[137],"solutions.":[138],"Further,":[139],"separate":[142],"across":[144],"different":[145],"early":[146],"stages":[147],"progressively":[149],"model":[150],"stage-wise":[152],"feature":[153],"relationships":[154],"rich":[157],"context":[158],"learning.":[159],"evaluate":[161],"effectiveness":[163],"our":[165],"approach":[166],"three":[168],"representative":[169],"tasks":[170],"novel":[174,201],"classes,":[175],"new":[176],"target":[177],"datasets":[178],"unseen":[180],"domain":[181],"shifts.":[182],"Compared":[183],"with":[184],"state-of-the-art":[186],"method":[187],"Co-CoOp,":[188],"MaPLe":[189],"exhibits":[190],"favorable":[191],"performance":[192],"achieves":[194],"an":[195],"absolute":[196],"gain":[197],"3.45%":[199],"classes":[202],"2.72%":[204],"overall":[206],"harmonic-mean,":[207],"averaged":[208],"over":[209],"11":[210],"diverse":[211],"image":[212],"recognition":[213],"datasets.":[214],"code":[216],"pre-trained":[218],"available":[221],"at":[222],"https://github.com/muzairkhattak/multimodal-prompt-learning.":[223]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4386071547","counts_by_year":[{"year":2024,"cited_by_count":198},{"year":2023,"cited_by_count":44}],"updated_date":"2025-01-16T03:04:23.393485","created_date":"2023-08-23"}