{"id":"https://openalex.org/W4399198098","doi":"https://doi.org/10.48550/arxiv.2405.18740","title":"Reverse Image Retrieval Cues Parametric Memory in Multimodal LLMs","display_name":"Reverse Image Retrieval Cues Parametric Memory in Multimodal LLMs","publication_year":2024,"publication_date":"2024-05-29","ids":{"openalex":"https://openalex.org/W4399198098","doi":"https://doi.org/10.48550/arxiv.2405.18740"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.18740","pdf_url":"http://arxiv.org/pdf/2405.18740","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"type":"preprint","type_crossref":"posted-content","indexed_in":["arxiv"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://arxiv.org/pdf/2405.18740","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023143758","display_name":"Jialiang Xu","orcid":"https://orcid.org/0000-0003-2441-4809"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Jialiang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021730842","display_name":"Michael Moor","orcid":"https://orcid.org/0000-0003-4911-6437"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Moor, Michael","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5091272738","display_name":"Jure Leskovec","orcid":"https://orcid.org/0000-0002-5411-923X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Leskovec, Jure","raw_affiliation_strings":[],"affiliations":[]}],"institution_assertions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":0,"max":84},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9412,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9412,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[],"concepts":[{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.5197068},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4773976},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.40911245},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3852841},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3571396},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.30101582}],"mesh":[],"locations_count":1,"locations":[{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.18740","pdf_url":"http://arxiv.org/pdf/2405.18740","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.18740","pdf_url":"http://arxiv.org/pdf/2405.18740","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[],"grants":[],"datasets":[],"versions":[],"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W3116076068","https://openalex.org/W2951359407","https://openalex.org/W2775347418","https://openalex.org/W2772917594","https://openalex.org/W2755342338","https://openalex.org/W2229312674","https://openalex.org/W2166024367","https://openalex.org/W2079911747","https://openalex.org/W2058170566","https://openalex.org/W1969923398"],"abstract_inverted_index":{"Despite":[0],"impressive":[1],"advances":[2],"in":[3,68,121],"recent":[4],"multimodal":[5],"large":[6],"language":[7],"models":[8,11],"(MLLMs),":[9],"state-of-the-art":[10],"such":[12],"as":[13],"from":[14],"the":[15,83,110,136,163],"GPT-4":[16,60],"suite":[17],"still":[18],"struggle":[19],"with":[20,42],"knowledge-intensive":[21,51],"tasks.":[22],"To":[23,75],"address":[24],"this,":[25],"we":[26,78,118,133],"consider":[27],"Reverse":[28],"Image":[29],"Retrieval":[30],"(RIR)":[31],"augmented":[32],"generation,":[33],"a":[34,114,129],"simple":[35],"yet":[36],"effective":[37],"strategy":[38],"to":[39,85,113,151,154],"augment":[40],"MLLMs":[41],"web-scale":[43],"reverse":[44],"image":[45],"search":[46],"results.":[47],"RIR":[48,81,97,123,141,153,161],"robustly":[49],"improves":[50],"visual":[52,103],"question":[53],"answering":[54],"(VQA)":[55],"of":[56,70,139],"GPT-4V":[57],"by":[58,62,66,100],"37-43%,":[59],"Turbo":[61],"25-27%,":[63],"and":[64,104,127],"GPT-4o":[65],"18-20%":[67],"terms":[69],"open-ended":[71],"VQA":[72],"evaluation":[73],"metrics.":[74],"our":[76,93],"surprise,":[77],"discover":[79],"that":[80,96,135,148],"helps":[82,99],"model":[84],"better":[86,156],"access":[87],"its":[88],"own":[89],"world":[90],"knowledge.":[91],"Concretely,":[92],"experiments":[94],"suggest":[95],"augmentation":[98],"providing":[101],"further":[102],"textual":[105],"cues":[106],"without":[107],"necessarily":[108],"containing":[109],"direct":[111],"answer":[112],"query.":[115],"In":[116],"addition,":[117],"elucidate":[119],"cases":[120],"which":[122],"can":[124,149],"hurt":[125],"performance":[126],"conduct":[128],"human":[130],"evaluation.":[131],"Finally,":[132],"find":[134],"overall":[137],"advantage":[138],"using":[140],"makes":[142],"it":[143],"difficult":[144],"for":[145],"an":[146,158],"agent":[147],"choose":[150],"use":[152],"perform":[155],"than":[157],"approach":[159],"where":[160],"is":[162],"default":[164],"setting.":[165]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4399198098","counts_by_year":[],"updated_date":"2024-12-11T13:38:55.363494","created_date":"2024-05-31"}