{"id":"https://openalex.org/W4391673080","doi":"https://doi.org/10.48550/arxiv.2402.04615","title":"ScreenAI: A Vision-Language Model for UI and Infographics Understanding","display_name":"ScreenAI: A Vision-Language Model for UI and Infographics Understanding","publication_year":2024,"publication_date":"2024-02-07","ids":{"openalex":"https://openalex.org/W4391673080","doi":"https://doi.org/10.48550/arxiv.2402.04615"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.04615","pdf_url":"http://arxiv.org/pdf/2402.04615","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"type":"preprint","type_crossref":"posted-content","indexed_in":["arxiv"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://arxiv.org/pdf/2402.04615","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5072231116","display_name":"Gilles Baechler","orcid":"https://orcid.org/0000-0002-8453-6093"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Baechler, Gilles","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070178342","display_name":"Srinivas Sunkara","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sunkara, Srinivas","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065433568","display_name":"Maria Wang","orcid":"https://orcid.org/0009-0006-1151-7648"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Maria","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093895782","display_name":"Fedir Zubach","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zubach, Fedir","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089795752","display_name":"Hassan Mansoor","orcid":"https://orcid.org/0000-0002-7198-0582"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mansoor, Hassan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093895783","display_name":"Vincent Etter","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Etter, Vincent","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079740254","display_name":"Victor C\u0103rbune","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"C\u0103rbune, Victor","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113306691","display_name":"Jason Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Jason","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101933323","display_name":"Jindong Chen","orcid":"https://orcid.org/0000-0003-4551-2344"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Jindong","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5000005511","display_name":"Abhanshu Sharma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sharma, Abhanshu","raw_affiliation_strings":[],"affiliations":[]}],"institution_assertions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.875085,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":77,"max":88},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10803","display_name":"Innovative Human-Technology Interaction","score":0.9208,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10803","display_name":"Innovative Human-Technology Interaction","score":0.9208,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14074","display_name":"Persona Design and Applications","score":0.9082,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/infographic","display_name":"Infographic","score":0.99623615}],"concepts":[{"id":"https://openalex.org/C156365220","wikidata":"https://www.wikidata.org/wiki/Q845734","display_name":"Infographic","level":2,"score":0.99623615},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.52085835},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.04615","pdf_url":"http://arxiv.org/pdf/2402.04615","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.04615","pdf_url":"http://arxiv.org/pdf/2402.04615","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[],"grants":[],"datasets":[],"versions":[],"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4389434054","https://openalex.org/W4389433970","https://openalex.org/W3160274551","https://openalex.org/W3021287274","https://openalex.org/W2996895802","https://openalex.org/W2985775149","https://openalex.org/W2799836731","https://openalex.org/W2783937668","https://openalex.org/W2748952813","https://openalex.org/W2523628251"],"abstract_inverted_index":{"Screen":[0],"user":[1],"interfaces":[2],"(UIs)":[3],"and":[4,10,19,32,49,79,96,103,131,138,141,150,171],"infographics,":[5],"sharing":[6],"similar":[7,156],"visual":[8],"language":[9],"design":[11,119],"principles,":[12],"play":[13],"important":[14],"roles":[15],"in":[16,30,70],"human":[17],"communication":[18],"human-machine":[20],"interaction.":[21],"We":[22,84,109],"introduce":[23],"ScreenAI,":[24],"a":[25,53,65],"vision-language":[26],"model":[27,36,73],"that":[28],"specializes":[29],"UI":[31,82,101],"infographics":[33],"understanding.":[34],"Our":[35],"improves":[37],"upon":[38],"the":[39,43,59,72,77,115,167],"PaLI":[40],"architecture":[41],"with":[42],"flexible":[44],"patching":[45],"strategy":[46],"of":[47,56,61,81,117,155],"pix2struct":[48],"is":[50,64],"trained":[51],"on":[52,129,145,166,175],"unique":[54],"mixture":[55,63],"datasets.":[57],"At":[58,121],"heart":[60],"this":[62],"novel":[66],"screen":[67,168],"annotation":[68,169],"task":[69,170],"which":[71],"has":[74],"to":[75,89,92,113,153],"identify":[76],"type":[78],"location":[80],"elements.":[83],"use":[85],"these":[86,118],"text":[87],"annotations":[88],"describe":[90],"screens":[91],"Large":[93],"Language":[94],"Models":[95],"automatically":[97],"generate":[98],"question-answering":[99],"(QA),":[100],"navigation,":[102],"summarization":[104],"training":[105],"datasets":[106],"at":[107],"scale.":[108],"run":[110],"ablation":[111],"studies":[112],"demonstrate":[114],"impact":[116],"choices.":[120],"only":[122],"5B":[123],"parameters,":[124],"ScreenAI":[125],"achieves":[126],"new":[127,142,162],"state-of-the-artresults":[128],"UI-":[130],"infographics-based":[132],"tasks":[133],"(Multi-page":[134],"DocVQA,":[135,149],"WebSRC,":[136],"MoTIF":[137],"Widget":[139],"Captioning),":[140],"best-in-class":[143],"performance":[144],"others":[146,173],"(Chart":[147],"QA,":[148],"InfographicVQA)":[151],"compared":[152],"models":[154],"size.":[157],"Finally,":[158],"we":[159],"release":[160],"three":[161],"datasets:":[163],"one":[164],"focused":[165,174],"two":[172],"question":[176],"answering.":[177]},"abstract_inverted_index_v3":null,"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4391673080","counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2025-04-23T22:18:21.639464","created_date":"2024-02-09"}