{"id":"https://openalex.org/W4403322803","doi":"https://doi.org/10.48550/arxiv.2410.04652","title":"Multimodal 3D Fusion and In-Situ Learning for Spatially Aware AI","display_name":"Multimodal 3D Fusion and In-Situ Learning for Spatially Aware AI","publication_year":2024,"publication_date":"2024-10-06","ids":{"openalex":"https://openalex.org/W4403322803","doi":"https://doi.org/10.48550/arxiv.2410.04652"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.04652","pdf_url":"http://arxiv.org/pdf/2410.04652","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"type":"preprint","type_crossref":"posted-content","indexed_in":["arxiv"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://arxiv.org/pdf/2410.04652","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028194519","display_name":"Chengyuan Xu","orcid":"https://orcid.org/0000-0001-9981-8889"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Chengyuan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024055231","display_name":"Radha Kumaran","orcid":"https://orcid.org/0000-0001-7161-3048"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kumaran, Radha","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002901804","display_name":"Noah Stier","orcid":"https://orcid.org/0000-0001-9602-4637"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Stier, Noah","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059751909","display_name":"Kangyou Yu","orcid":"https://orcid.org/0000-0001-5362-3162"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu, Kangyou","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5028885566","display_name":"Tobias H\u00f6llerer","orcid":"https://orcid.org/0000-0002-6240-0291"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"H\u00f6llerer, Tobias","raw_affiliation_strings":[],"affiliations":[]}],"institution_assertions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":0,"max":78},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10191","display_name":"Robotics and Sensor-Based Localization","score":0.9487,"subfield":{"id":"https://openalex.org/subfields/2202","display_name":"Aerospace Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10191","display_name":"Robotics and Sensor-Based Localization","score":0.9487,"subfield":{"id":"https://openalex.org/subfields/2202","display_name":"Aerospace Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10331","display_name":"Video Surveillance and Tracking Methods","score":0.9454,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[],"concepts":[{"id":"https://openalex.org/C2777822432","wikidata":"https://www.wikidata.org/wiki/Q216681","display_name":"In situ","level":2,"score":0.6989013},{"id":"https://openalex.org/C158525013","wikidata":"https://www.wikidata.org/wiki/Q2593739","display_name":"Fusion","level":2,"score":0.42676717},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4199792},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4189836},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3740397},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.21860841},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.09758431},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.087658465},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.04652","pdf_url":"http://arxiv.org/pdf/2410.04652","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.04652","pdf_url":"http://arxiv.org/pdf/2410.04652","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[],"grants":[],"datasets":[],"versions":[],"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4402327032","https://openalex.org/W4396701345","https://openalex.org/W4396696052","https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2382290278","https://openalex.org/W2376932109","https://openalex.org/W2358668433","https://openalex.org/W2001405890"],"abstract_inverted_index":{"Seamless":[0],"integration":[1],"of":[2,27,156],"virtual":[3],"and":[4,78,115,134,143,148,178,196,206],"physical":[5,17,90,141,173],"worlds":[6],"in":[7,37,53,125,145,172,208],"augmented":[8],"reality":[9],"benefits":[10],"from":[11],"the":[12,16,25,35,38,46,82,113,128,154,157],"system":[13,159,183],"semantically":[14],"\"understanding\"":[15],"environment.":[18],"AR":[19,106,163],"research":[20,207],"has":[21,50],"long":[22],"focused":[23],"on":[24,165],"potential":[26],"context":[28],"awareness,":[29],"demonstrating":[30],"novel":[31],"capabilities":[32],"that":[33,74,101,184],"leverage":[34],"semantics":[36],"3D":[39,71,98],"environment":[40,59,114],"for":[41,61,136],"various":[42],"object-level":[43],"interactions.":[44],"Meanwhile,":[45],"computer":[47],"vision":[48],"community":[49],"made":[51],"leaps":[52],"neural":[54],"vision-language":[55,110],"understanding":[56,104],"to":[57,105,138,202],"enhance":[58],"perception":[60],"autonomous":[62],"tasks.":[63],"In":[64],"this":[65],"work,":[66],"we":[67],"introduce":[68],"a":[69,95,146],"multimodal":[70,97,129],"object":[72,116,186],"representation":[73],"unifies":[75],"both":[76],"semantic":[77],"linguistic":[79,103],"knowledge":[80],"with":[81,127,140,175],"geometric":[83],"representation,":[84,130],"enabling":[85],"user-guided":[86],"machine":[87,122],"learning":[88],"involving":[89],"objects.":[91],"We":[92,118,152,190],"first":[93],"present":[94],"fast":[96],"reconstruction":[99],"pipeline":[100],"brings":[102],"by":[107],"fusing":[108],"CLIP":[109],"features":[111],"into":[112],"models.":[117],"then":[119],"propose":[120],"\"in-situ\"":[121],"learning,":[123],"which,":[124],"conjunction":[126],"enables":[131],"new":[132],"tools":[133],"interfaces":[135],"users":[137],"interact":[139],"spaces":[142],"objects":[144],"spatially":[147,209],"linguistically":[149],"meaningful":[150],"manner.":[151],"demonstrate":[153],"usefulness":[155],"proposed":[158],"through":[160],"two":[161],"real-world":[162],"applications":[164],"Magic":[166],"Leap":[167],"2:":[168],"a)":[169],"spatial":[170],"search":[171],"environments":[174],"natural":[176],"language":[177],"b)":[179],"an":[180],"intelligent":[181],"inventory":[182],"tracks":[185],"changes":[187],"over":[188],"time.":[189],"also":[191],"make":[192],"our":[193],"full":[194],"implementation":[195],"demo":[197],"data":[198],"available":[199],"at":[200],"(https://github.com/cy-xu/spatially_aware_AI)":[201],"encourage":[203],"further":[204],"exploration":[205],"aware":[210],"AI.":[211]},"abstract_inverted_index_v3":null,"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4403322803","counts_by_year":[],"updated_date":"2025-04-10T20:42:26.965841","created_date":"2024-10-12"}