{"id":"https://openalex.org/W4381586832","doi":"https://doi.org/10.48550/arxiv.2306.11706","title":"RoboCat: A Self-Improving Generalist Agent for Robotic Manipulation","display_name":"RoboCat: A Self-Improving Generalist Agent for Robotic Manipulation","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4381586832","doi":"https://doi.org/10.48550/arxiv.2306.11706"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2306.11706","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false},"type":"preprint","type_crossref":"posted-content","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/abs/2306.11706","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5079462322","display_name":"Konstantinos Bousmalis","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bousmalis, Konstantinos","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001501751","display_name":"Giulia Vezzani","orcid":"https://orcid.org/0000-0002-4151-6447"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vezzani, Giulia","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008638246","display_name":"Dushyant Rao","orcid":"https://orcid.org/0000-0002-1372-2884"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rao, Dushyant","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079786074","display_name":"Coline Devin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Devin, Coline","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027666200","display_name":"Alex X. Lee","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lee, Alex X.","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004820590","display_name":"Maria Bauz\u00e1","orcid":"https://orcid.org/0000-0002-8251-8019"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bauza, Maria","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082626279","display_name":"Todor Davchev","orcid":"https://orcid.org/0000-0002-0584-5163"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Davchev, Todor","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101900686","display_name":"Yuxiang Zhou","orcid":"https://orcid.org/0000-0003-1239-2526"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Yuxiang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079573718","display_name":"Agrim Gupta","orcid":"https://orcid.org/0000-0002-5213-0224"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gupta, Agrim","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021081842","display_name":"Akhil Raju","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Raju, Akhil","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047530171","display_name":"Antoine Laurens","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Laurens, Antoine","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063252645","display_name":"Claudio Fantacci","orcid":"https://orcid.org/0000-0002-9980-3617"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fantacci, Claudio","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044929367","display_name":"Valentin Dalibard","orcid":"https://orcid.org/0009-0005-4944-6745"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dalibard, Valentin","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074462312","display_name":"Martina Zambelli","orcid":"https://orcid.org/0000-0001-6061-191X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zambelli, Martina","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109616438","display_name":"Murilo Martins","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Martins, Murilo","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063215180","display_name":"Rugile Pevceviciute","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pevceviciute, Rugile","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092231032","display_name":"Michiel Blokzijl","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Blokzijl, Michiel","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008260665","display_name":"Misha Denil","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Denil, Misha","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010818667","display_name":"Nathan Batchelor","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Batchelor, Nathan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054636066","display_name":"Thomas Lampe","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lampe, Thomas","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045585535","display_name":"Emilio Parisotto","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Parisotto, Emilio","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032140556","display_name":"Konrad \u017bo\u0142na","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"\u017bo\u0142na, Konrad","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059110895","display_name":"Scott Reed","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Reed, Scott","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073651612","display_name":"Sergio G\u00f3mez Colmenarejo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Colmenarejo, Sergio G\u00f3mez","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014848474","display_name":"Jon Scholz","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Scholz, Jon","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007133617","display_name":"Abbas Abdolmaleki","orcid":"https://orcid.org/0000-0001-6692-5856"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Abdolmaleki, Abbas","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056401069","display_name":"Oliver Groth","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Groth, Oliver","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090859552","display_name":"Jean-Baptiste Regli","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Regli, Jean-Baptiste","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041307982","display_name":"O. P. Sushkov","orcid":"https://orcid.org/0000-0001-7828-5569"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sushkov, Oleg","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092231033","display_name":"Tom Roth\u00f6rl","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Roth\u00f6rl, Tom","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034908467","display_name":"Jos\u00e9 Enrique Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Jos\u00e9 Enrique","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056410851","display_name":"Yusuf Aytar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Aytar, Yusuf","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111468813","display_name":"D. B. Barker","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Barker, Dave","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114060213","display_name":"Joy Ortiz","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ortiz, Joy","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041323275","display_name":"Martin Riedmiller","orcid":"https://orcid.org/0000-0002-8465-5690"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Riedmiller, Martin","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017985443","display_name":"Jost Tobias Springenberg","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Springenberg, Jost Tobias","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079415139","display_name":"Raia Hadsell","orcid":"https://orcid.org/0000-0002-2390-1771"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hadsell, Raia","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063532849","display_name":"Francesco Nori","orcid":"https://orcid.org/0000-0003-3763-6873"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nori, Francesco","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5062951341","display_name":"Nicolas Heess","orcid":"https://orcid.org/0000-0001-7876-9256"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Heess, Nicolas","raw_affiliation_strings":[],"affiliations":[]}],"institution_assertions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.703406,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":78,"max":84},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.995,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.995,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.994,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9829,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.64399755}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7462405},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.6906634},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.64399755},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6169552},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5636085},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5383836},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5186893},{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.4303538},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.106766194},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2306.11706","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2306.11706","pdf_url":"http://arxiv.org/pdf/2306.11706","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":false,"landing_page_url":"https://api.datacite.org/dois/10.48550/arxiv.2306.11706","pdf_url":null,"source":{"id":"https://openalex.org/S4393179698","display_name":"DataCite API","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I4210145204","host_organization_name":"DataCite","host_organization_lineage":["https://openalex.org/I4210145204"],"host_organization_lineage_names":["DataCite"],"type":"metadata"},"license":null,"license_id":null,"version":null}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2306.11706","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[{"score":0.53,"display_name":"Peace, justice, and strong institutions","id":"https://metadata.un.org/sdg/16"}],"grants":[],"datasets":[],"versions":[],"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W8302103","https://openalex.org/W4381280689","https://openalex.org/W4242726756","https://openalex.org/W3213722473","https://openalex.org/W3128025644","https://openalex.org/W2847365777","https://openalex.org/W2787993192","https://openalex.org/W2372403409","https://openalex.org/W2355048207","https://openalex.org/W2158269427"],"abstract_inverted_index":{"The":[0],"ability":[1,91],"to":[2,12,22,92,94,125,187],"leverage":[3],"heterogeneous":[4],"robotic":[5,45,77],"experience":[6],"from":[7,73],"different":[8,157],"robots":[9],"and":[10,17,35,75,84,97,154,167],"tasks":[11,96],"quickly":[13],"master":[14],"novel":[15],"skills":[16,72],"embodiments":[18],"has":[19],"the":[20,90,111,145],"potential":[21],"transform":[23],"robot":[24,159],"learning.":[25],"Inspired":[26],"by":[27],"recent":[28],"advances":[29],"in":[30,152],"foundation":[31],"models":[32],"for":[33,44,110,128,138],"vision":[34],"language,":[36],"we":[37,88,165],"propose":[38],"a":[39,52,66,118,134],"multi-embodiment,":[40],"multi-task":[41],"generalist":[42],"agent":[43],"manipulation.":[46],"This":[47,63],"agent,":[48],"named":[49],"RoboCat,":[50,87],"is":[51],"visual":[53,61],"goal-conditioned":[54],"decision":[55],"transformer":[56],"capable":[57],"of":[58,69,82,177],"consuming":[59],"action-labelled":[60],"experience.":[62],"data":[64,127],"spans":[65],"large":[67],"repertoire":[68],"motor":[70],"control":[71],"simulated":[74],"real":[76,158],"arms":[78],"with":[79,148],"varying":[80],"sets":[81],"observations":[83],"actions.":[85],"With":[86],"demonstrate":[89],"generalise":[93],"new":[95,188],"robots,":[98],"both":[99,151],"zero-shot":[100],"as":[101,103,164],"well":[102],"through":[104],"adaptation":[105],"using":[106],"only":[107,174],"100-1000":[108],"examples":[109],"target":[112],"task.":[113],"We":[114,143,161],"also":[115,181],"show":[116],"how":[117],"trained":[119],"model":[120],"itself":[121],"can":[122],"be":[123],"used":[124],"generate":[126],"subsequent":[129],"training":[130,170],"iterations,":[131],"thus":[132],"providing":[133],"basic":[135],"building":[136],"block":[137],"an":[139],"autonomous":[140],"improvement":[141],"loop.":[142],"investigate":[144],"agent's":[146],"capabilities,":[147],"large-scale":[149],"evaluations":[150],"simulation":[153],"on":[155],"three":[156],"embodiments.":[160],"find":[162],"that":[163],"grow":[166],"diversify":[168],"its":[169],"data,":[171],"RoboCat":[172],"not":[173],"shows":[175],"signs":[176],"cross-task":[178],"transfer,":[179],"but":[180],"becomes":[182],"more":[183],"efficient":[184],"at":[185],"adapting":[186],"tasks.":[189]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4381586832","counts_by_year":[{"year":2023,"cited_by_count":2}],"updated_date":"2025-01-21T06:42:24.974698","created_date":"2023-06-22"}