{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2024,9,6]],"date-time":"2024-09-06T04:34:50Z","timestamp":1725597290733},"reference-count":71,"publisher":"Frontiers Media SA","license":[{"start":{"date-parts":[[2019,8,2]],"date-time":"2019-08-02T00:00:00Z","timestamp":1564704000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["frontiersin.org"],"crossmark-restriction":true},"short-container-title":["Front. Comput. Neurosci."],"DOI":"10.3389\/fncom.2019.00046","type":"journal-article","created":{"date-parts":[[2019,8,2]],"date-time":"2019-08-02T11:41:10Z","timestamp":1564746070000},"update-policy":"http:\/\/dx.doi.org\/10.3389\/crossmark-policy","source":"Crossref","is-referenced-by-count":8,"title":["A Closed-Loop Toolchain for Neural Network Simulations of Learning Autonomous Agents"],"prefix":"10.3389","volume":"13","author":[{"given":"Jakob","family":"Jordan","sequence":"first","affiliation":[]},{"given":"Philipp","family":"Weidel","sequence":"additional","affiliation":[]},{"given":"Abigail","family":"Morrison","sequence":"additional","affiliation":[]}],"member":"1965","published-online":{"date-parts":[[2019,8,2]]},"reference":[{"key":"B1","article-title":"Tensorflow: Large-scale machine learning on heterogeneous distributed systems","volume-title":"arXiv preprint arXiv:1603.04467","author":"Abadi","year":"2016"},{"key":"B2","doi-asserted-by":"publisher","first-page":"36","DOI":"10.3389\/fncom.2015.00036","article-title":"RM-SORN: a reward-modulated self-organizing recurrent neural network","volume":"9","author":"Aswolinskiy","year":"2015","journal-title":"Front. Comput. Neurosci."},{"key":"B3","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.neunet.2015.03.002","article-title":"A spiking neural network based on the basal ganglia functional anatomy","volume":"67","author":"Baladron","year":"2015","journal-title":"Neural Netw."},{"key":"B4","doi-asserted-by":"publisher","first-page":"48","DOI":"10.3389\/fninf.2013.00048","article-title":"Nengo: a python tool for building large-scale functional brain models","volume":"7","author":"Bekolay","year":"2013","journal-title":"Front. Neuroinform."},{"key":"B5","article-title":"The arcade learning environment: an evaluation platform for general agents","volume-title":"Twenty-Fourth International Joint Conference on Artificial Intelligence","author":"Bellemare","year":"2015"},{"key":"B6","first-page":"153","article-title":"Greedy layer-wise training of deep networks","volume":"19","author":"Bengio","year":"2007","journal-title":"Adv. Neural Inform. Process. Syst."},{"key":"B7","doi-asserted-by":"publisher","first-page":"1383","DOI":"10.4249\/scholarpedia.1383","article-title":"GENESIS (simulation environment)","volume":"2","author":"Bower","year":"2007","journal-title":"Scholarpedia"},{"key":"B8","article-title":"OpenAI Gym","volume-title":"ArXiv e-prints. arXiv preprint arXiv:1606.01540","author":"Brockman","year":"2016"},{"key":"B9","doi-asserted-by":"crossref","DOI":"10.1017\/CBO9780511541612","volume-title":"The NEURON Book","author":"Carnevale","year":"2006"},{"key":"B10","author":"Collobert","year":"2002","journal-title":"Torch: A Modular Machine Learning Software Library."},{"key":"B11","doi-asserted-by":"crossref","DOI":"10.1109\/CVPR.2009.5206848","article-title":"ImageNet: a large-scale hierarchical image database","volume-title":"CVPR09","author":"Deng","year":"2009"},{"key":"B12","doi-asserted-by":"publisher","first-page":"43","DOI":"10.1007\/s12021-010-9064-z","article-title":"Run-time interoperability between neuronal simulators based on the music framework","volume":"8","author":"Djurfeldt","year":"2010","journal-title":"Neuroinformatics"},{"key":"B13","doi-asserted-by":"publisher","first-page":"219","DOI":"10.1162\/089976600300015961","article-title":"Reinforcement learning in continuous time and space","volume":"12","author":"Doya","year":"2000","journal-title":"Neural Comput."},{"key":"B14","doi-asserted-by":"crossref","first-page":"46","DOI":"10.1109\/ICRA.2011.5980252","article-title":"Modular open robots simulation engine: Morse","volume-title":"2011 IEEE International Conference on Robotics and Automation","author":"Echeverria","year":"2011"},{"key":"B15","doi-asserted-by":"publisher","first-page":"12","DOI":"10.3389\/neuro.11.012.2008","article-title":"PyNEST: a convenient interface to the NEST simulator","volume":"2","author":"Eppler","year":"2009","journal-title":"Front. Neuroinform."},{"key":"B16","doi-asserted-by":"publisher","first-page":"2","DOI":"10.3389\/fnbot.2017.00002","article-title":"Connecting artificial brains to robots in a comprehensive simulation framework: the neurorobotics platform","volume":"11","author":"Falotico","year":"2017","journal-title":"Front. Neurorobot."},{"key":"B17","doi-asserted-by":"publisher","first-page":"13326","DOI":"10.1523\/JNEUROSCI.6249-09.2010","article-title":"Functional requirements for reward-modulated spike-timing-dependent plasticity","volume":"30","author":"Fr\u00e9maux","year":"2010","journal-title":"J. Neurosci."},{"key":"B18","doi-asserted-by":"publisher","first-page":"e1003024","DOI":"10.1371\/journal.pcbi.1003024","article-title":"Reinforcement learning using a continuous time actor-critic framework with spiking neurons","volume":"9","author":"Fr\u00e9maux","year":"2013","journal-title":"PLoS Comput. Biol."},{"key":"B19","doi-asserted-by":"publisher","first-page":"1529","DOI":"10.1523\/JNEUROSCI.2854-15.2016","article-title":"Goal-directed decision making with spiking neurons","volume":"36","author":"Friedrich","year":"2016","journal-title":"J. Neurosci."},{"key":"B20","doi-asserted-by":"publisher","first-page":"1450002","DOI":"10.1142\/S0129065714500026","article-title":"Code-specific learning rules improve action selection by populations of spiking neurons","volume":"24","author":"Friedrich","year":"2014","journal-title":"Int. J. Neural Syst."},{"key":"B21","doi-asserted-by":"publisher","first-page":"379","DOI":"10.1126\/science.1181936","article-title":"How good are neuron models?","volume":"326","author":"Gerstner","year":"2009","journal-title":"Science"},{"key":"B22","doi-asserted-by":"publisher","first-page":"1430","DOI":"10.4249\/scholarpedia.1430","article-title":"NEST (NEural Simulation Tool)","volume":"2","author":"Gewaltig","year":"2007","journal-title":"Scholarpedia"},{"key":"B23","doi-asserted-by":"publisher","first-page":"192","DOI":"10.3389\/neuro.01.026.2009","article-title":"The brian simulator","volume":"3","author":"Goodman","year":"2009","journal-title":"Front. Neurosci."},{"key":"B24","doi-asserted-by":"publisher","first-page":"34","DOI":"10.3389\/fninf.2017.00034","article-title":"Integration of continuous-time dynamics in a spiking neural network simulator","volume":"11","author":"Hahne","year":"2017","journal-title":"Front. Neuroinform."},{"key":"B25","doi-asserted-by":"publisher","first-page":"89","DOI":"10.3389\/fninf.2018.00089","article-title":"Bindsnet: a machine learning-oriented spiking neural networks library in python","volume":"12","author":"Hazan","year":"2018","journal-title":"Front. Neuroinform."},{"key":"B26","volume-title":"ZeroMQ: Messaging for Many Applications","author":"Hintjens","year":"2013"},{"key":"B27","doi-asserted-by":"publisher","first-page":"82","DOI":"10.1109\/MSP.2012.2205597","article-title":"Deep neural networks for acoustic modeling in speech recognition: the shared views of four research groups","volume":"29","author":"Hinton","year":"2012","journal-title":"IEEE Signal Process. Mag."},{"key":"B28","doi-asserted-by":"publisher","first-page":"504","DOI":"10.1126\/science.1127647","article-title":"Reducing the dimensionality of data with neural networks","volume":"313","author":"Hinton","year":"2006","journal-title":"Science"},{"key":"B29","article-title":"Improving neural networks by preventing co-adaptation of feature detectors","volume-title":"arXiv preprint arXiv:1207.0580","author":"Hinton","year":"2012"},{"key":"B30","unstructured":"Project Website2014"},{"key":"B31","doi-asserted-by":"publisher","first-page":"2443","DOI":"10.1093\/cercor\/bhl152","article-title":"Solving the distal reward problem through linkage of STDP and dopamine signaling","volume":"17","author":"Izhikevich","year":"2007","journal-title":"Cereb. Cortex"},{"key":"B32","article-title":"Theano: a CPU and GPU math expression compiler","volume-title":"Proceedings of the Python for Scientific Computing Conference (SciPy)","author":"James","year":"2010"},{"key":"B33","doi-asserted-by":"crossref","DOI":"10.1145\/2647868.2654889","article-title":"Caffe: convolutional architecture for fast feature embedding","volume-title":"arXiv preprint arXiv:1408.5093","author":"Jia","year":"2014"},{"key":"B34","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1109\/IJCNN.2012.6252834","article-title":"Learning from positive and negative rewards in a spiking neural network model of basal ganglia","volume-title":"The 2012 International Joint Conference on Neural Networks (IJCNN)","author":"Jitsev","year":"2012"},{"key":"B35","doi-asserted-by":"publisher","first-page":"2","DOI":"10.3389\/fninf.2018.00034","article-title":"Extremely scalable spiking neural network simulation code: from laptops to exascale computers","volume":"12","author":"Jordan","year":"2018","journal-title":"Front. Neuroinform."},{"key":"B36","doi-asserted-by":"publisher","first-page":"e1005145","DOI":"10.1371\/journal.pcbi.1005145","article-title":"Forgetting in reinforcement learning links sustained dopamine signals to motivation","volume":"12","author":"Kato","year":"2016","journal-title":"PLoS Comput. Biol."},{"key":"B37","article-title":"Adam: a method for stochastic optimization","volume-title":"arXiv preprint arXiv:1412.6980","author":"Kingma","year":"2014"},{"key":"B38","doi-asserted-by":"crossref","first-page":"2149","DOI":"10.1109\/IROS.2004.1389727","article-title":"Design and use paradigms for gazebo, an open-source multi-robot simulator","volume-title":"2004 IEEE\/RSJ International Conference on Intelligent Robots and Systems (IROS)(IEEE Cat. No. 04CH37566), Vol. 3","author":"Koenig","year":"2004"},{"key":"B39","volume-title":"Learning Multiple Layers of Features From Tiny Images.","author":"Krizhevsky","year":"2009"},{"key":"B40","first-page":"1097","article-title":"Imagenet classification with deep convolutional neural networks","author":"Krizhevsky","year":"2012","journal-title":"Advances in Neural Information Processing Systems"},{"key":"B41","doi-asserted-by":"publisher","first-page":"78","DOI":"10.3389\/fninf.2014.00078","article-title":"Spiking network simulation code for petascale computers","volume":"8","author":"Kunkel","year":"2014","journal-title":"Front. Neuroinform."},{"key":"B42","unstructured":"LeCunY. The MNIST Database of Handwritten Digits1998"},{"key":"B43","volume-title":"Reinforcement Learning for Robots Using Neural Networks.","author":"Lin","year":"1993"},{"key":"B44","doi-asserted-by":"publisher","first-page":"5","DOI":"10.5772\/5618","article-title":"Cyberbotics Ltd. Webots\u2122 : professional mobile robot simulation","volume":"1","author":"Michel","year":"2004","journal-title":"Int. J. Adv. Robot. Syst."},{"key":"B45","first-page":"1928","article-title":"Asynchronous methods for deep reinforcement learning","author":"Mnih","year":"2016","journal-title":"Int. Conf. Mach. Learn."},{"key":"B46","article-title":"Playing atari with deep reinforcement learning","volume-title":"arXiv preprint arXiv:1312.5602","author":"Mnih","year":"2013"},{"key":"B47","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1038\/nature14236","article-title":"Human-level control through deep reinforcement learning","volume":"518","author":"Mnih","year":"2015","journal-title":"Nature"},{"key":"B48","doi-asserted-by":"publisher","first-page":"125","DOI":"10.3902\/jnns.22.125","article-title":"Real-time utilization of system-scale neuroscience models","volume":"22","author":"Moren","year":"2015","journal-title":"J. Jap. Neural Netw. Soc."},{"key":"B49","doi-asserted-by":"publisher","first-page":"69","DOI":"10.1146\/annurev.neuro.31.061307.090723","article-title":"Place cells, grid cells, and the brain's spatial representation system","volume":"31","author":"Moser","year":"2008","journal-title":"Annu. Rev. Neurosci."},{"key":"B50","article-title":"Automatic Differentiation in Pytorch","author":"Paszke","year":"2017"},{"key":"B51","doi-asserted-by":"publisher","first-page":"e1001133","DOI":"10.1371\/journal.pcbi.1001133","article-title":"An imperfect dopaminergic error signal can drive temporal-difference learning","volume":"7","author":"Potjans","year":"2011","journal-title":"PLoS Comput. Biol."},{"key":"B52","doi-asserted-by":"publisher","first-page":"301","DOI":"10.1162\/neco.2008.08-07-593","article-title":"A spiking neural network model of an actor-critic learning agent","volume":"21","author":"Potjans","year":"2009","journal-title":"Neural Comput."},{"key":"B53","article-title":"ROS: an open-source Robot Operating System","volume-title":"ICRA Workshop on Open Source Software, Vol. 3","author":"Quigley","year":"2009"},{"key":"B54","doi-asserted-by":"crossref","first-page":"873","DOI":"10.1145\/1553374.1553486","article-title":"Large-scale deep unsupervised learning using graphics processors","volume-title":"Proceedings of the 26th Annual International Conference on Machine Learning","author":"Raina","year":"2009"},{"key":"B55","article-title":"A neural model of hierarchical reinforcement learning","volume-title":"CogSci","author":"Rasmussen","year":"2014"},{"key":"B56","doi-asserted-by":"publisher","first-page":"e1004060","DOI":"10.1371\/journal.pcbi.1004060","article-title":"How attention can create synaptic tags for the learning of working memories in sequential tasks","volume":"11","author":"Rombouts","year":"2015","journal-title":"PLoS Comput. Biol."},{"key":"B57","doi-asserted-by":"publisher","first-page":"21142","DOI":"10.1038\/srep21142","article-title":"Recurrent spiking networks solve planning tasks","volume":"6","author":"Rueckert","year":"2016","journal-title":"Sci. Rep."},{"key":"B58","doi-asserted-by":"publisher","first-page":"211","DOI":"10.1007\/s11263-015-0816-y","article-title":"ImageNet large scale visual recognition challenge","volume":"115","author":"Russakovsky","year":"2015","journal-title":"Int. J. Comput. Vis."},{"key":"B59","doi-asserted-by":"publisher","first-page":"10","DOI":"10.3389\/fninf.2013.00010","article-title":"The virtual brain: a simulator of primate brain network dynamics","volume":"7","author":"Sanz Leon","year":"2013","journal-title":"Front. Neuroinform."},{"key":"B60","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.3218477","author":"Sheik","year":"2019","journal-title":"SINABS - A Spiking Deep Neural Network Inference Emulator"},{"key":"B61","doi-asserted-by":"publisher","first-page":"484","DOI":"10.1038\/nature16961","article-title":"Mastering the game of go with deep neural networks and tree search","volume":"529","author":"Silver","year":"2016","journal-title":"Nature"},{"key":"B62","volume-title":"Reinforcement Learning: An Introduction","author":"Sutton","year":"1998"},{"key":"B63","doi-asserted-by":"crossref","first-page":"58","DOI":"10.1145\/203330.203343","article-title":"Temporal difference learning and TD-gammon","volume":"38","author":"Tesauro","year":"1995","journal-title":"Commun. ACM"},{"key":"B64","doi-asserted-by":"crossref","first-page":"5026","DOI":"10.1109\/IROS.2012.6386109","article-title":"Mujoco: a physics engine for model-based control","author":"Todorov","year":"2012","journal-title":"2012 IEEE\/RSJ International Conference on Intelligent Robots and Systems (IROS)"},{"key":"B65","doi-asserted-by":"publisher","first-page":"250","DOI":"10.1038\/nn.2264","article-title":"Reinforcement learning in populations of spiking neurons","volume":"12","author":"Urbanczik","year":"2009","journal-title":"Nat. Neurosci."},{"key":"B66","doi-asserted-by":"publisher","first-page":"e1000586","DOI":"10.1371\/journal.pcbi.1000586","article-title":"Spike-based reinforcement learning in continuous state and action space: when policy gradient methods fail","volume":"5","author":"Vasilaki","year":"2009","journal-title":"PLoS Comput. Biol."},{"key":"B67","doi-asserted-by":"crossref","first-page":"279","DOI":"10.1007\/BF00992698","article-title":"Q-learning","volume":"8","author":"Watkins","year":"1992","journal-title":"Mach. Learn."},{"key":"B68","unstructured":"WatkinsC. J. C. H. University of CambridgeLearning from delayed rewards1989"},{"key":"B69","doi-asserted-by":"publisher","first-page":"31","DOI":"10.3389\/fninf.2016.00031","article-title":"Closed loop interactions between spiking neural network and robotic simulators based on music and ros","volume":"10","author":"Weidel","year":"2016","journal-title":"Front. Neuroinform."},{"key":"B70","doi-asserted-by":"publisher","first-page":"15","DOI":"10.3389\/neuro.11.015.2009","article-title":"STEPS: modeling and simulating complex reaction-diffusion systems with Python","volume":"3","author":"Wils","year":"2009","journal-title":"Front. Neuroinform."},{"key":"B71","doi-asserted-by":"publisher","first-page":"625","DOI":"10.3758\/BF03196322","article-title":"Six views of embodied cognition","volume":"9","author":"Wilson","year":"2002","journal-title":"Psychon. Bull. Rev."}],"container-title":["Frontiers in Computational Neuroscience"],"original-title":[],"link":[{"URL":"https:\/\/www.frontiersin.org\/article\/10.3389\/fncom.2019.00046\/full","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,1,15]],"date-time":"2021-01-15T21:57:54Z","timestamp":1610747874000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.frontiersin.org\/article\/10.3389\/fncom.2019.00046\/full"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,8,2]]},"references-count":71,"alternative-id":["10.3389\/fncom.2019.00046"],"URL":"https:\/\/doi.org\/10.3389\/fncom.2019.00046","relation":{},"ISSN":["1662-5188"],"issn-type":[{"value":"1662-5188","type":"electronic"}],"subject":[],"published":{"date-parts":[[2019,8,2]]}}}