{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2024,9,6]],"date-time":"2024-09-06T02:48:21Z","timestamp":1725590901941},"reference-count":42,"publisher":"IEEE","license":[{"start":{"date-parts":[[2022,5,1]],"date-time":"2022-05-01T00:00:00Z","timestamp":1651363200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2022,5,1]],"date-time":"2022-05-01T00:00:00Z","timestamp":1651363200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022,5]]},"DOI":"10.1109\/icse-seip55303.2022.9793981","type":"proceedings-article","created":{"date-parts":[[2022,6,17]],"date-time":"2022-06-17T19:35:14Z","timestamp":1655494514000},"page":"263-272","source":"Crossref","is-referenced-by-count":0,"title":["Testing Machine Learning Systems in Industry: An Empirical Study"],"prefix":"10.1109","author":[{"given":"Shuyue","family":"Li","sequence":"first","affiliation":[{"name":"Xi'an Jiaotong University,Xi'an, Shaanxi,China"}]},{"given":"Jiaqi","family":"Guo","sequence":"additional","affiliation":[{"name":"Xi'an Jiaotong University,Xi'an, Shaanxi,China"}]},{"given":"Jian-Guang","family":"Lou","sequence":"additional","affiliation":[{"name":"Xi'an Jiaotong University,Xi'an, Shaanxi,China"}]},{"given":"Ming","family":"Fan","sequence":"additional","affiliation":[{"name":"Xi'an Jiaotong University,Xi'an, Shaanxi,China"}]},{"given":"Ting","family":"Liu","sequence":"additional","affiliation":[{"name":"Xi'an Jiaotong University,Xi'an, Shaanxi,China"}]},{"given":"Dongmei","family":"Zhang","sequence":"additional","affiliation":[{"name":"Xi'an Jiaotong University,Xi'an, Shaanxi,China"}]}],"member":"263","reference":[{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330579"},{"journal-title":"IEEE Transactions on Software Engineering","article-title":"How does Machine Learning Change Software Development Practices?","year":"2019","author":"wan","key":"ref38"},{"journal-title":"Adoption and Effects of Software Engineering Best Practices in Machine Learning","year":"2020","author":"serban","key":"ref33"},{"journal-title":"Proceedings of the 28th International Conference on Neural Information Processing Systems","article-title":"Hidden Technical Debt in Machine Learning Systems","year":"0","author":"sculley","key":"ref32"},{"journal-title":"Machine Learning The High Interest Credit Card of Technical Debt","year":"2014","author":"sculley","key":"ref31"},{"journal-title":"Assuring the machine learning lifecycle Desiderata methods and challenges","year":"2019","author":"rob","key":"ref30"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-57959-7"},{"journal-title":"Attention is all you need","year":"2017","author":"vaswani","key":"ref36"},{"journal-title":"A\/B Testing The Most Powerful Way to Turn Clicks Into Customers","year":"2013","author":"siroker","key":"ref35"},{"journal-title":"Active Learning Literature Survey","year":"2009","author":"settles","key":"ref34"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1145\/1287624.1287646"},{"journal-title":"Interpreting and evaluating neural network robustness","year":"2019","author":"yu","key":"ref40"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/VLHCC.2016.7739680"},{"journal-title":"Alexa","year":"0","key":"ref12"},{"journal-title":"What do developers ask about ml libraries? a large-scale study using stack overflow","year":"2019","author":"johirul islam","key":"ref13"},{"journal-title":"A convolutional Neural Network for Modelling Sentences","year":"2014","author":"kalchbrenner","key":"ref14"},{"journal-title":"Personal Opinion Surveys","first-page":"63","year":"2008","author":"kitchenham","key":"ref15"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177729694"},{"journal-title":"Advances in Neural IInformation Processing Systems","article-title":"Counterfactual Fairness","year":"2017","author":"kusner","key":"ref17"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1145\/2884781.2884874"},{"key":"ref19","first-page":"120","article-title":"DeepGauge: Multigranularity testing criteria for deep learning systems","author":"ma","year":"0","journal-title":"Proceedings of the 33rd ACM\/IEEE International Conference on Automated Software Engineering"},{"journal-title":"The prototype","year":"0","key":"ref28"},{"journal-title":"Metrics and methods for robustness evaluation of neural networks with generative models","year":"2020","author":"buzhinsky","key":"ref4"},{"key":"ref27","first-page":"771","article-title":"Problems and Opportunities in Training Deep Learning Software Systems: An Analysis of Variance","author":"viet pham","year":"0","journal-title":"Proceedings of the 35th ACM\/IEEE International Conference on Automated Software Engineering"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2017.8258038"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338954"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4419-8339-8_13"},{"journal-title":"Proceedings annual meeting of the Association for Computational Linguistics","article-title":"Beyond Accuracy: Behavioral Testing of NLP Models with CheckList","year":"0","author":"tulio ribeiro","key":"ref29"},{"journal-title":"Google Translate","year":"0","key":"ref8"},{"journal-title":"Rules of Machine Learning","year":"0","key":"ref7"},{"key":"ref2","first-page":"2621","article-title":"Measuring Neural Net Robustness with Constraints","author":"bastani","year":"0","journal-title":"Proceedings of the 30th International Conference on Neural Information Processing Systems"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"journal-title":"Monkey","year":"0","key":"ref9"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1016\/S0079-7421(08)60536-8"},{"journal-title":"NVIDIADIGITS","year":"0","key":"ref22"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2010-343"},{"journal-title":"Online Questionnaire","year":"0","key":"ref24"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2018.2870414"},{"journal-title":"Incident Database","year":"0","key":"ref23"},{"journal-title":"IEEE Transactions on Software Engineering","article-title":"Machine Learning Testing: Survey, Landscapes and Horizons","year":"2020","author":"zhang","key":"ref41"},{"journal-title":"Software Testing and Analysis Process Principles and Techniques","year":"2008","author":"pezze","key":"ref26"},{"key":"ref25","first-page":"1","article-title":"Deep-Xplore: Automated Whitebox Testing of Deep Learning Systems","author":"pei","year":"0","journal-title":"Proc Symposium Operating Systems Principles"}],"event":{"name":"2022 IEEE\/ACM 44th International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP)","start":{"date-parts":[[2022,5,22]]},"location":"Pittsburgh, PA, USA","end":{"date-parts":[[2022,5,24]]}},"container-title":["2022 IEEE\/ACM 44th International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/9793838\/9793543\/09793981.pdf?arnumber=9793981","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,7,11]],"date-time":"2022-07-11T20:04:30Z","timestamp":1657569870000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9793981\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,5]]},"references-count":42,"URL":"https:\/\/doi.org\/10.1109\/icse-seip55303.2022.9793981","relation":{},"subject":[],"published":{"date-parts":[[2022,5]]}}}