{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2024,7,22]],"date-time":"2024-07-22T12:33:54Z","timestamp":1721651634961},"reference-count":44,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2023,3,30]],"date-time":"2023-03-30T00:00:00Z","timestamp":1680134400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,3,30]],"date-time":"2023-03-30T00:00:00Z","timestamp":1680134400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100000038","name":"NSERC","doi-asserted-by":"crossref","id":[{"id":"10.13039\/501100000038","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100003151","name":"Fonds de recherche du Qu\u00e9bec - Nature et technologies","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100003151","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Software Qual J"],"published-print":{"date-parts":[[2023,12]]},"DOI":"10.1007\/s11219-023-09621-9","type":"journal-article","created":{"date-parts":[[2023,3,30]],"date-time":"2023-03-30T08:11:37Z","timestamp":1680163897000},"page":"1065-1119","update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":5,"title":["Machine learning application development: practitioners\u2019 insights"],"prefix":"10.1007","volume":"31","author":[{"ORCID":"http:\/\/orcid.org\/0000-0002-5677-5927","authenticated-orcid":false,"given":"Md Saidur","family":"Rahman","sequence":"first","affiliation":[]},{"given":"Foutse","family":"Khomh","sequence":"additional","affiliation":[]},{"given":"Alaleh","family":"Hamidi","sequence":"additional","affiliation":[]},{"given":"Jinghui","family":"Cheng","sequence":"additional","affiliation":[]},{"given":"Giuliano","family":"Antoniol","sequence":"additional","affiliation":[]},{"given":"Hironori","family":"Washizaki","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2023,3,30]]},"reference":[{"key":"9621_CR1","volume-title":"Software Engineering for Machine Learning: A Case Study","author":"S Amershi","year":"2019","unstructured":"Amershi, S., Begel, A., Bird, C., DeLine, R., Gall, H., Kamar, E., Nagappan, N., Nushi, B., & Zimmermann, T. (2019). Software Engineering for Machine Learning: A Case Study. ICSE: In Proc."},{"key":"9621_CR2","unstructured":"Anderson, D. J. (2010). Kanban: successful evolutionary change for your technology business. Blue Hole Press."},{"key":"9621_CR3","unstructured":"Appendix. (2020). Replication package with survey data and results. Available online at: https:\/\/preview.tinyurl.com\/ydaj9jh9"},{"key":"9621_CR4","doi-asserted-by":"crossref","unstructured":"Bangash, A. A., Sahar, H., Chowdhury, S., Wong, A. W., Hindle, A., & Ali, K. (2019). What do developers know about machine learning: a study of ML discussions on StackOverflow.","DOI":"10.1109\/MSR.2019.00052"},{"key":"9621_CR5","doi-asserted-by":"crossref","unstructured":"Belani, H., Vukovic, M., & Car, Z. (2019). Requirements Engineering Challenges in Building AI-Based Complex Systems. arXiv preprint arXiv:1908.11791","DOI":"10.1109\/REW.2019.00051"},{"key":"9621_CR6","doi-asserted-by":"publisher","unstructured":"Braiek, H. B., & Khomh, F. (2020). On Testing Machine Learning Programs. Journal of Systems and Software, 164, 110542, ISSN 0164\u20131212. https:\/\/doi.org\/10.1016\/j.jss.2020.110542","DOI":"10.1016\/j.jss.2020.110542"},{"key":"9621_CR7","unstructured":"Charmaz, K. (2006). Constructing Grounded Theory: A Practical Guide Through Qualitative Analysis. SAGE Publications."},{"key":"9621_CR8","doi-asserted-by":"crossref","unstructured":"Chawla, N. V., Bowyer, K. W., Hall, L. O., & Kegelmeyer, W. P.\u00a0 (2002). SMOTE: Synthetic Minority Over-sampling Technique. Journal of Artificial Intelligence Research, 16, 321\u2013357.","DOI":"10.1613\/jair.953"},{"key":"9621_CR9","doi-asserted-by":"crossref","unstructured":"Felderer, M., & Ramler, R. (2021). Quality Assurance for AI-Based Systems: Overview and Challenges In: Winkler, D., Biffl, S., Mendez, D., Wimmer, M., Bergsmann, J. (eds) Software Quality: Future Perspectives on Software Engineering Quality. SWQD, pp.33\u201342.","DOI":"10.1007\/978-3-030-65854-0_3"},{"key":"9621_CR10","doi-asserted-by":"crossref","unstructured":"Fink, A. (2003) The survey handbook. Sage.","DOI":"10.4135\/9781412986328"},{"key":"9621_CR11","volume-title":"Testing MCMC code","author":"RB Grosse","year":"2014","unstructured":"Grosse, R. B., & Duvenaud, D. K. (2014). Testing MCMC code. NIPS: In Proc."},{"key":"9621_CR12","doi-asserted-by":"crossref","unstructured":"Guo, Q., Chen, S., Xie, X., Ma, L., Hu, Q., Liu, H., Liu, Y., Zhao, J., & Li, X. (2019). An Empirical Study towards Characterizing Deep Learning Development and Deployment across Different Frameworks and Platforms. arXiv preprint arXiv:1909.06727","DOI":"10.1109\/ASE.2019.00080"},{"key":"9621_CR13","unstructured":"He, H., Bai, Y., Garcia, E. A., & Li, S. (2008). ADASYN: Adaptive synthetic sampling approach for imbalanced learning, 2008 IEEE International Joint Conference on Neural Networks, Hong Kong, 2008, pp. 1322-1328."},{"key":"9621_CR14","doi-asserted-by":"crossref","unstructured":"Huang, S., Liu, E. -H., Hui, Z. -W., Tang, S. -Q., & Zhang, S. -J. (2018). Challenges of Testing Machine Learning Applications arXiv:1806","DOI":"10.23940\/ijpe.18.06.p18.12751282"},{"key":"9621_CR15","doi-asserted-by":"crossref","unstructured":"Ishikawa, F., & Yoshioka, N. (2019). How do engineers perceive difficulties in engineering of machine-learning systems? questionnaire survey. In Proceedings of the Joint 7th International Workshop on Conducting Empirical Studies in Industry and 6th International Workshop on Software Engineering Research and Industrial Practice (CESSER-IP \u201919). IEEE Press, 2\u20139.","DOI":"10.1109\/CESSER-IP.2019.00009"},{"key":"9621_CR16","unstructured":"Islam, Md. J., Nguyen, H. A., Pan, R., & Rajan, H. (2019). What Do Developers Ask About ML Libraries? A Large-scale Study Using Stack Overflow. arXiv: 1906.11940v1"},{"key":"9621_CR17","unstructured":"Khomh,\u00a0F., & Antoniol, G. (2018). Bringing AI and machine learning data science into operation., Redhat Blog. Available at: https:\/\/www.redhat.com\/en\/blog\/bringing-ai-and-machine-learning-data-science-operation"},{"issue":"5","key":"9621_CR18","doi-asserted-by":"publisher","first-page":"81","DOI":"10.1109\/MS.2018.3571224","volume":"35","author":"F Khomh","year":"2018","unstructured":"Khomh, F., Adams, B., Cheng, J., Fokaefs, M., & Antoniol, G. (2018). Software Engineering for Machine-Learning Applications: The Road Ahead. IEEE Software, 35(5), 81\u201384.","journal-title":"IEEE Software"},{"key":"9621_CR19","unstructured":"Kriens, P., & Verbelen, T. (2019). Software Engineering Practices for Machine Learning. arXiv:1906.10366"},{"key":"9621_CR20","doi-asserted-by":"crossref","unstructured":"Ma, L., Juefei-Xu, F., Xue, M., Li, B., Li, L., Liu, Y., & Zhao, J. (2019). DeepCT: Tomographic Combinatorial Testing for Deep Learning Systems. In 2019 IEEE 26th International Conference on Software Analysis, Evolution and Reengineering (SANER). IEEE, 614\u2013618.","DOI":"10.1109\/SANER.2019.8668044"},{"key":"9621_CR21","doi-asserted-by":"crossref","unstructured":"Ma, L., Juefei-Xu, F., Zhang, F., Sun, J., Xue, M., Li, B., Chen, C., Su, T., Li, L., Liu, Y., et al. (2018a). Deepgauge: Multi-granularity testing criteria for deep learning systems. In Proceedings of the 33rd ACM\/IEEE International Conference on Automated Software Engineering. ACM, 120\u2013131.","DOI":"10.1145\/3238147.3238202"},{"key":"9621_CR22","doi-asserted-by":"crossref","unstructured":"Ma, L., Zhang, F., Sun, J., Xue, M., Li, B., Juefei-Xu, F., Xie, C., Li, L., Liu, Y., Zhao, J., et al. (2018b). Deepmutation: Mutation testing of deep learning systems. In 2018 IEEE 29th International Symposium on Software Reliability Engineering (ISSRE). IEEE, 100\u2013111.","DOI":"10.1109\/ISSRE.2018.00021"},{"key":"9621_CR23","doi-asserted-by":"crossref","unstructured":"Marijan, D., & Gotlieb, A. (2020). Software testing for machine learning. Proceedings of the AAAI Conference on Artificial Intelligence, 34.","DOI":"10.1609\/aaai.v34i09.7084"},{"key":"9621_CR24","doi-asserted-by":"crossref","unstructured":"Marijan, D., Gotlieb, A., & Ahuja M. K. (2019). Challenges of Testing Machine Learning Based Systems.","DOI":"10.1109\/AITest.2019.00010"},{"key":"9621_CR25","doi-asserted-by":"crossref","unstructured":"Nguyen-Duc, A., Sundb\u00f8, I., Nascimento, E., Conte, T., Ahmed, I., & Abrahamsson, P. (2020). A Multiple Case Study of Artificial Intelligent System Development in Industry. In Proceedings of the Evaluation and Assessment in Software Engineering (EASE \u201920), pp. 1\u201310.","DOI":"10.1145\/3383219.3383220"},{"key":"9621_CR26","doi-asserted-by":"crossref","unstructured":"Pei, K., Cao, Y., Yang, J., & Jana S. (2017). DeepXplore: Automated Whitebox Testing of Deep Learning Systems, In Proc. Symposium on Operating Systems Principles (SOSP \u201917). pp.1-18.","DOI":"10.1145\/3132747.3132785"},{"key":"9621_CR27","volume-title":"Lean Software Development: An Agile Toolkit: An Agile Toolkit","author":"M Poppendieck","year":"2003","unstructured":"Poppendieck, M., & Poppendieck, T. (2003). Lean Software Development: An Agile Toolkit: An Agile Toolkit. Addison-Wesley."},{"key":"9621_CR28","unstructured":"Renggli, C., et al. (2019). Continuous integration of machine learning models with ease. ML\/CI: Towards a rigorous yet practical treatment. arXiv:1903.00278"},{"key":"9621_CR29","unstructured":"Responsible AI Practices. (2020). Google AI. Available at: https:\/\/ai.google\/education\/responsible-ai-practices"},{"key":"9621_CR30","doi-asserted-by":"crossref","unstructured":"Sandberg,\u00a0A. B., & Crnkovic, I. (2017). Meeting Industry-Academia Research Collaboration Challenges with Agile Methodologies. 2017 IEEE\/ACM 39th International Conference on Software Engineering: Software Engineering in Practice Track (ICSE-SEIP), Buenos Aires, pp. 73-82.","DOI":"10.1109\/ICSE-SEIP.2017.20"},{"key":"9621_CR31","volume-title":"On Challenges in Machine Learning Model Management","author":"S Schelter","year":"2018","unstructured":"Schelter, S., Biessmann, F., Januschowski, T., Salinas, D., Seufert, S., & Szarvas, G. (2018). On Challenges in Machine Learning Model Management. Committee on Data Engineering: Bulletin of the IEEE CS Tech."},{"key":"9621_CR32","first-page":"117","volume-title":"Scrum development process","author":"Ken Schwaber","year":"1997","unstructured":"Schwaber, Ken. (1997). Scrum development process (pp. 117\u2013134). London: Business object design and implementation. Springer."},{"key":"9621_CR33","unstructured":"Sculley, D., Holt, G., Golovin, D., Davydov, E., Phillips, T., Ebner, D., Chaudhary, V., Young, M., Crespo, J., & Dennison, D. (2015). Hidden technical debt in machine learning systems. In Proc NIPS. pp.\u00a02503\u20132511."},{"key":"9621_CR34","doi-asserted-by":"crossref","unstructured":"Stol, K., Ralph, P., & Fitzgerald, B. (2016). Grounded Theory in Software Engineering Research: A Critical Review and Guidelines. 2016 IEEE\/ACM 38th International Conference on Software Engineering (ICSE), Austin, TX, pp. 120-131.","DOI":"10.1145\/2884781.2884833"},{"key":"9621_CR35","unstructured":"Storcheus, D., Rostamizadeh, A., & Kumar, S. (2015). A survey of modern questions and challenges in feature extraction. In Proc IWFE: Modern Questions and Challenges, NIPS. 1-18."},{"key":"9621_CR36","doi-asserted-by":"crossref","unstructured":"Sun, Y., Wu, M., Ruan, W., Huang, X., Kwiatkowska, M., & Kroening, D. (2018). Concolic testing for deep neural networks. In Proceedings of the 33rd ACM\/IEEE International Conference on Automated Software Engineering. ACM, 109\u2013119.","DOI":"10.1145\/3238147.3238172"},{"key":"9621_CR37","unstructured":"van der Maaten, L., & Hinton, G. (2008). Visualizing data using t-SNE. Journal of Machine Learning Research, 9, 2579-2605."},{"key":"9621_CR38","doi-asserted-by":"crossref","unstructured":"Vogelsang, A., & Borg, M. (2019). Requirements Engineering for Machine Learning: Perspectives from Data Scientists. In 2019 IEEE 27th International Requirements Engineering Conference Workshops (REW), pp. 245-251. IEEE.","DOI":"10.1109\/REW.2019.00050"},{"key":"9621_CR39","doi-asserted-by":"crossref","unstructured":"Wan, Z., Xia, X., Lo, D., & Murphy, G. C. (2019). How does Machine Learning Change Software Development Practices? IEEE Transactions on Software Engineering.","DOI":"10.1109\/TSE.2019.2937083"},{"key":"9621_CR40","doi-asserted-by":"crossref","unstructured":"Washizaki, H., Uchida, H., Khomh, F., & Gu\u00e9h\u00e9neuc, Y. (2019). Studying Software Engineering Patterns for Designing Machine Learning Systems. 2019 10th International Workshop on Empirical Software Engineering in Practice (IWESEP), Tokyo, Japan, pp. 49\u2013495.","DOI":"10.1109\/IWESEP49350.2019.00017"},{"key":"9621_CR41","unstructured":"Zhang, J. M., Harman, M., Ma, L., & Liu, Y. (2019a). Machine Learning Testing: Survey, Landscapes and Horizons. arXiv preprint arXiv:1906.10742"},{"key":"9621_CR42","doi-asserted-by":"crossref","unstructured":"Zhang, T., Gao, C., Ma, L., Lyu, M., & Kim, M. (2019b). An Empirical Study of Common Challenges in Developing Deep Learning Applications. 2019 IEEE 30th International Symposium on Software Reliability Engineering (ISSRE), pp. 104-115","DOI":"10.1109\/ISSRE.2019.00020"},{"key":"9621_CR43","unstructured":"Zhang, X., et al. (2019c). Software Engineering Practice in the Development of Deep Learning Applications. arXiv preprint arXiv:1910.03156"},{"key":"9621_CR44","unstructured":"Zinkevich, M. (2018). Rules of machine learning: Best practices for ML engineering, Google guide on machine learning. Available at: https:\/\/developers.google.com\/machine-learning\/guides\/rules-of-ml\/"}],"container-title":["Software Quality Journal"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11219-023-09621-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11219-023-09621-9\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11219-023-09621-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,11,10]],"date-time":"2023-11-10T15:13:32Z","timestamp":1699629212000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11219-023-09621-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,3,30]]},"references-count":44,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2023,12]]}},"alternative-id":["9621"],"URL":"https:\/\/doi.org\/10.1007\/s11219-023-09621-9","relation":{},"ISSN":["0963-9314","1573-1367"],"issn-type":[{"value":"0963-9314","type":"print"},{"value":"1573-1367","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,3,30]]},"assertion":[{"value":"20 February 2023","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"30 March 2023","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no competing interests.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}]}}