{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2024,7,8]],"date-time":"2024-07-08T10:48:17Z","timestamp":1720435697251},"reference-count":90,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2023,7,1]],"date-time":"2023-07-01T00:00:00Z","timestamp":1688169600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2023,7,1]],"date-time":"2023-07-01T00:00:00Z","timestamp":1688169600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-017"},{"start":{"date-parts":[[2023,7,1]],"date-time":"2023-07-01T00:00:00Z","timestamp":1688169600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"},{"start":{"date-parts":[[2023,7,1]],"date-time":"2023-07-01T00:00:00Z","timestamp":1688169600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-012"},{"start":{"date-parts":[[2023,7,1]],"date-time":"2023-07-01T00:00:00Z","timestamp":1688169600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2023,7,1]],"date-time":"2023-07-01T00:00:00Z","timestamp":1688169600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-004"}],"funder":[{"DOI":"10.13039\/501100002322","name":"Coordena\u00e7\u00e3o de Aperfei\u00e7oamento de Pessoal de N\u00edvel Superior","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100002322","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003593","name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100003593","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Science of Computer Programming"],"published-print":{"date-parts":[[2023,7]]},"DOI":"10.1016\/j.scico.2023.102961","type":"journal-article","created":{"date-parts":[[2023,5,12]],"date-time":"2023-05-12T15:21:54Z","timestamp":1683904914000},"page":"102961","update-policy":"http:\/\/dx.doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":0,"special_numbering":"C","title":["Characterization of continuous experimentation in software engineering: Expressions, models, and strategies"],"prefix":"10.1016","volume":"229","author":[{"ORCID":"http:\/\/orcid.org\/0000-0002-0637-3479","authenticated-orcid":false,"given":"Vladimir M.","family":"Erthal","sequence":"first","affiliation":[]},{"given":"Bruno P.","family":"de Souza","sequence":"additional","affiliation":[]},{"given":"Paulo S\u00e9rgio M.","family":"dos Santos","sequence":"additional","affiliation":[]},{"given":"Guilherme H.","family":"Travassos","sequence":"additional","affiliation":[]}],"member":"78","reference":[{"issue":"4","key":"10.1016\/j.scico.2023.102961_br0910","doi-asserted-by":"crossref","first-page":"7","DOI":"10.1080\/10686967.2001.11918977","article-title":"From continuous improvement to continuous innovation","volume":"8","author":"Cole","year":"2001","journal-title":"Qual. Manag. J."},{"key":"10.1016\/j.scico.2023.102961_br0920","article-title":"The lean startup: how today's entrepreneurs use continuous innovation to create radically successful businesses","author":"Ries","year":"2011","journal-title":"Currency"},{"key":"10.1016\/j.scico.2023.102961_br0930","series-title":"5th RCOSE ACM SIGSOFT Notes 44.3","article-title":"Continuous data-driven software engineering-towards a research agenda","author":"Gerostathopoulos","year":"2019"},{"issue":"1","key":"10.1016\/j.scico.2023.102961_br0940","doi-asserted-by":"crossref","first-page":"140","DOI":"10.1007\/s10618-008-0114-1","article-title":"Controlled experiments on the web: survey and practical guide","volume":"18","author":"Kohavi","year":"2009","journal-title":"Data Min. Knowl. Discov."},{"key":"10.1016\/j.scico.2023.102961_br0950","first-page":"6","article-title":"On the pragmatic design of literature studies in software engineering: an experience-based guideline","volume":"22","author":"Kuhrmann","year":"2017","journal-title":"ESE"},{"key":"10.1016\/j.scico.2023.102961_br0960","first-page":"528","article-title":"The goal question metric approach","author":"Basili","year":"1994","journal-title":"Encycl. Softw. Eng."},{"key":"10.1016\/j.scico.2023.102961_br0970","series-title":"Proceedings of the 18th EASE","article-title":"Guidelines for snowballing in systematic literature studies and a replication in software engineering","author":"Wohlin","year":"2014"},{"key":"10.1016\/j.scico.2023.102961_br0980","series-title":"A Handbook of Software and Systems Engineering \u2013 Empirical Observations, Laws and Theories","isbn-type":"print","author":"Endres","year":"2003","ISBN":"http:\/\/id.crossref.org\/isbn\/0321154207"},{"key":"10.1016\/j.scico.2023.102961_br0990","series-title":"2021 IEEE\/ACM 43rd International, Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP)","article-title":"Towards inclusive software engineering through A\/B testing: a case-study at windows","author":"Niculescu","year":"2021"},{"key":"10.1016\/j.scico.2023.102961_br1000","doi-asserted-by":"crossref","DOI":"10.1016\/j.infsof.2021.106551","article-title":"Controlled experimentation in continuous experimentation: knowledge and challenges","volume":"134","author":"Auer","year":"2021","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.scico.2023.102961_br1010","series-title":"2018 44th Euromicro Conference on Software Engineering and Advanced Applications (SEAA)","article-title":"Current state of research on continuous experimentation: a systematic mapping study","author":"Auer","year":"2018"},{"key":"10.1016\/j.scico.2023.102961_br1020","doi-asserted-by":"crossref","DOI":"10.1016\/j.jss.2020.110781","article-title":"Continuous experimentation and the cyber-physical systems challenge: an overview of the literature and the industrial perspective","volume":"170","author":"Giaimo","year":"2020","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.scico.2023.102961_br1030","series-title":"2018 IEEE\/ACM 4th International Workshop on Rapid Continuous Software Engineering (RCoSE)","article-title":"Continuous experimentation and a\/b testing: a mapping study","author":"Ros","year":"2018"},{"key":"10.1016\/j.scico.2023.102961_br1040","series-title":"International Conference on Agile Software Development","first-page":"277","article-title":"Challenges and strategies for undertaking continuous experimentation to embedded systems: industry and research perspectives","author":"Mattos","year":"2018"},{"key":"10.1016\/j.scico.2023.102961_br1050","series-title":"2020 46th Euromicro Conference on Software Engineering and Advanced Applications (SEAA)","article-title":"Continuous experiment definition characteristics","author":"Auer","year":"2020"},{"key":"10.1016\/j.scico.2023.102961_br1060","series-title":"Proceedings of the International Conference on Software and System Processes","article-title":"Experimentation for business-to-business mission-critical systems: a case study","author":"Mattos","year":"2020"},{"key":"10.1016\/j.scico.2023.102961_br1070","series-title":"Scientific Workshop Proceedings of the XP2015","article-title":"Towards continuous validation of customer value","author":"Olsson","year":"2015"},{"key":"10.1016\/j.scico.2023.102961_br1080","doi-asserted-by":"crossref","first-page":"201","DOI":"10.1016\/j.infsof.2018.10.006","article-title":"Identifying, categorizing and mitigating threats to validity in software engineering secondary studies","volume":"106","author":"Ampatzoglou","year":"2019","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.scico.2023.102961_br0190","series-title":"Proceedings of the 13th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","article-title":"Practical guide to controlled experiments on the web: listen to your customers, not to the hippo","author":"Kohavi","year":"2007"},{"key":"10.1016\/j.scico.2023.102961_br0200","article-title":"The HURRIER process for experimentation in business-to-business mission-critical systems","author":"Mattos","year":"2021","journal-title":"J. Softw. Evol. Process"},{"key":"10.1016\/j.scico.2023.102961_br0210","series-title":"International Conference of Software Business","article-title":"Post-deployment data collection in software-intensive embedded products","author":"Olsson","year":"2013"},{"issue":"5","key":"10.1016\/j.scico.2023.102961_br0220","doi-asserted-by":"crossref","first-page":"60","DOI":"10.1109\/MS.2017.3571569","article-title":"Toward evidence-based organizations: lessons from embedded systems, online games, and the Internet of Things","volume":"34","author":"Bosch","year":"2017","journal-title":"IEEE Softw."},{"key":"10.1016\/j.scico.2023.102961_br0230","series-title":"International Conference of Software Business","article-title":"Building products as innovation experiment systems","author":"Bosch","year":"2012"},{"issue":"4","key":"10.1016\/j.scico.2023.102961_br0240","doi-asserted-by":"crossref","first-page":"8","DOI":"10.1109\/MIC.2013.25","article-title":"Development and deployment at Facebook","volume":"17","author":"Feitelson","year":"2013","journal-title":"IEEE Internet Comput."},{"issue":"2","key":"10.1016\/j.scico.2023.102961_br0250","doi-asserted-by":"crossref","first-page":"26","DOI":"10.1109\/MS.2018.111094748","article-title":"Continuous experimentation: challenges, implementation techniques, and current research","volume":"35","author":"Schermann","year":"2018","journal-title":"IEEE Softw."},{"key":"10.1016\/j.scico.2023.102961_br0260","series-title":"2017 IEEE\/ACM 39th International Conference on Software Engineering (ICSE)","article-title":"The evolution of continuous experimentation in software product development: from data to a data-driven organization at scale","author":"Fabijan","year":"2017"},{"key":"10.1016\/j.scico.2023.102961_br0270","series-title":"2015 41st Euromicro Conference on Software Engineering and Advanced Applications","article-title":"Towards customer-centric software development: a multiple-case study","author":"Sauvola","year":"2015"},{"key":"10.1016\/j.scico.2023.102961_br0280","doi-asserted-by":"crossref","first-page":"176","DOI":"10.1016\/j.jss.2015.06.063","article-title":"Continuous software engineering: a roadmap and agenda","volume":"123","author":"Fitzgerald","year":"2017","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.scico.2023.102961_br0290","doi-asserted-by":"crossref","DOI":"10.1007\/978-3-319-19593-3_10","article-title":"Hitting the Target: Practices and Steps for Moving Towards Innovation Experiment Systems","author":"Karvonen","year":"2015"},{"issue":"12","key":"10.1016\/j.scico.2023.102961_br0300","doi-asserted-by":"crossref","DOI":"10.1002\/smr.2113","article-title":"Experimentation growth: evolving trustworthy A\/B testing capabilities in online software companies","volume":"30","author":"Fabijan","year":"2018","journal-title":"J. Softw. Evol. Process"},{"key":"10.1016\/j.scico.2023.102961_br0310","doi-asserted-by":"crossref","first-page":"80","DOI":"10.1016\/j.infsof.2016.04.008","article-title":"Raising the odds of success: the current state of experimentation in product development","volume":"77","author":"Lindgren","year":"2016","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.scico.2023.102961_br0320","series-title":"Proceedings of the Scientific Workshop Proceedings of XP2016","article-title":"Continuous experimentation on cyber-physical systems: challenges and opportunities","author":"Giaimo","year":"2016"},{"key":"10.1016\/j.scico.2023.102961_br0330","series-title":"2015 IEEE\/ACM 2nd International Workshop on Rapid Continuous Software Engineering","article-title":"Continuous experimentation in the B2B domain: a case study","author":"Rissanen","year":"2015"},{"key":"10.1016\/j.scico.2023.102961_br0340","series-title":"2014 40th EUROMICRO Conference on Software Engineering and Advanced Applications","article-title":"From opinions to data-driven software R&D: a multi-case study on how to close the \u2018open loop\u2019 problem","author":"Olsson","year":"2014"},{"key":"10.1016\/j.scico.2023.102961_br0350","series-title":"2019 IEEE\/ACM Joint 4th International Workshop on Rapid Continuous Software Engineering and 1st International Workshop on Data-Driven Decisions, Experimentation, and Evolution (RCoSE\/DDrEE)","article-title":"Hypotheses engineering: first essential steps of experiment-driven software development","author":"Melegati","year":"2019"},{"key":"10.1016\/j.scico.2023.102961_br0360","series-title":"International Conference on Product-Focused Software Process Improvement","article-title":"Data-driven development: challenges in online, embedded and on-premise software","author":"Olsson","year":"2019"},{"key":"10.1016\/j.scico.2023.102961_br0370","series-title":"2016 IEEE\/ACM 11th International Symposium on Software Engineering for Adaptive and Self-Managing Systems (SEAMS)","article-title":"Data-driven continuous evolution of smart systems","author":"Bosch","year":"2016"},{"key":"10.1016\/j.scico.2023.102961_br0380","series-title":"2018 44th Euromicro Conference on Software Engineering and Advanced Applications (SEAA)","article-title":"Online controlled experimentation at scale: an empirical survey on the current state of A\/B testing","author":"Fabijan","year":"2018"},{"issue":"2","key":"10.1016\/j.scico.2023.102961_br0390","doi-asserted-by":"crossref","first-page":"60","DOI":"10.1109\/MS.2018.2875842","article-title":"The online controlled experiment lifecycle","volume":"37","author":"Fabijan","year":"2018","journal-title":"IEEE Softw."},{"key":"10.1016\/j.scico.2023.102961_br0400","series-title":"2019 IEEE\/ACM 41st International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP)","article-title":"Three key checklists and remedies for trustworthy analysis of online controlled experiments at scale","author":"Fabijan","year":"2019"},{"key":"10.1016\/j.scico.2023.102961_br0410","series-title":"2019 45th Euromicro Conference on Software Engineering and Advanced Applications (SEAA)","article-title":"The automotive take on continuous experimentation: a multiple case study","author":"Giaimo","year":"2019"},{"key":"10.1016\/j.scico.2023.102961_br0420","doi-asserted-by":"crossref","DOI":"10.1007\/978-3-030-58923-3_5","article-title":"Continuous Experimentation for Automotive Software on the Example of a Heavy Commercial Vehicle in Daily Operation","author":"Giaimo","year":"2020"},{"issue":"1","key":"10.1016\/j.scico.2023.102961_br0430","doi-asserted-by":"crossref","first-page":"20","DOI":"10.1145\/3331651.3331655","article-title":"Top challenges from the first practical online controlled experiments summit","volume":"21","author":"Gupta","year":"2019","journal-title":"ACM SIGKDD Explor. Newsl."},{"key":"10.1016\/j.scico.2023.102961_br0440","series-title":"2019 45th Euromicro Conference on Software Engineering and Advanced Applications (SEAA)","article-title":"Enterprise-level controlled experiments at scale: challenges and solutions","author":"Liu","year":"2019"},{"key":"10.1016\/j.scico.2023.102961_br0450","series-title":"International Conference on Product-Focused Software Process Improvement","article-title":"An activity and metric model for online controlled experiments","author":"Mattos","year":"2018"},{"key":"10.1016\/j.scico.2023.102961_br0460","series-title":"International Conference of Software Business","article-title":"Experimentation that matters: a multi-case study on the challenges with A\/B testing","author":"Olsson","year":"2017"},{"key":"10.1016\/j.scico.2023.102961_br0470","series-title":"Proceedings of the 2019 27th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering","article-title":"Improving requirements engineering practices to support experimentation in software startups","author":"Melegati","year":"2019"},{"key":"10.1016\/j.scico.2023.102961_br0480","series-title":"International Conference on Product-Focused Software Process Improvement","article-title":"Enablers and inhibitors of experimentation in early-stage software startups","author":"Melegati","year":"2019"},{"key":"10.1016\/j.scico.2023.102961_br0490","article-title":"XPro: a model to explain the limited adoption and implementation of experimentation in software startups","author":"Melegati","year":"2020","journal-title":"IEEE Trans. Softw. Eng."},{"key":"10.1016\/j.scico.2023.102961_br0500","series-title":"2020 46th Euromicro Conference on Software Engineering and Advanced Applications (SEAA)","article-title":"MVP and experimentation in software startups: a qualitative survey","author":"Melegati","year":"2020"},{"key":"10.1016\/j.scico.2023.102961_br0510","doi-asserted-by":"crossref","first-page":"292","DOI":"10.1016\/j.jss.2016.03.034","article-title":"The RIGHT model for continuous experimentation","volume":"123","author":"Fagerholm","year":"2017","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.scico.2023.102961_br0520","doi-asserted-by":"crossref","first-page":"41","DOI":"10.1016\/j.infsof.2018.02.010","article-title":"We're doing it live: a multi-method empirical study on continuous experimentation","volume":"99","author":"Schermann","year":"2018","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.scico.2023.102961_br0530","series-title":"International Conference on Product-Focused Software Process Improvement","article-title":"Continuous experimentation for software organizations with low control of roadmap and a large distance to users: an exploratory case study","author":"Sveningson","year":"2019"},{"key":"10.1016\/j.scico.2023.102961_br0540","series-title":"Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","article-title":"Evaluating mobile apps with A\/B and quasi A\/B tests","author":"Xu","year":"2016"},{"key":"10.1016\/j.scico.2023.102961_br0550","doi-asserted-by":"crossref","DOI":"10.1016\/j.infsof.2019.106244","article-title":"Patterns of user involvement in experiment-driven software development","volume":"120","author":"Yaman","year":"2020","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.scico.2023.102961_br0560","series-title":"2021 XLVII Latin American Computing Conference (CLEI)","first-page":"1","article-title":"Experiment-driven improvements in human-in-the-loop machine learning annotation via significance-based A\/B testing","author":"Alfaro-Flores","year":"2021"},{"key":"10.1016\/j.scico.2023.102961_br0570","series-title":"International Conference on Software Quality","article-title":"An approach for platform-independent online controlled experimentation","author":"Auer","year":"2021"},{"key":"10.1016\/j.scico.2023.102961_br0580","series-title":"International Conference on Current Trends in Theory and Practice of Informatics","first-page":"445","article-title":"An infrastructure for platform-independent experimentation of software changes","author":"Auer","year":"2021"},{"key":"10.1016\/j.scico.2023.102961_br0590","series-title":"2021 IEEE 45th Annual Computers, Software, and Applications Conference, (COMPSAC)","first-page":"992","article-title":"An architecture for enabling A\/B experiments in automotive embedded software","author":"Liu","year":"2021"},{"issue":"2","key":"10.1016\/j.scico.2023.102961_br0600","first-page":"228","article-title":"Towards data-driven culture in a Spanish automobile manufacturer: a case study","volume":"13","author":"Esteller-Cucala","year":"2020","journal-title":"J. Ind. Eng. Manag."},{"key":"10.1016\/j.scico.2023.102961_br0610","series-title":"2017 IEEE International Conference on Software Architecture (ICSA)","article-title":"Design criteria to architect continuous experimentation for self-driving vehicles","author":"Giaimo","year":"2017"},{"key":"10.1016\/j.scico.2023.102961_br0620","series-title":"2020 46th Euromicro Conference on Software Engineering and Advanced Applications (SEAA)","article-title":"Automotive a\/b testing: challenges and lessons learned from practice","author":"Mattos","year":"2020"},{"key":"10.1016\/j.scico.2023.102961_br0630","series-title":"International Conference on Agile Software Development","article-title":"Hypotheses elicitation in early-stage software startups based on cognitive mapping","author":"Melegati","year":"2020"},{"key":"10.1016\/j.scico.2023.102961_br0640","article-title":"Understanding hypotheses engineering in software startups through a gray literature review","author":"Melegati","year":"2020","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.scico.2023.102961_br0650","series-title":"International Conference on Lean Enterprise Software and Systems","article-title":"Towards data-driven product development: a multiple case study on post-deployment data usage in software-intensive embedded systems","author":"Olsson","year":"2013"},{"key":"10.1016\/j.scico.2023.102961_br0660","series-title":"International Conference of Software Business","article-title":"Towards continuous customer validation: a conceptual model for combining qualitative customer feedback with quantitative customer observation","author":"Olsson","year":"2015"},{"key":"10.1016\/j.scico.2023.102961_br0670","series-title":"International Conference on Software Business","article-title":"Continuous experimentation with product-led business models: a comparative case study","author":"Ros","year":"2020"},{"key":"10.1016\/j.scico.2023.102961_br0680","series-title":"European Conference on Software Architecture","article-title":"Considerations about continuous experimentation for resource-constrained platforms in self-driving vehicles","author":"Giaimo","year":"2017"},{"key":"10.1016\/j.scico.2023.102961_br0690","series-title":"2017 IEEE\/ACM 39th International Conference on Software Engineering: Software Engineering in Practice Track (ICSE-SEIP)","article-title":"Characterizing experimentation in continuous deployment: a case study on Bing","author":"Kevic","year":"2017"},{"key":"10.1016\/j.scico.2023.102961_br0700","series-title":"International Conference on Agile Software Development","article-title":"Software development as an experiment system: a qualitative survey on the state of the practice","author":"Lindgren","year":"2015"},{"key":"10.1016\/j.scico.2023.102961_br0710","series-title":"2018 44th Euromicro Conference on Software Engineering and Advanced Applications (SEAA)","article-title":"Continuous experimentation scenarios: a case study in e-commerce","author":"Ros","year":"2018"},{"key":"10.1016\/j.scico.2023.102961_br0720","series-title":"International Conference on Product-Focused Software Process Improvement","article-title":"Transitioning towards continuous experimentation in a large software product and service development organization\u2013a case study","author":"Yaman","year":"2016"},{"key":"10.1016\/j.scico.2023.102961_br0730","series-title":"2018 44th Euromicro Conference on Software Engineering and Advanced Applications (SEAA)","article-title":"Continuous experimentation in mobile game development","author":"Yaman","year":"2018"},{"key":"10.1016\/j.scico.2023.102961_br0740","series-title":"Proceedings of the 15th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","article-title":"Seven pitfalls to avoid when running controlled experiments on the web","author":"Crook","year":"2009"},{"key":"10.1016\/j.scico.2023.102961_br0750","series-title":"Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","article-title":"Data-driven metric development for online controlled experiments: seven lessons learned","author":"Deng","year":"2016"},{"key":"10.1016\/j.scico.2023.102961_br0760","series-title":"Proceedings of the Conference on Fairness, Accountability, and Transparency","article-title":"Who's the Guinea Pig? Investigating online A\/B\/n tests in-the-wild","author":"Jiang","year":"2019"},{"key":"10.1016\/j.scico.2023.102961_br0770","series-title":"Proceedings of the 18th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","article-title":"Trustworthy online controlled experiments: five puzzling outcomes explained","author":"Kohavi","year":"2012"},{"issue":"4","key":"10.1016\/j.scico.2023.102961_br0780","first-page":"1","article-title":"The Netflix recommender system: algorithms, business value, and innovation","volume":"6","author":"Gomez-Uribe","year":"2015","journal-title":"ACM Transactions on Management Information Systems (TMIS)"},{"key":"10.1016\/j.scico.2023.102961_br0790","series-title":"Proceedings of the 20th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","article-title":"Seven rules of thumb for website experimenters","author":"Kohavi","year":"2014"},{"key":"10.1016\/j.scico.2023.102961_br0800","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/j.future.2014.10.007","article-title":"A survey of general-purpose experiment management tools for distributed systems","volume":"45","author":"Buchert","year":"2015","journal-title":"Future Gener. Comput. Syst."},{"key":"10.1016\/j.scico.2023.102961_br0810","series-title":"2018 IEEE International Conference on Software Architecture Companion (ICSA-C)","article-title":"Cost-aware stage-based experimentation: challenges and emerging results","author":"Gerostathopoulos","year":"2018"},{"key":"10.1016\/j.scico.2023.102961_br0820","series-title":"International Conference on Product-Focused Software Process Improvement","article-title":"How do software startups approach experimentation? Empirical results from a qualitative interview study","author":"Gutbrod","year":"2017"},{"key":"10.1016\/j.scico.2023.102961_br0830","series-title":"2017 ACM\/IEEE International Symposium on Empirical Software Engineering and Measurement (ESEM)","article-title":"Notifying and involving users in experimentation: ethical perceptions of software practitioners","author":"Yaman","year":"2017"},{"key":"10.1016\/j.scico.2023.102961_br0840","series-title":"Proceedings of the 17th International Middleware Conference","article-title":"Bifrost: supporting continuous deployment with automated enactment of multi-phase live testing strategies","author":"Schermann","year":"2016"},{"key":"10.1016\/j.scico.2023.102961_br0850","series-title":"Proceedings of the 16th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","article-title":"Overlapping experiment infrastructure: more, better, faster experimentation","author":"Tang","year":"2010"},{"key":"10.1016\/j.scico.2023.102961_br0860","series-title":"International Symposium on Leveraging Applications of Formal Methods, Verification, and Validation","article-title":"Eternal embedded software: towards innovation experiment systems","author":"Bosch","year":"2012"},{"key":"10.1016\/j.scico.2023.102961_br0870","series-title":"2012 Joint Working IEEE\/IFIP Conference on Software Architecture and European Conference on Software Architecture","article-title":"Architecture for large-scale innovation experiment systems","author":"Eklund","year":"2012"},{"key":"10.1016\/j.scico.2023.102961_br0880","series-title":"Proceedings of the IASTED","article-title":"Towards R&D as innovation experiment systems: a framework for moving beyond agile software development","author":"Olsson","year":"2013"},{"issue":"5","key":"10.1016\/j.scico.2023.102961_br0890","doi-asserted-by":"crossref","first-page":"793","DOI":"10.1111\/jpim.12233","article-title":"User involvement throughout the innovation process in high-tech industries","volume":"32","author":"Bosch-Sijtsema","year":"2015","journal-title":"J. Prod. Innov. Manag."},{"key":"10.1016\/j.scico.2023.102961_br0900","series-title":"Proceedings of the 19th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","article-title":"Online controlled experiments at large scale","author":"Kohavi","year":"2013"}],"container-title":["Science of Computer Programming"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0167642323000436?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0167642323000436?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2024,4,1]],"date-time":"2024-04-01T10:52:34Z","timestamp":1711968754000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0167642323000436"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,7]]},"references-count":90,"alternative-id":["S0167642323000436"],"URL":"https:\/\/doi.org\/10.1016\/j.scico.2023.102961","relation":{},"ISSN":["0167-6423"],"issn-type":[{"value":"0167-6423","type":"print"}],"subject":[],"published":{"date-parts":[[2023,7]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Characterization of continuous experimentation in software engineering: Expressions, models, and strategies","name":"articletitle","label":"Article Title"},{"value":"Science of Computer Programming","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.scico.2023.102961","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2023 Elsevier B.V. All rights reserved.","name":"copyright","label":"Copyright"}],"article-number":"102961"}}