{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,2,21]],"date-time":"2025-02-21T00:52:42Z","timestamp":1740099162025,"version":"3.37.3"},"publisher-location":"Cham","reference-count":29,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030036720"},{"type":"electronic","value":"9783030036737"}],"license":[{"start":{"date-parts":[[2018,1,1]],"date-time":"2018-01-01T00:00:00Z","timestamp":1514764800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2018]]},"DOI":"10.1007\/978-3-030-03673-7_14","type":"book-chapter","created":{"date-parts":[[2018,11,2]],"date-time":"2018-11-02T03:53:44Z","timestamp":1541130824000},"page":"182-198","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":8,"title":["An Activity and Metric Model for Online Controlled Experiments"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2501-9926","authenticated-orcid":false,"given":"David","family":"Issa Mattos","sequence":"first","affiliation":[]},{"given":"Pavel","family":"Dmitriev","sequence":"additional","affiliation":[]},{"given":"Aleksander","family":"Fabijan","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2854-722X","authenticated-orcid":false,"given":"Jan","family":"Bosch","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7700-1816","authenticated-orcid":false,"given":"Helena","family":"Holmstr\u00f6m Olsson","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2018,11,3]]},"reference":[{"key":"14_CR1","doi-asserted-by":"crossref","unstructured":"Fabijan, A., Dmitriev, P., Olsson, H.H., Bosch, J.: The benefits of controlled experimentation at scale. In: Proceedings of the 43rd Euromicro Conference on Software Engineering and Advanced Applications, SEAA 2017, pp. 18\u201326 (2017)","DOI":"10.1109\/SEAA.2017.47"},{"key":"14_CR2","doi-asserted-by":"crossref","unstructured":"Kohavi, R., Deng, A., Frasca, B., Longbotham, R., Walker, T., Xu, Y.: Trustworthy online controlled experiments. In: Proceedings of the 18th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, KDD 2012, p. 786 (2012)","DOI":"10.1145\/2339530.2339653"},{"key":"14_CR3","doi-asserted-by":"crossref","unstructured":"Bakshy, E., Eckles, D., Bernstein, M.S.: Designing and deploying online field experiments. In: Proceedings of the 23rd International Conference on World Wide Web, WWW 2014, pp. 283\u2013292, September 2014","DOI":"10.1145\/2566486.2567967"},{"key":"14_CR4","doi-asserted-by":"crossref","unstructured":"Gui, H., Xu, Y., Bhasin, A., Han, J.: Network A\/B testing. In: Proceedings of the 24th International Conference on World Wide Web, WWW 2015, pp. 399\u2013409 (2015)","DOI":"10.1145\/2736277.2741081"},{"key":"14_CR5","doi-asserted-by":"crossref","unstructured":"Tang, D., Agarwal, A., O\u2019Brien, D., Meyer, M.: Overlapping experiment infrastructure. In: Proceedings of the 16th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, KDD 2010, p. 17 (2010)","DOI":"10.1145\/1835804.1835810"},{"key":"14_CR6","doi-asserted-by":"crossref","unstructured":"Dmitriev, P., Frasca, B., Gupta, S., Kohavi, R., Vaz, G.: Pitfalls of long-term online controlled experiments. In: Proceedings of the 2016 IEEE International Conference on Big Data, Big Data 2016, pp. 1367\u20131376 (2016)","DOI":"10.1109\/BigData.2016.7840744"},{"key":"14_CR7","doi-asserted-by":"publisher","first-page":"292","DOI":"10.1016\/j.jss.2016.03.034","volume":"123","author":"F Fagerholm","year":"2017","unstructured":"Fagerholm, F., Sanchez Guinea, A., M\u00e4enp\u00e4\u00e4, H., M\u00fcnch, J.: The RIGHT model for continuous experimentation. J. Syst. Softw. 123, 292\u2013305 (2017)","journal-title":"J. Syst. Softw."},{"key":"14_CR8","doi-asserted-by":"publisher","first-page":"155","DOI":"10.1007\/978-3-319-11283-1_13","volume-title":"Continuous Software Engineering","author":"HH Olsson","year":"2014","unstructured":"Olsson, H.H., Bosch, J.: The HYPEX model: from opinions to data-driven software development. In: Bosch, J. (ed.) Continuous Software Engineering, pp. 155\u2013164. Springer, Cham (2014). https:\/\/doi.org\/10.1007\/978-3-319-11283-1_13"},{"key":"14_CR9","series-title":"Lecture Notes in Business Information Processing","doi-asserted-by":"publisher","first-page":"154","DOI":"10.1007\/978-3-319-19593-3_13","volume-title":"Software Business","author":"HH Olsson","year":"2015","unstructured":"Olsson, H.H., Bosch, J.: Towards continuous customer validation: a conceptual model for combining qualitative customer feedback with quantitative customer observation. In: Fernandes, J.M., Machado, R.J., Wnuk, K. (eds.) ICSOB 2015. LNBIP, vol. 210, pp. 154\u2013166. Springer, Cham (2015). https:\/\/doi.org\/10.1007\/978-3-319-19593-3_13"},{"key":"14_CR10","doi-asserted-by":"crossref","unstructured":"Dmitriev, P., Gupta, S., Dong Woo, K., Vaz, G.: A dirty dozen: twelve common metric interpretation pitfalls in online controlled experiments. In: Proceedings of the 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, KDD 2017, pp. 1427\u20131436 (2017)","DOI":"10.1145\/3097983.3098024"},{"key":"14_CR11","doi-asserted-by":"crossref","unstructured":"Crook, T., Frasca, B., Kohavi, R., Longbotham, R.: Seven pitfalls to avoid when running controlled experiments on the web. In: Proceedings of the 15th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, KDD 2009, p. 1105 (2009)","DOI":"10.1145\/1557019.1557139"},{"key":"14_CR12","unstructured":"Kluck, T., Vermeer, L.: Leaky abstraction in online experimentation platforms: a conceptual framework to categorize common challenges (2017)"},{"key":"14_CR13","doi-asserted-by":"crossref","unstructured":"Chen, R., Chen, M., Jadav, M.R., Bae, J., Matheson, D.: Faster online experimentation by eliminating traditional A\/A validation, pp. 1635\u20131641 (2017)","DOI":"10.1109\/BigData.2017.8258098"},{"key":"14_CR14","unstructured":"Kaufman, R.L., Pitchforth, J., Vermeer, L.: Democratizing online controlled experiments at Booking.com. http:\/\/arxiv.org\/abs\/1710.08217 . Accessed 23 Oct 2017"},{"issue":"1","key":"14_CR15","doi-asserted-by":"publisher","first-page":"140","DOI":"10.1007\/s10618-008-0114-1","volume":"18","author":"R Kohavi","year":"2009","unstructured":"Kohavi, R., Longbotham, R., Sommerfield, D., Henne, R.M.: Controlled experiments on the web: survey and practical guide. Data Min. Knowl. Discov. 18(1), 140\u2013181 (2009)","journal-title":"Data Min. Knowl. Discov."},{"key":"14_CR16","series-title":"Lecture Notes in Business Information Processing","doi-asserted-by":"publisher","first-page":"139","DOI":"10.1007\/978-3-319-19593-3_12","volume-title":"Software Business","author":"A Fabijan","year":"2015","unstructured":"Fabijan, A., Olsson, H.H., Bosch, J.: Customer feedback and data collection techniques in software R&D: a literature review. In: Fernandes, J.M., Machado, R.J., Wnuk, K. (eds.) ICSOB 2015. LNBIP, vol. 210, pp. 139\u2013153. Springer, Cham (2015). https:\/\/doi.org\/10.1007\/978-3-319-19593-3_12"},{"key":"14_CR17","first-page":"74","volume":"95","author":"R Kohavi","year":"2017","unstructured":"Kohavi, R., Thomke, S.: The surprising power of online experiments. Harv. Bus. Rev. 95, 74 (2017)","journal-title":"Harv. Bus. Rev."},{"key":"14_CR18","doi-asserted-by":"crossref","unstructured":"Gupta, S., Bhardwaj, S., Dmitriev, P., Ulanova, L., Raff, P., Fabijan, A.: The anatomy of a large-scale online experimentation platform. In: International Conference on Software Architecture, ICSA 2018, May 2018","DOI":"10.1109\/ICSA.2018.00009"},{"key":"14_CR19","doi-asserted-by":"crossref","unstructured":"Kevic, K., Murphy, B., Williams, L., Beckmann, J.: Characterizing experimentation in continuous deployment: a case study on bing. In: Proceedings of the 2017 IEEE\/ACM 39th International Conference on Software Engineering: Software Engineering in Practice Track, ICSE-SEIP 2017, pp. 123\u2013132 (2017)","DOI":"10.1109\/ICSE-SEIP.2017.19"},{"key":"14_CR20","doi-asserted-by":"crossref","unstructured":"Fabijan, A., Dmitriev, P., Olsson, H.H., Bosch, J.: The evolution of continuous experimentation in software product development. In: Proceedings of the 39th International Conference on Software Engineering, ICSE 2017 (2017)","DOI":"10.1109\/ICSE.2017.76"},{"key":"14_CR21","doi-asserted-by":"crossref","unstructured":"Dmitriev, P., Wu, X.: Measuring metrics. In: Proceedings of the 25th ACM International on Conference on Information and Knowledge Management, CIKM 2016, pp. 429\u2013437 (2016)","DOI":"10.1145\/2983323.2983356"},{"key":"14_CR22","doi-asserted-by":"crossref","unstructured":"Deng, A., Shi, X.: Data-driven metric development for online controlled experiments. In: Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, KDD 2016, pp. 77\u201386 (2016)","DOI":"10.1145\/2939672.2939700"},{"key":"14_CR23","unstructured":"Ries, E.: The Lean Startup: How Today\u2019s Entrepreneurs Use Continuous Innovation to Create Radically Successful Businesses (2011)"},{"issue":"2","key":"14_CR24","doi-asserted-by":"publisher","first-page":"131","DOI":"10.1007\/s10664-008-9102-8","volume":"14","author":"P Runeson","year":"2009","unstructured":"Runeson, P., H\u00f6st, M.: Guidelines for conducting and reporting case study research in software engineering. Empir. Softw. Eng. 14(2), 131\u2013164 (2009)","journal-title":"Empir. Softw. Eng."},{"key":"14_CR25","volume-title":"Real World Research","author":"C Robson","year":"2016","unstructured":"Robson, C., McCartan, K.: Real World Research, 4th edn. John Wiley & Sons Ltd., New York (2016)","edition":"4"},{"key":"14_CR26","doi-asserted-by":"crossref","unstructured":"Kohavi, R., Deng, A., Frasca, B., Walker, T., Xu, Y., Pohlmann, N.: Online controlled experiments at large scale. In: Proceedings of the 19th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, KDD 2013, p. 1168 (2013)","DOI":"10.1145\/2487575.2488217"},{"key":"14_CR27","doi-asserted-by":"crossref","unstructured":"Deng, A., Lu, J., Litz, J.: Trustworthy analysis of online A\/B tests. In: Proceedings of the Tenth ACM International Conference on Web Search and Data Mining, WSDM 2017, pp. 641\u2013649 (2017)","DOI":"10.1145\/3018661.3018677"},{"key":"14_CR28","first-page":"3207","volume":"14","author":"L Bottou","year":"2013","unstructured":"Bottou, L., et al.: Counterfactual reasoning and learning systems. J. Mach. Learn. Res. 14, 3207\u20133260 (2013)","journal-title":"J. Mach. Learn. Res."},{"key":"14_CR29","doi-asserted-by":"crossref","unstructured":"Kohavi, R., Deng, A., Longbotham, R., Xu, Y.: Seven rules of thumb for web site experimenters. In: Proceedings of the 20th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, KDD 2014, pp. 1857\u20131866 (2014)","DOI":"10.1145\/2623330.2623341"}],"container-title":["Lecture Notes in Computer Science","Product-Focused Software Process Improvement"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-03673-7_14","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,10,30]],"date-time":"2019-10-30T15:51:32Z","timestamp":1572450692000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-030-03673-7_14"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018]]},"ISBN":["9783030036720","9783030036737"],"references-count":29,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-03673-7_14","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2018]]},"assertion":[{"value":"PROFES","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Product-Focused Software Process Improvement","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Wolfsburg","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Germany","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2018","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"28 November 2018","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"30 November 2018","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"19","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"profes2018","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/profes2018.wordpress.com\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information"}},{"value":"65","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information"}},{"value":"16","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information"}},{"value":"8","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information"}},{"value":"25% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information"}},{"value":"4","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information"}}]}}