{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T18:42:29Z","timestamp":1726080149740},"publisher-location":"Cham","reference-count":56,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030454388"},{"type":"electronic","value":"9783030454395"}],"license":[{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020]]},"DOI":"10.1007\/978-3-030-45439-5_27","type":"book-chapter","created":{"date-parts":[[2020,4,11]],"date-time":"2020-04-11T04:02:50Z","timestamp":1586577770000},"page":"399-414","update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":9,"title":["You Can Teach an Old Dog New Tricks: Rank Fusion applied to\u00a0Coordination Level Matching for\u00a0Ranking in Systematic Reviews"],"prefix":"10.1007","author":[{"ORCID":"http:\/\/orcid.org\/0000-0001-9578-7157","authenticated-orcid":false,"given":"Harrisen","family":"Scells","sequence":"first","affiliation":[]},{"ORCID":"http:\/\/orcid.org\/0000-0003-0271-5563","authenticated-orcid":false,"given":"Guido","family":"Zuccon","sequence":"additional","affiliation":[]},{"ORCID":"http:\/\/orcid.org\/0000-0001-5577-3391","authenticated-orcid":false,"given":"Bevan","family":"Koopman","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2020,4,8]]},"reference":[{"key":"27_CR1","doi-asserted-by":"crossref","unstructured":"Abualsaud, M., Ghelani, N., Zhang, H., Smucker, M.D., Cormack, G.V., Grossman, M.R.: A system for efficient high-recall retrieval. In: The 41st International ACM SIGIR Conference on Research & Development in Information Retrieval, pp. 1317\u20131320 (2018)","DOI":"10.1145\/3209978.3210176"},{"key":"27_CR2","unstructured":"Alharbi, A., Briggs, W., Stevenson, M.: Retrieving and ranking studies for systematic reviews: University of Sheffield\u2019s approach to CLEF eHealth 2018 task 2. In: CEUR Workshop Proceedings, vol. 2125. CEUR Workshop Proceedings (2018)"},{"key":"27_CR3","unstructured":"Alharbi, A., Stevenson, M.: Ranking abstracts to identify relevant evidence for systematic reviews: The University of Sheffield\u2019s approach to CLEF eHealth 2017 task 2. In: CLEF (Working Notes) (2017)"},{"key":"27_CR4","unstructured":"Anagnostou, A., Lagopoulos, A., Tsoumakas, G., Vlahavas, I.P.: Combining inter-review learning-to-rank and intra-review incremental training for title and abstract screening in systematic reviews. In: CLEF (Working Notes) (2017)"},{"key":"27_CR5","unstructured":"Benham, R., Culpepper, J.S., Gallagher, L., Lu, X., Mackenzie, J.: Towards efficient and effective query variant generation. In: DESIRES, pp. 62\u201367 (2018)"},{"issue":"5","key":"27_CR6","doi-asserted-by":"publisher","first-page":"249","DOI":"10.1016\/0306-4573(81)90019-4","volume":"17","author":"DA Buell","year":"1981","unstructured":"Buell, D.A.: A general model of query processing in information retrieval systems. Inf. Process. Manag. 17(5), 249\u2013262 (1981)","journal-title":"Inf. Process. Manag."},{"key":"27_CR7","unstructured":"Chen, J., et al.: ECNU at 2017 eHealth task 2: Technologically assisted reviews in empirical medicine. In: CLEF (Working Notes) (2017)"},{"key":"27_CR8","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-37131-8_12","volume-title":"Methods of Clinical Epidemiology","author":"J Clark","year":"2013","unstructured":"Clark, J.: Systematic reviewing. In: Doi, S.A.R., Williams, G.M. (eds.) Methods of Clinical Epidemiology. Springer, Heidelberg (2013). https:\/\/doi.org\/10.1007\/978-3-642-37131-8_12"},{"key":"27_CR9","unstructured":"Cohen, A.M., Smalheiser, N.R.: UIC\/OHSU CLEF 2018 task 2 diagnostic test accuracy ranking using publication type cluster similarity measures. In: CEUR Workshop Proceedings, vol. 2125 (2018)"},{"issue":"2","key":"27_CR10","first-page":"206","volume":"13","author":"A Cohen","year":"2006","unstructured":"Cohen, A., Hersh, W., Peterson, K., Yen, P.: Reducing workload in systematic review preparation using automated citation classification. JAMIA 13(2), 206\u2013219 (2006)","journal-title":"JAMIA"},{"key":"27_CR11","doi-asserted-by":"crossref","unstructured":"Cormack, G.V., Grossman, M.R.: Engineering quality and reliability in technology-assisted review. In: Proceedings of the 39th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 75\u201384 (2016)","DOI":"10.1145\/2911451.2911510"},{"key":"27_CR12","unstructured":"Cormack, G.V., Grossman, M.R.: Technology-assisted review in empirical medicine: Waterloo participation in CLEF eHealth 2017. In: CLEF (Working Notes) (2017)"},{"key":"27_CR13","unstructured":"Cormack, G.V., Grossman, M.R.: Technology-assisted review in empirical medicine: Waterloo participation in clef eHealth 2018. In: CLEF (Working Notes) (2018)"},{"issue":"1","key":"27_CR14","doi-asserted-by":"publisher","first-page":"27","DOI":"10.1023\/A:1009973415168","volume":"2","author":"F Crestani","year":"2000","unstructured":"Crestani, F.: Exploiting the similarity of non-matching terms at retrieval time. Inf. Retrieval 2(1), 27\u201347 (2000). https:\/\/doi.org\/10.1023\/A:1009973415168","journal-title":"Inf. Retrieval"},{"issue":"2","key":"27_CR15","doi-asserted-by":"publisher","first-page":"71","DOI":"10.1002\/(SICI)1097-4571(198603)37:2<71::AID-ASI3>3.0.CO;2-4","volume":"37","author":"WB Croft","year":"1986","unstructured":"Croft, W.B.: Boolean queries and term dependencies in probabilistic retrieval models. J. Am. Soc. Inf. Sci. 37(2), 71\u201377 (1986)","journal-title":"J. Am. Soc. Inf. Sci."},{"key":"27_CR16","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"672","DOI":"10.1007\/978-3-319-76941-7_61","volume-title":"Advances in Information Retrieval","author":"GM Di Nunzio","year":"2018","unstructured":"Di Nunzio, G.M.: A study of an automatic stopping strategy for technologically assisted medical reviews. In: Pasi, G., Piwowarski, B., Azzopardi, L., Hanbury, A. (eds.) ECIR 2018. LNCS, vol. 10772, pp. 672\u2013677. Springer, Cham (2018). https:\/\/doi.org\/10.1007\/978-3-319-76941-7_61"},{"key":"27_CR17","unstructured":"Di Nunzio, G.M., Beghini, F., Vezzani, F., Henrot, G.: An interactive two-dimensional approach to query aspects rewriting in systematic reviews. IMS unipd at CLEF eHealth task 2. In: CLEF (Working Notes) (2017)"},{"key":"27_CR18","unstructured":"Di Nunzio, G.M., Ciuffreda, G., Vezzani, F.: Interactive sampling for systematic reviews. IMS unipd at CLEF 2018 eHealth task 2. In: CLEF (Working Notes) (2018)"},{"key":"27_CR19","doi-asserted-by":"crossref","unstructured":"Fang, H., Zhai, C.: An exploration of axiomatic approaches to information retrieval. In: Proceedings of the 28th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 480\u2013487. ACM (2005)","DOI":"10.1145\/1076034.1076116"},{"issue":"8","key":"27_CR20","doi-asserted-by":"publisher","first-page":"e2005343","DOI":"10.1371\/journal.pbio.2005343","volume":"16","author":"N Fiorini","year":"2018","unstructured":"Fiorini, N., et al.: Best match: new relevance search for pubmed. PLoS Biol. 16(8), e2005343 (2018)","journal-title":"PLoS Biol."},{"key":"27_CR21","unstructured":"Hollmann, N., Eickhoff, C.: Ranking and feedback-based stopping for recall-centric document retrieval. In: CLEF (Working Notes) (2017)"},{"issue":"3","key":"27_CR22","doi-asserted-by":"publisher","first-page":"449","DOI":"10.1007\/s10791-005-6994-4","volume":"8","author":"DF Hsu","year":"2005","unstructured":"Hsu, D.F., Taksa, I.: Comparing rank and score combination methods for data fusion in information retrieval. Inf. Retrieval 8(3), 449\u2013480 (2005). https:\/\/doi.org\/10.1007\/s10791-005-6994-4","journal-title":"Inf. Retrieval"},{"key":"27_CR23","unstructured":"Higgins, J.P.T., Green, S.: Cochrane handbook for systematic reviews of interventions version 5.1.0 [updated March 2011]. The Cochrane Collaboration (2011)"},{"key":"27_CR24","unstructured":"Kalphov, V., Georgiadis, G., Azzopardi, L.: SiS at CLEF 2017 eHealth TAR task. In: CEUR Workshop Proceedings, vol. 1866, pp. 1\u20135 (2017)"},{"key":"27_CR25","unstructured":"Kanoulas, E., Li, D., Azzopardi, L., Spijker, R.: CLEF 2017 technologically assisted reviews in empirical medicine overview. In: CLEF 2017 (2017)"},{"key":"27_CR26","unstructured":"Kanoulas, E., Spijker, R., Li, D., Azzopardi, L.: CLEF 2018 technology assisted reviews in empirical medicine overview. In: CLEF 2018 Evaluation Labs and Workshop: Online Working Notes (2018)"},{"issue":"1","key":"27_CR27","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/1471-2210-10-1","volume":"10","author":"S Karimi","year":"2010","unstructured":"Karimi, S., Pohl, S., Scholer, F., Cavedon, L., Zobel, J.: Boolean versus ranked querying for biomedical systematic reviews. BMC MIDM 10(1), 1 (2010)","journal-title":"BMC MIDM"},{"key":"27_CR28","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"52","DOI":"10.1007\/978-3-319-98932-7_5","volume-title":"Experimental IR Meets Multilinguality, Multimodality, and Interaction","author":"A Lagopoulos","year":"2018","unstructured":"Lagopoulos, A., Anagnostou, A., Minas, A., Tsoumakas, G.: Learning-to-rank and relevance feedback for literature appraisal in empirical medicine. In: Bellot, P., et al. (eds.) CLEF 2018. LNCS, vol. 11018, pp. 52\u201363. Springer, Cham (2018). https:\/\/doi.org\/10.1007\/978-3-319-98932-7_5"},{"key":"27_CR29","doi-asserted-by":"crossref","unstructured":"Lee, G.E., Sun, A.: Seed-driven document ranking for systematic reviews in evidence-based medicine. In: Proceedings of the 41st Annual International ACM SIGIR Conference on Research & Development in Information Retrieval, SIGIR 2018, pp. 455\u2013464 (2018)","DOI":"10.1145\/3209978.3209994"},{"key":"27_CR30","unstructured":"Lee, G.E.: A study of convolutional neural networks for clinical document classification in systematic reviews: SysReview at CLEF eHealth 2017 (2017)"},{"issue":"4","key":"27_CR31","doi-asserted-by":"publisher","first-page":"239","DOI":"10.1002\/(SICI)1097-4571(198707)38:4<239::AID-ASI4>3.0.CO;2-6","volume":"38","author":"R Losee","year":"1987","unstructured":"Losee, R.: Probabilistic retrieval and coordination level matching. J. Am. Soc. Inf. Sci. 38(4), 239\u2013244 (1987)","journal-title":"J. Am. Soc. Inf. Sci."},{"key":"27_CR32","doi-asserted-by":"crossref","unstructured":"Macdonald, C., Ounis, I.: Voting for candidates: adapting data fusion techniques for an expert search task. In: Proceedings of the 15th ACM International Conference on Information and Knowledge management, pp. 387\u2013396. ACM (2006)","DOI":"10.1145\/1183614.1183671"},{"key":"27_CR33","doi-asserted-by":"publisher","first-page":"193","DOI":"10.1093\/jamia\/ocv044","volume":"23","author":"IJ Marshall","year":"2015","unstructured":"Marshall, I.J., Kuiper, J., Wallace, B.C.: RobotReviewer: evaluation of a system for automatically assessing bias in clinical trials. J. Am. Med. Inform. Assoc. 23, 193\u2013201 (2015)","journal-title":"J. Am. Med. Inform. Assoc."},{"key":"27_CR34","doi-asserted-by":"publisher","first-page":"30","DOI":"10.1016\/j.jclinepi.2018.12.015","volume":"109","author":"IJ Marshall","year":"2019","unstructured":"Marshall, I.J., Marshall, R., Wallace, B.C., Brassey, J., Thomas, J.: Rapid reviews may produce different results to systematic reviews: a meta-epidemiological study. J. Clin. Epidemiol. 109, 30\u201341 (2019)","journal-title":"J. Clin. Epidemiol."},{"issue":"1","key":"27_CR35","first-page":"74","volume":"93","author":"J McGowan","year":"2005","unstructured":"McGowan, J., Sampson, M.: Systematic reviews need systematic searchers (IRP). J. Med. Libr. Assoc. 93(1), 74 (2005)","journal-title":"J. Med. Libr. Assoc."},{"key":"27_CR36","unstructured":"Minas, A., Lagopoulos, A., Tsoumakas, G.: Aristotle university\u2019s approach to the technologically assisted reviews in empirical medicine task of the 2018 CLEF eHealth lab. In: CLEF (Working Notes) (2018)"},{"key":"27_CR37","first-page":"242","volume":"51","author":"M Miwa","year":"2014","unstructured":"Miwa, M., Thomas, J., O\u2019Mara-Eves, A., Ananiadou, S.: Reducing systematic review workload through certainty-based screening. JBI 51, 242\u2013253 (2014)","journal-title":"JBI"},{"key":"27_CR38","unstructured":"Norman, C., Leeflang, M., N\u00e9v\u00e9ol, A.: LIMSI@CLEF eHealth 2018 task 2: technology assisted reviews by stacking active and static learning. In: CLEF (Working Notes) (2018)"},{"key":"27_CR39","unstructured":"Norman12, C., Leeflang, M., N\u00e9v\u00e9ol, A.: LIMSI@CLEF eHealth 2017 task 2: logistic regression for automatic article ranking (2017)"},{"issue":"1","key":"27_CR40","doi-asserted-by":"publisher","first-page":"5","DOI":"10.1186\/2046-4053-4-5","volume":"4","author":"A O\u2019Mara-Eves","year":"2015","unstructured":"O\u2019Mara-Eves, A., Thomas, J., McNaught, J., Miwa, M., Ananiadou, S.: Using text mining for study identification in systematic reviews: a systematic review of current approaches. Syst. Rev. 4(1), 5 (2015)","journal-title":"Syst. Rev."},{"issue":"6","key":"27_CR41","doi-asserted-by":"publisher","first-page":"365","DOI":"10.1002\/asi.4630330603","volume":"33","author":"T Radecki","year":"1982","unstructured":"Radecki, T.: A probabilistic approach to information retrieval in systems with boolean search request formulations. J. Am. Soc. Inf. Sci. 33(6), 365\u2013370 (1982)","journal-title":"J. Am. Soc. Inf. Sci."},{"key":"27_CR42","unstructured":"Salton, G., Fox, E.A., Wu, H.: Extended Boolean information retrieval. Technical report. Cornell University (1982)"},{"issue":"1","key":"27_CR43","doi-asserted-by":"publisher","first-page":"168","DOI":"10.1017\/S0266462303000163","volume":"19","author":"I Savoie","year":"2003","unstructured":"Savoie, I., Helmer, D., Green, C.J., Kazanjian, A.: Beyond medline: reducing bias through extended systematic review search. Int. J. Technol. Assess. Health Care 19(1), 168\u2013178 (2003)","journal-title":"Int. J. Technol. Assess. Health Care"},{"key":"27_CR44","unstructured":"Sayers, E.: A general introduction to the e-utilities. Entrez Programming Utilities Help [Internet]. National Center for Biotechnology Information, Bethesda (2010)"},{"key":"27_CR45","doi-asserted-by":"crossref","unstructured":"Scells, H., Locke, D., Zuccon, G.: An information retrieval experiment framework for domain specific applications. In: The 41st International ACM SIGIR Conference on Research & Development in Information Retrieval (2018)","DOI":"10.1145\/3209978.3210167"},{"key":"27_CR46","doi-asserted-by":"crossref","unstructured":"Scells, H., Zuccon, G.: Generating better queries for systematic reviews. In: Proceedings of the 41st Annual International ACM SIGIR Conference on Research & Development in Information Retrieval, SIGIR 2018 (2018)","DOI":"10.1145\/3209978.3210020"},{"key":"27_CR47","unstructured":"Scells, H., Zuccon, G., Deacon, A., Koopman, B.: QUT ielab at CLEF eHealth 2017 technology assisted reviews track: initial experiments with learning to rank. In: CEUR Workshop Proceedings: Working Notes of CLEF 2017: Conference and Labs of the Evaluation Forum, vol. 1866, pp. Paper-98. CEUR Workshop Proceedings (2017)"},{"key":"27_CR48","doi-asserted-by":"crossref","unstructured":"Scells, H., Zuccon, G., Koopman, B.: Automatic Boolean query refinement for systematic review literature search. In: The World Wide Web Conference, pp. 1646\u20131656 (2019)","DOI":"10.1145\/3308558.3313544"},{"key":"27_CR49","doi-asserted-by":"crossref","unstructured":"Shaw, J.A., Fox, E.A.: Combination of multiple searches. NIST Special Publication SP, pp. 105\u2013105 (1995)","DOI":"10.6028\/NIST.SP.500-225.vpi"},{"issue":"4","key":"27_CR50","doi-asserted-by":"publisher","first-page":"19","DOI":"10.1145\/2926790","volume":"34","author":"A Shtok","year":"2016","unstructured":"Shtok, A., Kurland, O., Carmel, D.: Query performance prediction using reference lists. ACM Trans. Inf. Syst. (TOIS) 34(4), 19 (2016)","journal-title":"ACM Trans. Inf. Syst. (TOIS)"},{"key":"27_CR51","unstructured":"Singh, G., Marshall, I., Thomas, J., Wallace, B.: Identifying diagnostic test accuracy publications using a deep model. In: CEUR Workshop Proceedings, vol. 1866. CEUR Workshop Proceedings (2017)"},{"key":"27_CR52","unstructured":"Singh, J., Thomas, L.: IIIT-H at CLEF eHealth 2017 task 2: technologically assisted reviews in empirical medicine. In: CLEF (Working Notes) (2017)"},{"issue":"3","key":"27_CR53","doi-asserted-by":"publisher","first-page":"151","DOI":"10.1023\/A:1009980820262","volume":"1","author":"CC Vogt","year":"1999","unstructured":"Vogt, C.C., Cottrell, G.W.: Fusion via a linear combination of scores. Inf. Retrieval 1(3), 151\u2013173 (1999)","journal-title":"Inf. Retrieval"},{"issue":"5","key":"27_CR54","first-page":"7","volume":"4","author":"H Wu","year":"2018","unstructured":"Wu, H., Wang, T., Chen, J., Chen, S., Hu, Q., He, L.: ECNU at 2018 eHealth task 2: technologically assisted reviews in empirical medicine. Methods 4(5), 7 (2018)","journal-title":"Methods"},{"key":"27_CR55","unstructured":"Yu, Z., Menzies, T.: Data balancing for technologically assisted reviews: undersampling or reweighting. In: CLEF (Working Notes) (2017)"},{"key":"27_CR56","doi-asserted-by":"crossref","unstructured":"Zou, J., Li, D., Kanoulas, E.: Technology assisted reviews: finding the last few relevant documents by asking yes\/no questions to reviewers. In: The 41st International ACM SIGIR Conference on Research & Development in Information Retrieval, pp. 949\u2013952 (2018)","DOI":"10.1145\/3209978.3210102"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-45439-5_27","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,8,3]],"date-time":"2024-08-03T20:45:01Z","timestamp":1722717901000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-45439-5_27"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020]]},"ISBN":["9783030454388","9783030454395"],"references-count":56,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-45439-5_27","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2020]]},"assertion":[{"value":"8 April 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Lisbon","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Portugal","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2020","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"14 April 2020","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"17 April 2020","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"42","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2020","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecir2020.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"457","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"55","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"46","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"12% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Also included: 8 reproducibility papers, 10 demonstration papers, 12 CLEF organizers lab track papers, 7 doctoral consortium papers, 4 workshops, 3 tutorials. Due to the COVID-19 pandemic, this conference was held virtually.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}