{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2023,10,7]],"date-time":"2023-10-07T02:44:37Z","timestamp":1696646677664},"reference-count":24,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2012,9,22]],"date-time":"2012-09-22T00:00:00Z","timestamp":1348272000000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Lang Resources & Evaluation"],"published-print":{"date-parts":[[2013,6]]},"DOI":"10.1007\/s10579-012-9198-8","type":"journal-article","created":{"date-parts":[[2012,9,21]],"date-time":"2012-09-21T14:55:25Z","timestamp":1348239325000},"page":"337-369","source":"Crossref","is-referenced-by-count":14,"title":["Analyzing the capabilities of crowdsourcing services for text summarization"],"prefix":"10.1007","volume":"47","author":[{"given":"Elena","family":"Lloret","sequence":"first","affiliation":[]},{"given":"Laura","family":"Plaza","sequence":"additional","affiliation":[]},{"given":"Ahmet","family":"Aker","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2012,9,22]]},"reference":[{"key":"9198_CR1","unstructured":"Aker, A., & Gaizauskas, R. (2010). Model summaries for location-related images. In Proceedings of the seventh conference on international language resources and evaluation (LREC\u201910), European Language Resources Association (ELRA), Valletta, Malta."},{"key":"9198_CR2","unstructured":"Aker, A., El-Haj, M., Albakour, M. D., & Kruschwitz, U. (2012). Assessing crowdsourcing quality through objective tasks. In Proceedings of LREC."},{"issue":"2","key":"9198_CR3","doi-asserted-by":"crossref","first-page":"9","DOI":"10.1145\/1480506.1480508","volume":"42","author":"O. Alonso","year":"2008","unstructured":"Alonso, O., Rose, D. E., & Stewart, B. (2008). Crowdsourcing for relevance evaluation. SIGIR Forum 42(2), 9\u201315.","journal-title":"SIGIR Forum"},{"key":"9198_CR4","unstructured":"Buzek, O., Resnik, P., & Bederson, B. B. (2010). Error driven paraphrase annotation using mechanical turk. In Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with Amazons mechanical turk."},{"key":"9198_CR5","doi-asserted-by":"crossref","unstructured":"Callison-Burch, C. (2009). Fast, cheap, and creative: Evaluating translation quality using amazon\u2019s mechanical turk. In Proceedings of the 2009 conference on empirical methods in natural language processing (pp. 286\u2013295).","DOI":"10.3115\/1699510.1699548"},{"key":"9198_CR6","unstructured":"Dakka, W., & Ipeirotis, P.G. (2008). Automatic extraction of useful facet hierarchies from text databases. In ICDE \u201908: Proceedings of the 2008 IEEE 24th international conference on data engineering (pp. 466\u2013475)."},{"key":"9198_CR7","unstructured":"El-Haj, M., Kruschwitz, U., & Fox, C. (2010). Using mechanicel turk to create a corpus of arabic summaries. In Proceedings of the seventh conference on international language resources and evaluation (LREC\u201910), Valletta, Malta."},{"key":"9198_CR8","unstructured":"Feng, D., Besana, S., & Zajac, R. (2009). Acquiring High Quality Non-expert Knowledge from On-demand Workforce. In Proceedings of the 2009 workshop on the people\u2019s web meets NLP: Collaboratively constructed semantic resources (pp. 51\u201356). Association for computational linguistics, Morristown, NJ, People\u2019s Web \u201909. http:\/\/portal.acm.org\/citation.cfm?id=1699765.1699773 ."},{"key":"9198_CR9","unstructured":"Finin, T., Murnane, W., Karandikar, A., Keller, N., Martineau, J., & Dredze, M. (2010). Annotating named entities in twitter data with crowdsourcing. In CSLDAMT \u201910: Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with Amazon\u2019s mechanical turk, association for computational linguistics (pp. 80\u201388). Morristown, NJ"},{"issue":"3","key":"9198_CR10","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/1410358.1410359","volume":"5","author":"G. Giannakopoulos","year":"2008","unstructured":"Giannakopoulos, G., Karkaletsis, V., Vouros, G., & Stamatopoulos, P. (2008). Summarization system evaluation revisited: N-gram graphs. ACM transactions on speech and language processing 5(3), 1\u201339.","journal-title":"ACM Transactions on Speech and Language Processing"},{"key":"9198_CR11","unstructured":"Gillick, D., & Liu, Y. (2010). Non-expert evaluation of summarization systems is risky. In Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with Amazons mechanical turk."},{"key":"9198_CR12","unstructured":"Heilman, M., & Smith, N. (2010). Rating computer-generated questions with mechanical turk. In Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with Amazon\u2019s mechanical turk (pp. 00\u201300)."},{"key":"9198_CR13","unstructured":"Hsueh, P. Y., Melville, P., & Sindhwani, V. (2009). Data quality from crowdsourcing: a study of annotation selection criteria. In HLT \u201909: Proceedings of the NAACL HLT 2009 workshop on active learning for natural language processing (pp. 27\u201335)."},{"key":"9198_CR14","unstructured":"Kaisser, M., Hearst, M. A., & Lowe, J. B. (2008). Improving search results quality by customizing summary lengths. In Proceedings of ACL-08: HLT, Columbus, Ohio (pp. 701\u2013709)."},{"key":"9198_CR15","doi-asserted-by":"crossref","unstructured":"Kittur, A., Chi, E. H., & Suh, B. (2008). Crowdsourcing user studies with mechanical turk. In Proceeding of the twenty-sixth annual SIGCHI conference on human factors in computing systems (pp. 453\u2013456).","DOI":"10.1145\/1357054.1357127"},{"key":"9198_CR16","unstructured":"Le, J., Edmonds, A., Hester, V., & Biewald, L. (2010). Ensuring quality in crowdsourced search relevance evaluation: The effects of training question distribution. In Proceedings of the ACM SIGIR 2010 workshop on crowdsourcing for search evaluation (CSE 2010) (pp. 17\u201320), Switzerland: Geneva."},{"key":"9198_CR17","unstructured":"Lin, C. Y. (2004). ROUGE: a package for automatic evaluation of summaries. In Proceedings of ACL text summarization workshop (pp. 74\u201381)."},{"key":"9198_CR18","doi-asserted-by":"crossref","first-page":"100","DOI":"10.1145\/1809400.1809422","volume":"11","author":"W. Mason","year":"2010","unstructured":"Mason, W., & Watts, D.J. (2010) Financial incentives and the \u201cPerformance of Crowds\u201d. SIGKDD Explor Newsl 11, 100\u2013108.","journal-title":"SIGKDD Explor Newsl"},{"key":"9198_CR19","unstructured":"Nakov, P. (2008). Noun compound interpretation using paraphrasing verbs: Feasibility study. In AIMSA \u201908: Proceedings of the 13th international conference on artificial intelligence (pp. 103\u2013117)."},{"key":"9198_CR20","unstructured":"Negri, M., & Mehdad, Y. (2010). Creating a bi-lingual entailment corpus through translations with mechanical turk: $$\\hbox{\\$}100$$ for a 10-day rush. In CSLDAMT \u201910: Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with Amazon\u2019s mechanical turk (pp. 212\u2013216). Morristown, NJ: Association for Computational Linguistics."},{"key":"9198_CR21","unstructured":"Sheng, V. S., Provost, F., & Ipeirotis, P. G. (2008). Get another label? improving data quality and data mining using multiple, noisy labelers. In KDD \u201908: Proceedings of the 14th ACM SIGKDD international conference on knowledge discovery and data mining (pp. 614\u2013622)."},{"key":"9198_CR22","unstructured":"Snow, R., O\u2019Connor, B., Jurafsky, D., & Ng, A. (2008). Cheap and fast \u2013 but is it good? evaluating non-expert annotations for natural language tasks. In Proceedings of the 2008 conference on empirical methods in natural language processing (pp. 254\u2013263), Honolulu, Hawaii."},{"key":"9198_CR23","doi-asserted-by":"crossref","unstructured":"Sorokin, A., & Forsyth, D. (2008). Utility data annotation with amazon mechanical turk. pp. 1\u20138.","DOI":"10.1109\/CVPRW.2008.4562953"},{"key":"9198_CR24","doi-asserted-by":"crossref","unstructured":"Tang, J., & Sanderson, M. (2010). Evaluation and user preference study on spatial diversity. In Proceedings of the 32nd European conference on information retrieval (ECIR).","DOI":"10.1007\/978-3-642-12275-0_18"}],"container-title":["Language Resources and Evaluation"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10579-012-9198-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10579-012-9198-8\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10579-012-9198-8","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,7,4]],"date-time":"2019-07-04T03:54:39Z","timestamp":1562212479000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10579-012-9198-8"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2012,9,22]]},"references-count":24,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2013,6]]}},"alternative-id":["9198"],"URL":"https:\/\/doi.org\/10.1007\/s10579-012-9198-8","relation":{},"ISSN":["1574-020X","1574-0218"],"issn-type":[{"value":"1574-020X","type":"print"},{"value":"1574-0218","type":"electronic"}],"subject":[],"published":{"date-parts":[[2012,9,22]]}}}