{"id":"https://openalex.org/W2534147738","doi":"https://doi.org/10.1145/2983323.2983694","title":"Optimizing Nugget Annotations with Active Learning","display_name":"Optimizing Nugget Annotations with Active Learning","publication_year":2016,"publication_date":"2016-10-24","ids":{"openalex":"https://openalex.org/W2534147738","doi":"https://doi.org/10.1145/2983323.2983694","mag":"2534147738"},"language":"en","primary_location":{"is_oa":false,"landing_page_url":"https://doi.org/10.1145/2983323.2983694","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false},"type":"article","type_crossref":"proceedings-article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5080828705","display_name":"Gaurav Baruah","orcid":"https://orcid.org/0000-0002-4992-0524"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"funder","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Gaurav Baruah","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100392954","display_name":"Haotian Zhang","orcid":"https://orcid.org/0000-0001-7786-389X"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"funder","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Haotian Zhang","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061857595","display_name":"Rakesh Guttikonda","orcid":null},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"funder","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Rakesh Guttikonda","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082997975","display_name":"Jimmy Lin","orcid":"https://orcid.org/0000-0002-0661-7189"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"funder","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jimmy Lin","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049222552","display_name":"Mark D. Smucker","orcid":"https://orcid.org/0000-0003-4968-6405"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"funder","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Mark D. Smucker","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028031104","display_name":"Olga Vechtomova","orcid":"https://orcid.org/0000-0001-7371-0837"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"funder","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Olga Vechtomova","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]}],"institution_assertions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.546,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.649713,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":83,"max":84},"biblio":{"volume":null,"issue":null,"first_page":"2359","last_page":"2364"},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9991,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9964,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.51011235}],"concepts":[{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.8903929},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7648393},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.73526794},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.64579034},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5153603},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.51011235},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.50910896},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.5074579},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.46388847},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45693153},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.43278548},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.074507385},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"is_oa":false,"landing_page_url":"https://doi.org/10.1145/2983323.2983694","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Decent work and economic growth","score":0.65,"id":"https://metadata.un.org/sdg/8"}],"grants":[{"funder":"https://openalex.org/F4320309327","funder_display_name":"Google","award_id":null},{"funder":"https://openalex.org/F4320322676","funder_display_name":"University of Waterloo","award_id":null},{"funder":"https://openalex.org/F4320334593","funder_display_name":"Natural Sciences and Engineering Research Council of Canada","award_id":null}],"datasets":[],"versions":[],"referenced_works_count":22,"referenced_works":["https://openalex.org/W135190683","https://openalex.org/W1493399377","https://openalex.org/W1523019309","https://openalex.org/W1700671489","https://openalex.org/W1977570008","https://openalex.org/W1986827153","https://openalex.org/W1988613011","https://openalex.org/W1989220366","https://openalex.org/W2017633929","https://openalex.org/W2029075138","https://openalex.org/W2037140704","https://openalex.org/W2066703653","https://openalex.org/W2096623622","https://openalex.org/W2100520226","https://openalex.org/W2138090731","https://openalex.org/W2145410165","https://openalex.org/W2185884126","https://openalex.org/W219090214","https://openalex.org/W2341384651","https://openalex.org/W2404416613","https://openalex.org/W2604321711","https://openalex.org/W2918053736"],"related_works":["https://openalex.org/W4389760904","https://openalex.org/W4367460280","https://openalex.org/W3148229873","https://openalex.org/W2405355225","https://openalex.org/W2366403280","https://openalex.org/W2358294942","https://openalex.org/W2241813887","https://openalex.org/W2150160875","https://openalex.org/W2091301346","https://openalex.org/W1495108544"],"abstract_inverted_index":{"Nugget-based":[0],"evaluations,":[1],"such":[2],"as":[3,34],"those":[4],"deployed":[5,119],"in":[6,25,52,120],"the":[7,50,64,67,76],"TREC":[8,97],"Temporal":[9],"Summarization":[10],"and":[11,86],"Question":[12],"Answering":[13],"tracks,":[14],"require":[15],"human":[16],"assessors":[17],"to":[18,59,83],"determine":[19],"whether":[20],"a":[21,26,70,109],"nugget":[22,35,79],"is":[23,37,81],"present":[24,43],"given":[27,110],"piece":[28],"of":[29,112],"text.":[30],"This":[31],"process,":[32],"known":[33],"annotation,":[36],"labor-intensive.":[38],"In":[39],"this":[40],"paper,":[41],"we":[42,87],"two":[44],"active":[45],"learning":[46],"techniques":[47,103],"that":[48,66,78,101,116],"prioritize":[49],"sequence":[51],"which":[53],"candidate":[54],"nugget/sentence":[55],"pairs":[56],"are":[57,117],"presented":[58],"an":[60],"assessor,":[61],"based":[62],"on":[63,75],"likelihood":[65],"sentence":[68],"contains":[69],"nugget.":[71],"Our":[72],"approach":[73],"builds":[74],"recognition":[77],"annotation":[80],"similar":[82],"high-recall":[84],"retrieval,":[85],"adapt":[88],"proven":[89],"existing":[90,96],"solutions.":[91],"Simulation":[92],"experiments":[93],"with":[94],"four":[95],"test":[98],"collections":[99],"show":[100],"our":[102],"yield":[104],"far":[105],"more":[106],"matches":[107],"for":[108],"level":[111],"effort":[113],"than":[114],"baselines":[115],"typically":[118],"previous":[121],"nugget-based":[122],"evaluations.":[123]},"abstract_inverted_index_v3":null,"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2534147738","counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":2}],"updated_date":"2025-04-20T20:48:01.456586","created_date":"2016-10-28"}