{"id":"https://openalex.org/W2892228078","doi":"https://doi.org/10.18653/v1/k18-1031","title":"Sentence-Level Fluency Evaluation: References Help, But Can Be Spared!","display_name":"Sentence-Level Fluency Evaluation: References Help, But Can Be Spared!","publication_year":2018,"publication_date":"2018-01-01","ids":{"openalex":"https://openalex.org/W2892228078","doi":"https://doi.org/10.18653/v1/k18-1031","mag":"2892228078"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/k18-1031","pdf_url":"https://www.aclweb.org/anthology/K18-1031.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"type":"article","type_crossref":"proceedings-article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://www.aclweb.org/anthology/K18-1031.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5082874907","display_name":"Katharina Kann","orcid":null},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]},{"id":"https://openalex.org/I4210100430","display_name":"Google (Switzerland)","ror":"https://ror.org/014f9c269","country_code":"CH","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210100430","https://openalex.org/I4210128969"]}],"countries":["CH","US"],"is_corresponding":false,"raw_author_name":"Katharina Kann","raw_affiliation_strings":["Center for Data Science New York University New York, USA","Google Research Berlin, Germany","Google Research Zurich, Switzerland"],"affiliations":[{"raw_affiliation_string":"Center for Data Science New York University New York, USA","institution_ids":["https://openalex.org/I57206974"]},{"raw_affiliation_string":"Google Research Zurich, Switzerland","institution_ids":["https://openalex.org/I4210100430"]},{"raw_affiliation_string":"Google Research Berlin, Germany","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004258058","display_name":"Sascha Rothe","orcid":null},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]},{"id":"https://openalex.org/I4210100430","display_name":"Google (Switzerland)","ror":"https://ror.org/014f9c269","country_code":"CH","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210100430","https://openalex.org/I4210128969"]}],"countries":["CH","US"],"is_corresponding":false,"raw_author_name":"Sascha Rothe","raw_affiliation_strings":["Center for Data Science New York University New York, USA","Google Research Berlin, Germany","Google Research Zurich, Switzerland"],"affiliations":[{"raw_affiliation_string":"Center for Data Science New York University New York, USA","institution_ids":["https://openalex.org/I57206974"]},{"raw_affiliation_string":"Google Research Zurich, Switzerland","institution_ids":["https://openalex.org/I4210100430"]},{"raw_affiliation_string":"Google Research Berlin, Germany","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5037657908","display_name":"Katja Filippova","orcid":"https://orcid.org/0009-0007-5308-0904"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]},{"id":"https://openalex.org/I4210100430","display_name":"Google (Switzerland)","ror":"https://ror.org/014f9c269","country_code":"CH","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210100430","https://openalex.org/I4210128969"]}],"countries":["CH","US"],"is_corresponding":false,"raw_author_name":"Katja Filippova","raw_affiliation_strings":["Center for Data Science New York University New York, USA","Google Research Berlin, Germany","Google Research Zurich, Switzerland"],"affiliations":[{"raw_affiliation_string":"Center for Data Science New York University New York, USA","institution_ids":["https://openalex.org/I57206974"]},{"raw_affiliation_string":"Google Research Zurich, Switzerland","institution_ids":["https://openalex.org/I4210100430"]},{"raw_affiliation_string":"Google Research Berlin, Germany","institution_ids":[]}]}],"institution_assertions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.099,"has_fulltext":false,"cited_by_count":59,"citation_normalized_percentile":{"value":0.999897,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":98},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9999,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9977,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.61367524}],"concepts":[{"id":"https://openalex.org/C2777413886","wikidata":"https://www.wikidata.org/wiki/Q3276013","display_name":"Fluency","level":2,"score":0.8331934},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.763944},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6371291},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6213493},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.61367524},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6052247},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.5678093},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.5618342},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.49331275},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.4775751},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4664777},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3277763},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.18374279},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.18220374},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/k18-1031","pdf_url":"https://www.aclweb.org/anthology/K18-1031.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/1809.08731","pdf_url":"https://arxiv.org/pdf/1809.08731","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/k18-1031","pdf_url":"https://www.aclweb.org/anthology/K18-1031.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"sustainable_development_goals":[{"score":0.84,"id":"https://metadata.un.org/sdg/4","display_name":"Quality education"}],"grants":[],"datasets":[],"versions":[],"referenced_works_count":51,"referenced_works":["https://openalex.org/W108011198","https://openalex.org/W1489525520","https://openalex.org/W1498416817","https://openalex.org/W1501668372","https://openalex.org/W1522301498","https://openalex.org/W1552182777","https://openalex.org/W1581097513","https://openalex.org/W1664028424","https://openalex.org/W1828773426","https://openalex.org/W1964326564","https://openalex.org/W2001565536","https://openalex.org/W2024988999","https://openalex.org/W2037789405","https://openalex.org/W2064675550","https://openalex.org/W2068390867","https://openalex.org/W2098297786","https://openalex.org/W2101105183","https://openalex.org/W2101234009","https://openalex.org/W2108325777","https://openalex.org/W2118119027","https://openalex.org/W2121879602","https://openalex.org/W2123891489","https://openalex.org/W2128856065","https://openalex.org/W2138761920","https://openalex.org/W2151222319","https://openalex.org/W2170716495","https://openalex.org/W2250591774","https://openalex.org/W2251930319","https://openalex.org/W2329847998","https://openalex.org/W2341457423","https://openalex.org/W2402268235","https://openalex.org/W2470324779","https://openalex.org/W2508316494","https://openalex.org/W2511538013","https://openalex.org/W2525778437","https://openalex.org/W2531882892","https://openalex.org/W2557249038","https://openalex.org/W2571713365","https://openalex.org/W2613253298","https://openalex.org/W2730712696","https://openalex.org/W2745039414","https://openalex.org/W2915756181","https://openalex.org/W2916548775","https://openalex.org/W2962875960","https://openalex.org/W2963541170","https://openalex.org/W2963746123","https://openalex.org/W2963903950","https://openalex.org/W2964121744","https://openalex.org/W2964130895","https://openalex.org/W4297792382","https://openalex.org/W4302417898"],"related_works":["https://openalex.org/W595497825","https://openalex.org/W4236323843","https://openalex.org/W3174418441","https://openalex.org/W2975827637","https://openalex.org/W2944691285","https://openalex.org/W2400151637","https://openalex.org/W2365169615","https://openalex.org/W2354089692","https://openalex.org/W2002616876","https://openalex.org/W1970538215"],"abstract_inverted_index":{"Motivated":[0],"by":[1],"recent":[2],"findings":[3],"on":[4,79,124],"the":[5,35,62,101],"probabilistic":[6],"modeling":[7],"of":[8,29,64,83,98,103],"acceptability":[9],"judgments,":[10],"we":[11,87],"propose":[12],"syntactic":[13],"log-odds":[14],"ratio":[15],"(SLOR),":[16],"a":[17,23,42,48,71,80,90,95,111],"normalized":[18],"language":[19,31,51],"model":[20],"score,":[21],"as":[22],"metric":[24,92],"for":[25],"referenceless":[26,68],"fluency":[27,77],"evaluation":[28],"natural":[30,96],"generation":[32],"output":[33],"at":[34],"sentence":[36],"level.":[37],"We":[38,106],"further":[39],"introduce":[40],"WPSLOR,":[41],"novel":[43],"WordPiece-based":[44],"version,":[45],"which":[46,93],"harnesses":[47],"more":[49],"compact":[50],"model.":[52],"Even":[53],"though":[54],"word-overlap":[55],"metrics":[56],"like":[57],"ROUGE":[58],"are":[59],"computed":[60],"with":[61,75,115],"help":[63],"hand-written":[65],"references,":[66],"our":[67],"methods":[69],"obtain":[70],"significantly":[72,112],"higher":[73,113],"correlation":[74,114],"human":[76,116],"scores":[78],"benchmark":[81],"dataset":[82],"compressed":[84],"sentences.":[85],"Finally,":[86],"present":[88],"ROUGE-LM,":[89],"reference-based":[91],"is":[94],"extension":[97],"WPSLOR":[99,123],"to":[100],"case":[102],"available":[104],"references.":[105],"show":[107],"that":[108],"ROUGE-LM":[109],"yields":[110],"judgments":[117],"than":[118],"all":[119],"baseline":[120],"metrics,":[121],"including":[122],"its":[125],"own.":[126]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2892228078","counts_by_year":[{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":9},{"year":2022,"cited_by_count":11},{"year":2021,"cited_by_count":9},{"year":2020,"cited_by_count":16},{"year":2019,"cited_by_count":8},{"year":2018,"cited_by_count":1}],"updated_date":"2024-12-07T05:28:19.306841","created_date":"2018-09-27"}