{"id":"https://openalex.org/W4399489935","doi":"https://doi.org/10.1016/j.eswa.2024.124456","title":"Factual consistency evaluation of summarization in the Era of large language models","display_name":"Factual consistency evaluation of summarization in the Era of large language models","publication_year":2024,"publication_date":"2024-06-10","ids":{"openalex":"https://openalex.org/W4399489935","doi":"https://doi.org/10.1016/j.eswa.2024.124456"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.eswa.2024.124456","pdf_url":null,"source":{"id":"https://openalex.org/S13144211","display_name":"Expert Systems with Applications","issn_l":"0957-4174","issn":["0957-4174","1873-6793"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true},"type":"article","type_crossref":"journal-article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1016/j.eswa.2024.124456","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075367840","display_name":"Zheheng Luo","orcid":"https://orcid.org/0000-0001-8246-5511"},"institutions":[{"id":"https://openalex.org/I28407311","display_name":"University of Manchester","ror":"https://ror.org/027m9bs27","country_code":"GB","type":"education","lineage":["https://openalex.org/I28407311"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Zheheng Luo","raw_affiliation_strings":["The University of Manchester, Oxford Road, Manchester, M13 9PL, United Kingdom"],"affiliations":[{"raw_affiliation_string":"The University of Manchester, Oxford Road, Manchester, M13 9PL, United Kingdom","institution_ids":["https://openalex.org/I28407311"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101868563","display_name":"Qianqian Xie","orcid":"https://orcid.org/0000-0002-9588-7454"},"institutions":[],"countries":["SG"],"is_corresponding":true,"raw_author_name":"Qianqian Xie","raw_affiliation_strings":["The Fin AI, Singapore"],"affiliations":[{"raw_affiliation_string":"The Fin AI, Singapore","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5077976343","display_name":"Sophia Ananiadou","orcid":"https://orcid.org/0000-0002-4097-9191"},"institutions":[{"id":"https://openalex.org/I28407311","display_name":"University of Manchester","ror":"https://ror.org/027m9bs27","country_code":"GB","type":"education","lineage":["https://openalex.org/I28407311"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Sophia Ananiadou","raw_affiliation_strings":["The University of Manchester, Oxford Road, Manchester, M13 9PL, United Kingdom"],"affiliations":[{"raw_affiliation_string":"The University of Manchester, Oxford Road, Manchester, M13 9PL, United Kingdom","institution_ids":["https://openalex.org/I28407311"]}]}],"institution_assertions":[],"countries_distinct_count":2,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5101868563"],"corresponding_institution_ids":[],"apc_list":{"value":3220,"currency":"USD","value_usd":3220,"provenance":"doaj"},"apc_paid":{"value":3220,"currency":"USD","value_usd":3220,"provenance":"doaj"},"fwci":4.598,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.79379,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":97},"biblio":{"volume":"254","issue":null,"first_page":"124456","last_page":"124456"},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9992,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9862,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[],"concepts":[{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.7997565},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7236315},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5190097},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44009745},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.36890626}],"mesh":[],"locations_count":1,"locations":[{"is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.eswa.2024.124456","pdf_url":null,"source":{"id":"https://openalex.org/S13144211","display_name":"Expert Systems with Applications","issn_l":"0957-4174","issn":["0957-4174","1873-6793"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.eswa.2024.124456","pdf_url":null,"source":{"id":"https://openalex.org/S13144211","display_name":"Expert Systems with Applications","issn_l":"0957-4174","issn":["0957-4174","1873-6793"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality education","score":0.65}],"grants":[],"datasets":[],"versions":[],"referenced_works_count":24,"referenced_works":["https://openalex.org/W2093157872","https://openalex.org/W2111722073","https://openalex.org/W2599674900","https://openalex.org/W2606974598","https://openalex.org/W2607303097","https://openalex.org/W2750779823","https://openalex.org/W2888211956","https://openalex.org/W2970419734","https://openalex.org/W2996264288","https://openalex.org/W3034999214","https://openalex.org/W3100258764","https://openalex.org/W3159259047","https://openalex.org/W3170432046","https://openalex.org/W3213990450","https://openalex.org/W4205477024","https://openalex.org/W4233907442","https://openalex.org/W4288089799","https://openalex.org/W4292905395","https://openalex.org/W4297435087","https://openalex.org/W4387321091","https://openalex.org/W4389518784","https://openalex.org/W4389519239","https://openalex.org/W4389520387","https://openalex.org/W4389520749"],"related_works":["https://openalex.org/W4396701345","https://openalex.org/W4396696052","https://openalex.org/W4391375266","https://openalex.org/W3204019825","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2382290278","https://openalex.org/W2376932109","https://openalex.org/W2358668433","https://openalex.org/W2001405890"],"abstract_inverted_index":{"Factual":[0],"inconsistency":[1],"with":[2],"source":[3],"documents":[4],"in":[5,31,40,46,49,196],"automatically":[6],"generated":[7],"summaries":[8,106],"can":[9],"lead":[10],"to":[11,77,192],"misinformation":[12],"or":[13],"pose":[14],"risks.":[15],"Existing":[16],"factual":[17,194],"consistency":[18],"(FC)":[19],"metrics":[20],"are":[21,75,190],"constrained":[22],"by":[23,100,113],"their":[24,44,66],"performance,":[25],"efficiency,":[26],"and":[27,126,129,138,174,187],"explainability.":[28],"Recent":[29],"advances":[30],"Large":[32],"language":[33],"models":[34,147],"(LLMs)":[35],"have":[36],"demonstrated":[37],"remarkable":[38],"potential":[39,159],"text":[41],"evaluation":[42,73,123],"but":[43],"effectiveness":[45],"assessing":[47],"FC":[48,72,87,112,122,204],"summarization":[50],"remains":[51],"underexplored.":[52],"Prior":[53],"research":[54],"has":[55],"mostly":[56],"focused":[57],"on":[58,82,90,149,180],"proprietary":[59,146],"LLMs,":[60],"leaving":[61],"essential":[62],"factors":[63],"that":[64,144,183],"affect":[65],"assessment":[67],"capabilities":[68],"unexplored.":[69],"Additionally,":[70],"current":[71],"benchmarks":[74],"restricted":[76],"news":[78,125],"articles,":[79],"casting":[80],"doubt":[81],"the":[83,86,98,131,150,162],"generality":[84],"of":[85,104,107,133,164],"methods":[88,186],"tested":[89],"them.":[91],"In":[92],"this":[93],"paper,":[94],"we":[95,117],"first":[96],"address":[97],"gap":[99],"introducing":[101],"TreatFact\u2014a":[102],"dataset":[103],"LLM-generated":[105],"clinical":[108,127,197],"texts,":[109],"annotated":[110],"for":[111,121,160,203],"domain":[114],"experts.":[115],"Moreover,":[116],"benchmark":[118],"11":[119],"LLMs":[120,153,166],"across":[124],"domains":[128],"analyse":[130],"impact":[132],"model":[134,169],"size,":[135,170],"prompts,":[136],"pre-training":[137,172],"fine-tuning":[139,177],"data.":[140,178],"Our":[141],"findings":[142],"reveal":[143],"despite":[145],"prevailing":[148],"task,":[151],"open-source":[152,165],"lag":[154],"behind.":[155],"Nevertheless,":[156],"there":[157],"is":[158],"enhancing":[161],"performance":[163],"through":[167],"increasing":[168],"expanding":[171],"data,":[173],"developing":[175],"well-curated":[176],"Experiments":[179],"TreatFact":[181],"suggest":[182],"both":[184],"previous":[185],"LLM-based":[188],"evaluators":[189],"unable":[191],"capture":[193],"inconsistencies":[195],"summaries,":[198],"posing":[199],"a":[200],"new":[201],"challenge":[202],"evaluation.":[205]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4399489935","counts_by_year":[{"year":2024,"cited_by_count":3}],"updated_date":"2025-01-16T20:22:32.007815","created_date":"2024-06-11"}