{"id":"https://openalex.org/W4400064797","doi":"https://doi.org/10.48550/arxiv.2406.17274","title":"Can We Trust the Performance Evaluation of Uncertainty Estimation\n Methods in Text Summarization?","display_name":"Can We Trust the Performance Evaluation of Uncertainty Estimation\n Methods in Text Summarization?","publication_year":2024,"publication_date":"2024-06-25","ids":{"openalex":"https://openalex.org/W4400064797","doi":"https://doi.org/10.48550/arxiv.2406.17274"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2406.17274","pdf_url":"https://arxiv.org/pdf/2406.17274","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"type":"preprint","type_crossref":"posted-content","indexed_in":["arxiv"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2406.17274","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103096924","display_name":"Jianfeng He","orcid":"https://orcid.org/0000-0002-7700-788X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Jianfeng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109735386","display_name":"Runing Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Runing","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111075772","display_name":"Linlin Yu","orcid":"https://orcid.org/0009-0009-5455-3245"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu, Linlin","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039203958","display_name":"Changbin Li","orcid":"https://orcid.org/0000-0001-9239-3304"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Changbin","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032275274","display_name":"Ruoxi Jia","orcid":"https://orcid.org/0000-0001-9662-9556"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jia, Ruoxi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100570784","display_name":"Feng Chen","orcid":"https://orcid.org/0009-0005-1032-1398"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Feng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101484129","display_name":"Ming Jin","orcid":"https://orcid.org/0000-0001-7909-4545"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jin, Ming","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5038002204","display_name":"Chang\u2010Tien Lu","orcid":"https://orcid.org/0000-0003-3675-0199"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Chang-Tien","raw_affiliation_strings":[],"affiliations":[]}],"institution_assertions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":0,"max":77},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9946,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9946,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.994,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9892,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[],"concepts":[{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.9255669},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6401784},{"id":"https://openalex.org/C96250715","wikidata":"https://www.wikidata.org/wiki/Q965330","display_name":"Estimation","level":2,"score":0.6107358},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.36958945},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.35118026},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34578627},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.32273316},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.11364713},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.090874165}],"mesh":[],"locations_count":1,"locations":[{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2406.17274","pdf_url":"https://arxiv.org/pdf/2406.17274","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2406.17274","pdf_url":"https://arxiv.org/pdf/2406.17274","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[],"grants":[],"datasets":[],"versions":[],"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4389760904","https://openalex.org/W4323520239","https://openalex.org/W4306886878","https://openalex.org/W4242223894","https://openalex.org/W3148229873","https://openalex.org/W2366403280","https://openalex.org/W2150160875","https://openalex.org/W2091301346","https://openalex.org/W1517524280","https://openalex.org/W1495108544"],"abstract_inverted_index":{"Text":[0],"summarization,":[1],"a":[2,66],"key":[3],"natural":[4],"language":[5,87,92],"generation":[6],"(NLG)":[7],"task,":[8],"is":[9],"vital":[10],"in":[11,21],"various":[12],"domains.":[13],"However,":[14],"the":[15,32,47,80,106,120],"high":[16],"cost":[17],"of":[18,34,49,84,108,122,139],"inaccurate":[19],"summaries":[20],"risk-critical":[22],"applications,":[23],"particularly":[24],"those":[25],"involving":[26],"human-in-the-loop":[27],"decision-making,":[28],"raises":[29],"concerns":[30],"about":[31],"reliability":[33],"uncertainty":[35,50,81,111,130],"estimation":[36,82,112,131],"on":[37,53,94],"text":[38],"summarization":[39],"(UE-TS)":[40],"evaluation":[41,138],"methods.":[42],"This":[43],"concern":[44],"stems":[45],"from":[46],"dependency":[48],"model":[51,93],"metrics":[52,73,127],"diverse":[54,129],"and":[55,89,128,136],"potentially":[56],"conflicting":[57],"NLG":[58,72,126],"metrics.":[59],"To":[60],"address":[61],"this":[62,115],"issue,":[63],"we":[64],"introduce":[65],"comprehensive":[67],"UE-TS":[68,140],"benchmark":[69,78],"incorporating":[70],"31":[71],"across":[74],"four":[75],"dimensions.":[76],"The":[77],"evaluates":[79],"capabilities":[83],"two":[85],"large":[86],"models":[88],"one":[90],"pre-trained":[91],"three":[95],"datasets,":[96],"with":[97],"human-annotation":[98],"analysis":[99],"incorporated":[100],"where":[101],"applicable.":[102],"We":[103],"also":[104],"assess":[105],"performance":[107],"14":[109],"common":[110],"methods":[113,132],"within":[114],"benchmark.":[116],"Our":[117],"findings":[118],"emphasize":[119],"importance":[121],"considering":[123],"multiple":[124],"uncorrelated":[125],"to":[133],"ensure":[134],"reliable":[135],"efficient":[137],"techniques.":[141]},"abstract_inverted_index_v3":null,"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4400064797","counts_by_year":[],"updated_date":"2025-04-22T19:36:54.806157","created_date":"2024-06-27"}