{"id":"https://openalex.org/W4399695623","doi":"https://doi.org/10.48550/arxiv.2406.09098","title":"SciKnowEval: Evaluating Multi-level Scientific Knowledge of Large\n Language Models","display_name":"SciKnowEval: Evaluating Multi-level Scientific Knowledge of Large\n Language Models","publication_year":2024,"publication_date":"2024-06-13","ids":{"openalex":"https://openalex.org/W4399695623","doi":"https://doi.org/10.48550/arxiv.2406.09098"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.09098","pdf_url":"http://arxiv.org/pdf/2406.09098","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"type":"preprint","type_crossref":"posted-content","indexed_in":["arxiv"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://arxiv.org/pdf/2406.09098","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113178971","display_name":"Kehua Feng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Feng, Kehua","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032086683","display_name":"Keyan Ding","orcid":"https://orcid.org/0000-0003-2900-7313"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ding, Keyan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022968974","display_name":"Weijie Wang","orcid":"https://orcid.org/0000-0002-2602-5236"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Weijie","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101552401","display_name":"Xiang Zhuang","orcid":"https://orcid.org/0000-0002-0253-1476"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhuang, Xiang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100729939","display_name":"Zeyuan Wang","orcid":"https://orcid.org/0000-0001-7211-9543"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zeyuan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054995764","display_name":"Ming Qin","orcid":"https://orcid.org/0000-0002-8424-6299"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qin, Ming","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100701168","display_name":"Yu Zhao","orcid":"https://orcid.org/0000-0003-2815-3570"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Yu","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100695406","display_name":"Jianhua Yao","orcid":"https://orcid.org/0000-0001-9157-9596"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yao, Jianhua","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100381999","display_name":"Qiang Zhang","orcid":"https://orcid.org/0000-0003-3776-9799"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Qiang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5115589223","display_name":"Huajun Chen","orcid":"https://orcid.org/0000-0002-0998-3387"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Huajun","raw_affiliation_strings":[],"affiliations":[]}],"institution_assertions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":77,"max":88},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.984,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.984,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9427,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5108802},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.369583}],"mesh":[],"locations_count":1,"locations":[{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.09098","pdf_url":"http://arxiv.org/pdf/2406.09098","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.09098","pdf_url":"http://arxiv.org/pdf/2406.09098","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[],"grants":[],"datasets":[],"versions":[],"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4396701345","https://openalex.org/W4396696052","https://openalex.org/W4395014643","https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2382290278","https://openalex.org/W2376932109","https://openalex.org/W2358668433","https://openalex.org/W2001405890"],"abstract_inverted_index":{"The":[0,134,189],"burgeoning":[1],"utilization":[2],"of":[3,15,21,45,68,103,179],"Large":[4],"Language":[5],"Models":[6],"(LLMs)":[7],"in":[8,71,152,170],"scientific":[9,22,46,69,112,154,183],"research":[10,172],"necessitates":[11],"advanced":[12],"benchmarks":[13],"capable":[14],"evaluating":[16],"their":[17],"understanding":[18],"and":[19,56,66,77,81,85,97,105,114,125,130,156,173,175,191],"application":[20],"knowledge":[23,70,74,184],"comprehensively.":[24],"To":[25],"address":[26],"this":[27,118],"need,":[28],"we":[29,94,120],"introduce":[30],"the":[31,64,100,142,177],"SciKnowEval":[32,104,161],"benchmark,":[33],"a":[34,107,164],"novel":[35],"framework":[36],"that":[37,137,160,181],"systematically":[38],"evaluates":[39],"LLMs":[40,127,144,169,180],"across":[41],"five":[42],"progressive":[43],"levels":[44,60],"knowledge:":[47],"studying":[48],"extensively,":[49],"inquiring":[50],"earnestly,":[51],"thinking":[52],"profoundly,":[53],"discerning":[54],"clearly,":[55],"practicing":[57],"assiduously.":[58],"These":[59],"aim":[61],"to":[62],"assess":[63],"breadth":[65],"depth":[67],"LLMs,":[72],"including":[73],"coverage,":[75],"inquiry":[76],"exploration":[78],"capabilities,":[79],"reflection":[80],"reasoning":[82],"abilities,":[83],"ethic":[84],"safety":[86,187],"considerations,":[87],"as":[88,90,99],"well":[89],"practice":[91],"proficiency.":[92],"Specifically,":[93],"take":[95],"biology":[96],"chemistry":[98],"two":[101],"instances":[102],"construct":[106],"dataset":[108,190],"encompassing":[109],"50K":[110],"multi-level":[111],"problems":[113],"solutions.":[115],"By":[116],"leveraging":[117],"dataset,":[119],"benchmark":[121],"20":[122],"leading":[123],"open-source":[124],"proprietary":[126,143],"using":[128],"zero-shot":[129],"few-shot":[131],"prompting":[132],"strategies.":[133],"results":[135],"reveal":[136],"despite":[138],"achieving":[139],"state-of-the-art":[140],"performance,":[141],"still":[145],"have":[146],"considerable":[147],"room":[148],"for":[149,167],"improvement,":[150],"particularly":[151],"addressing":[153],"computations":[155],"applications.":[157],"We":[158],"anticipate":[159],"will":[162],"establish":[163],"comprehensive":[165],"standard":[166],"benchmarking":[168],"science":[171],"discovery,":[174],"promote":[176],"development":[178],"integrate":[182],"with":[185],"strong":[186],"awareness.":[188],"code":[192],"are":[193],"publicly":[194],"available":[195],"at":[196],"https://github.com/hicai-zju/sciknoweval":[197],".":[198]},"abstract_inverted_index_v3":null,"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4399695623","counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2025-04-30T20:48:14.869366","created_date":"2024-06-15"}