{"id":"https://openalex.org/W3170923959","doi":"https://doi.org/10.1007/s10579-021-09547-3","title":"Determinants of grader agreement: an analysis of multiple short answer corpora","display_name":"Determinants of grader agreement: an analysis of multiple short answer corpora","publication_year":2021,"publication_date":"2021-06-12","ids":{"openalex":"https://openalex.org/W3170923959","doi":"https://doi.org/10.1007/s10579-021-09547-3","mag":"3170923959"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10579-021-09547-3","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10579-021-09547-3.pdf","source":{"id":"https://openalex.org/S4306424877","display_name":"Language Resources and Evaluation","issn_l":"1574-020X","issn":["1574-020X","1574-0218"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319965","https://openalex.org/P4310319900"],"host_organization_lineage_names":["Springer Nature","Springer Science+Business Media"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"type":"article","type_crossref":"journal-article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10579-021-09547-3.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5009943536","display_name":"Ulrike Pad\u00f3","orcid":"https://orcid.org/0009-0000-0664-7487"},"institutions":[{"id":"https://openalex.org/I31582689","display_name":"Stuttgart University of Applied Sciences","ror":"https://ror.org/039gdg280","country_code":"DE","type":"education","lineage":["https://openalex.org/I31582689"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Ulrike Pad\u00f3","raw_affiliation_strings":["Hochschule f\u00fcr Technik, Stuttgart, Germany"],"affiliations":[{"raw_affiliation_string":"Hochschule f\u00fcr Technik, Stuttgart, Germany","institution_ids":["https://openalex.org/I31582689"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5003870894","display_name":"Sebastian Pad\u00f3","orcid":"https://orcid.org/0000-0002-7529-6825"},"institutions":[{"id":"https://openalex.org/I100066346","display_name":"University of Stuttgart","ror":"https://ror.org/04vnq7t77","country_code":"DE","type":"education","lineage":["https://openalex.org/I100066346"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Sebastian Pad\u00f3","raw_affiliation_strings":["Universit\u00e4t Stuttgart, Stuttgart, Germany"],"affiliations":[{"raw_affiliation_string":"Universit\u00e4t Stuttgart, Stuttgart, Germany","institution_ids":["https://openalex.org/I100066346"]}]}],"institution_assertions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5009943536"],"corresponding_institution_ids":["https://openalex.org/I31582689"],"apc_list":null,"apc_paid":null,"fwci":0.114,"has_fulltext":true,"fulltext_origin":"pdf","cited_by_count":1,"citation_normalized_percentile":{"value":0.444234,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":57,"max":67},"biblio":{"volume":"56","issue":"2","first_page":"387","last_page":"416"},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9962,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9962,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.9822,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11715","display_name":"Education and Critical Thinking Development","score":0.9582,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/grading","display_name":"Grading (engineering)","score":0.7567092},{"id":"https://openalex.org/keywords/agreement","display_name":"Agreement","score":0.48448417}],"concepts":[{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.7567092},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7209039},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.71586025},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6395402},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.52883685},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.48448417},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.47041467},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.43259376},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.3179896},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.077726424},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C147176958","wikidata":"https://www.wikidata.org/wiki/Q77590","display_name":"Civil engineering","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10579-021-09547-3","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10579-021-09547-3.pdf","source":{"id":"https://openalex.org/S4306424877","display_name":"Language Resources and Evaluation","issn_l":"1574-020X","issn":["1574-020X","1574-0218"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319965","https://openalex.org/P4310319900"],"host_organization_lineage_names":["Springer Nature","Springer Science+Business Media"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10579-021-09547-3","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10579-021-09547-3.pdf","source":{"id":"https://openalex.org/S4306424877","display_name":"Language Resources and Evaluation","issn_l":"1574-020X","issn":["1574-020X","1574-0218"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319965","https://openalex.org/P4310319900"],"host_organization_lineage_names":["Springer Nature","Springer Science+Business Media"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"sustainable_development_goals":[{"score":0.79,"display_name":"Quality education","id":"https://metadata.un.org/sdg/4"}],"grants":[],"datasets":[],"versions":[],"referenced_works_count":43,"referenced_works":["https://openalex.org/W141807713","https://openalex.org/W1592263067","https://openalex.org/W1592805114","https://openalex.org/W1967082761","https://openalex.org/W1990167011","https://openalex.org/W1998025025","https://openalex.org/W2009435671","https://openalex.org/W2019679004","https://openalex.org/W2032946087","https://openalex.org/W2073265902","https://openalex.org/W2091950237","https://openalex.org/W2115584598","https://openalex.org/W2116817443","https://openalex.org/W2117909082","https://openalex.org/W2118707092","https://openalex.org/W2123565526","https://openalex.org/W2125001590","https://openalex.org/W2126344546","https://openalex.org/W2133436118","https://openalex.org/W2134227585","https://openalex.org/W2141766660","https://openalex.org/W2141845152","https://openalex.org/W2149719430","https://openalex.org/W2166916420","https://openalex.org/W2167007060","https://openalex.org/W2172200076","https://openalex.org/W2183293350","https://openalex.org/W2251451860","https://openalex.org/W2289293839","https://openalex.org/W2296348267","https://openalex.org/W2571706648","https://openalex.org/W2756883610","https://openalex.org/W2799871693","https://openalex.org/W2805251832","https://openalex.org/W2922604858","https://openalex.org/W2930638636","https://openalex.org/W2956033709","https://openalex.org/W2970206146","https://openalex.org/W2983570302","https://openalex.org/W3034730645","https://openalex.org/W3037150633","https://openalex.org/W3101091275","https://openalex.org/W4243041937"],"related_works":["https://openalex.org/W4283160672","https://openalex.org/W4247536566","https://openalex.org/W4241418540","https://openalex.org/W3119814709","https://openalex.org/W2725786787","https://openalex.org/W2404647514","https://openalex.org/W2018477250","https://openalex.org/W1667647204","https://openalex.org/W1590965489","https://openalex.org/W1508895727"],"abstract_inverted_index":{"Abstract":[0],"The":[1,28,36],"\u2019short":[2],"answer\u2019":[3],"question":[4,84,132,200],"format":[5],"is":[6,41],"a":[7,111,174,219],"widely":[8],"used":[9],"tool":[10],"in":[11,14,22,47,53,68,106,152,208],"educational":[12,50],"assessment,":[13],"which":[15,235],"students":[16],"write":[17],"one":[18],"to":[19,24,74,141,190],"three":[20],"sentences":[21],"response":[23],"an":[25,64,97],"open":[26],"question.":[27],"answers":[29,184,187],"are":[30,99,188,206,230],"subsequently":[31],"rated":[32],"by":[33],"expert":[34],"graders.":[35],"agreement":[37,81,151,170],"between":[38],"these":[39],"graders":[40],"crucial":[42],"for":[43,59,157,239],"reliable":[44],"analysis,":[45],"both":[46],"terms":[48,54],"of":[49,55,103,113,119,145,176,222],"strategies":[51],"and":[52,88,115,163,185,203,232,242],"developing":[56],"automatic":[57],"models":[58,140],"short":[60,226],"answer":[61,86,89,227],"grading":[62,228],"(SAG),":[63],"active":[65],"research":[66],"topic":[67],"NLP.":[69],"This":[70,134],"makes":[71],"it":[72],"important":[73],"understand":[75],"the":[76,92,100,116,128,143,199,224],"properties":[77,122,148,177,193],"that":[78],"influence":[79],"grader":[80,150],"(such":[82],"as":[83],"difficulty,":[85],"length,":[87],"correctness).":[90],"However,":[91],"twin":[93],"challenges":[94],"towards":[95],"such":[96,147],"understanding":[98],"wide":[101],"range":[102],"SAG":[104,155],"corpora":[105,156,181,205,229],"use":[107],"(which":[108,123],"differ":[109],"along":[110],"number":[112,175],"dimensions)":[114],"hierarchical":[117],"structure":[118],"potentially":[120],"relevant":[121],"can":[124],"be":[125],"located":[126],"at":[127],"corpus,":[129],"answer,":[130],"or":[131],"levels).":[133],"article":[135],"uses":[136],"generalized":[137],"mixed":[138],"effects":[139],"analyze":[142],"effect":[144],"various":[146],"on":[149],"six":[153],"major":[154,225],"two":[158],"main":[159],"assessment":[160],"tasks":[161],"(language":[162],"content":[164],"assessment).":[165],"Overall,":[166],"we":[167,217,236],"find":[168],"broad":[169],"among":[171],"corpora,":[172],"with":[173,210],"behaving":[178],"similarly":[179],"across":[180],"(e.g.,":[182,198],"shorter":[183],"correct":[186],"easier":[189],"grade).":[191],"Some":[192],"show":[194],"more":[195,207],"corpus-specific":[196],"behavior":[197],"difficulty":[201],"level),":[202],"some":[204],"line":[209],"general":[211],"tendencies":[212],"than":[213],"others.":[214],"In":[215],"sum,":[216],"obtain":[218],"nuanced":[220],"picture":[221],"how":[223],"similar":[231],"dissimilar":[233],"from":[234],"derive":[237],"suggestions":[238],"corpus":[240],"development":[241],"analysis.":[243]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3170923959","counts_by_year":[{"year":2023,"cited_by_count":1}],"updated_date":"2024-12-17T05:26:11.568619","created_date":"2021-06-22"}