{"id":"https://openalex.org/W3096020573","doi":"https://doi.org/10.1007/s11517-020-02261-2","title":"Beyond kappa: an informational index for diagnostic agreement in dichotomous and multivalue ordered-categorical ratings","display_name":"Beyond kappa: an informational index for diagnostic agreement in dichotomous and multivalue ordered-categorical ratings","publication_year":2020,"publication_date":"2020-11-03","ids":{"openalex":"https://openalex.org/W3096020573","doi":"https://doi.org/10.1007/s11517-020-02261-2","mag":"3096020573","pmid":"https://pubmed.ncbi.nlm.nih.gov/33145661","pmcid":"https://www.ncbi.nlm.nih.gov/pmc/articles/7679268"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11517-020-02261-2","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11517-020-02261-2.pdf","source":{"id":"https://openalex.org/S46300952","display_name":"Medical & Biological Engineering & Computing","issn_l":"0140-0118","issn":["0140-0118","1741-0444"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319965","https://openalex.org/P4310319900"],"host_organization_lineage_names":["Springer Nature","Springer Science+Business Media"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"type":"article","type_crossref":"journal-article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s11517-020-02261-2.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052391923","display_name":"Alberto Casagrande","orcid":"https://orcid.org/0000-0002-8681-1482"},"institutions":[{"id":"https://openalex.org/I142444530","display_name":"University of Trieste","ror":"https://ror.org/02n742c10","country_code":"IT","type":"education","lineage":["https://openalex.org/I142444530"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Alberto Casagrande","raw_affiliation_strings":["Dipartimento di Matematica e Geoscienze, Universit\u00e0 degli Studi di Trieste, Trieste, Italy"],"affiliations":[{"raw_affiliation_string":"Dipartimento di Matematica e Geoscienze, Universit\u00e0 degli Studi di Trieste, Trieste, Italy","institution_ids":["https://openalex.org/I142444530"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052680051","display_name":"Francesco Fabris","orcid":"https://orcid.org/0000-0002-1950-8838"},"institutions":[{"id":"https://openalex.org/I142444530","display_name":"University of Trieste","ror":"https://ror.org/02n742c10","country_code":"IT","type":"education","lineage":["https://openalex.org/I142444530"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Francesco Fabris","raw_affiliation_strings":["Dipartimento di Matematica e Geoscienze, Universit\u00e0 degli Studi di Trieste, Trieste, Italy"],"affiliations":[{"raw_affiliation_string":"Dipartimento di Matematica e Geoscienze, Universit\u00e0 degli Studi di Trieste, Trieste, Italy","institution_ids":["https://openalex.org/I142444530"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5075137791","display_name":"Rossano Girometti","orcid":"https://orcid.org/0000-0002-0904-5147"},"institutions":[{"id":"https://openalex.org/I4210126152","display_name":"Ospedale Santa Maria della Misericordia di Udine","ror":"https://ror.org/02zpc2253","country_code":"IT","type":"healthcare","lineage":["https://openalex.org/I4210126152"]},{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Rossano Girometti","raw_affiliation_strings":["Dipartimento di Area Medica, Istituto di Radiologia, Ospedale S. Maria della Misericordia, Universit\u00e0 degli Studi di Udine, Udine, Italy"],"affiliations":[{"raw_affiliation_string":"Dipartimento di Area Medica, Istituto di Radiologia, Ospedale S. Maria della Misericordia, Universit\u00e0 degli Studi di Udine, Udine, Italy","institution_ids":["https://openalex.org/I4210126152","https://openalex.org/I129043915"]}]}],"institution_assertions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2890,"currency":"EUR","value_usd":3690,"provenance":"doaj"},"apc_paid":{"value":2890,"currency":"EUR","value_usd":3690,"provenance":"doaj"},"fwci":1.44,"has_fulltext":true,"fulltext_origin":"pdf","cited_by_count":19,"citation_normalized_percentile":{"value":0.999758,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":92},"biblio":{"volume":"58","issue":"12","first_page":"3089","last_page":"3099"},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9999,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9999,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14374","display_name":"Statistical Methods in Epidemiology","score":0.9761,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.9614,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/kappa","display_name":"Kappa","score":0.8369701},{"id":"https://openalex.org/keywords/categorical-variable","display_name":"Categorical variable","score":0.8090613},{"id":"https://openalex.org/keywords/agreement","display_name":"Agreement","score":0.6693922},{"id":"https://openalex.org/keywords/cohens-kappa","display_name":"Cohen's kappa","score":0.49298355},{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"Inter-Rater Reliability","score":0.45902404},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.41146624}],"concepts":[{"id":"https://openalex.org/C2778724333","wikidata":"https://www.wikidata.org/wiki/Q14401","display_name":"Kappa","level":2,"score":0.8369701},{"id":"https://openalex.org/C5274069","wikidata":"https://www.wikidata.org/wiki/Q2285707","display_name":"Categorical variable","level":2,"score":0.8090613},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.68479884},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.6693922},{"id":"https://openalex.org/C163864269","wikidata":"https://www.wikidata.org/wiki/Q1107106","display_name":"Cohen's kappa","level":2,"score":0.49298355},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.45902404},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.45046705},{"id":"https://openalex.org/C2777382242","wikidata":"https://www.wikidata.org/wiki/Q6017816","display_name":"Index (typography)","level":2,"score":0.4348196},{"id":"https://openalex.org/C44725695","wikidata":"https://www.wikidata.org/wiki/Q288156","display_name":"Normative","level":2,"score":0.4271958},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.41146624},{"id":"https://openalex.org/C19527891","wikidata":"https://www.wikidata.org/wiki/Q1120908","display_name":"Medical physics","level":1,"score":0.38824967},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3848539},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.38351232},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3691876},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.3232038},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.32304072},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.3206445},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2585411},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.18028489},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.096892625},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[{"descriptor_ui":"D001940","descriptor_name":"Breast","qualifier_ui":"","qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D001943","descriptor_name":"Breast Neoplasms","qualifier_ui":"","qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D005260","descriptor_name":"Female","qualifier_ui":"","qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":"","qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015588","descriptor_name":"Observer Variation","qualifier_ui":"","qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":"","qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D014463","descriptor_name":"Ultrasonography","qualifier_ui":"","qualifier_name":null,"is_major_topic":false}],"locations_count":4,"locations":[{"is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11517-020-02261-2","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11517-020-02261-2.pdf","source":{"id":"https://openalex.org/S46300952","display_name":"Medical & Biological Engineering & Computing","issn_l":"0140-0118","issn":["0140-0118","1741-0444"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319965","https://openalex.org/P4310319900"],"host_organization_lineage_names":["Springer Nature","Springer Science+Business Media"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},{"is_oa":true,"landing_page_url":"http://hdl.handle.net/11390/1193659","pdf_url":"https://air.uniud.it/bitstream/11390/1193659/1/Casagrande2020_Article_BeyondKappaAnInformationalInde.pdf","source":{"id":"https://openalex.org/S4306401163","display_name":"Institutional Research Information System (University of Udine)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I129043915","host_organization_name":"University of Udine","host_organization_lineage":["https://openalex.org/I129043915"],"host_organization_lineage_names":["University of Udine"],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true},{"is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7679268","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":["National Institutes of Health"],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true},{"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/33145661","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":["National Institutes of Health"],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11517-020-02261-2","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11517-020-02261-2.pdf","source":{"id":"https://openalex.org/S46300952","display_name":"Medical & Biological Engineering & Computing","issn_l":"0140-0118","issn":["0140-0118","1741-0444"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319965","https://openalex.org/P4310319900"],"host_organization_lineage_names":["Springer Nature","Springer Science+Business Media"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"sustainable_development_goals":[],"grants":[{"funder":"https://openalex.org/F4320306978","funder_display_name":"Ittleson Foundation","award_id":"-"}],"datasets":[],"versions":[],"referenced_works_count":49,"referenced_works":["https://openalex.org/W1487708858","https://openalex.org/W1493790738","https://openalex.org/W1965555277","https://openalex.org/W1966184253","https://openalex.org/W1966976587","https://openalex.org/W1983897914","https://openalex.org/W1995875735","https://openalex.org/W1996575035","https://openalex.org/W2003924600","https://openalex.org/W2033833022","https://openalex.org/W2036887984","https://openalex.org/W2037789405","https://openalex.org/W2051779656","https://openalex.org/W2053154970","https://openalex.org/W2064493677","https://openalex.org/W2065241327","https://openalex.org/W2076898331","https://openalex.org/W2078483536","https://openalex.org/W2084413241","https://openalex.org/W2086543716","https://openalex.org/W2087171258","https://openalex.org/W2091010131","https://openalex.org/W2110145224","https://openalex.org/W2131850886","https://openalex.org/W2133012565","https://openalex.org/W2141403362","https://openalex.org/W2156196235","https://openalex.org/W2157318188","https://openalex.org/W2163236485","https://openalex.org/W2164214266","https://openalex.org/W2165570175","https://openalex.org/W2171590660","https://openalex.org/W2307277543","https://openalex.org/W2327308124","https://openalex.org/W2330153163","https://openalex.org/W2498472323","https://openalex.org/W2531301769","https://openalex.org/W2581845030","https://openalex.org/W2604015085","https://openalex.org/W2767182834","https://openalex.org/W2900791133","https://openalex.org/W2975256032","https://openalex.org/W3120740533","https://openalex.org/W4233650312","https://openalex.org/W4234180827","https://openalex.org/W4294214983","https://openalex.org/W52880462","https://openalex.org/W625387107","https://openalex.org/W88719000"],"related_works":["https://openalex.org/W96748260","https://openalex.org/W4294214983","https://openalex.org/W2581818947","https://openalex.org/W2330902540","https://openalex.org/W2054015915","https://openalex.org/W1996751524","https://openalex.org/W1984207165","https://openalex.org/W1979852909","https://openalex.org/W1976976697","https://openalex.org/W1966976587"],"abstract_inverted_index":{"Abstract":[0],"Agreement":[1],"measures":[2],"are":[3,117],"useful":[4],"tools":[5],"to":[6,96,155,174,190],"both":[7,123],"compare":[8,191],"different":[9],"evaluations":[10],"of":[11,49,74,99,107,113,163,203],"the":[12,30,43,63,72,75,78,88,105,108,111,121,129,136,142,147,149,161,164,181,201],"same":[13],"diagnostic":[14,138],"outcomes":[15],"and":[16,38,103,125,160,185,194],"validate":[17],"new":[18,85],"rating":[19],"systems":[20],"or":[21],"devices.":[22],"Cohen\u2019s":[23,100],"kappa":[24],"(":[25,91],"\u03ba":[26,178],")":[27],"certainly":[28],"is":[29,68],"most":[31],"popular":[32],"agreement":[33,86,90],"method":[34,52],"between":[35,132,146],"two":[36,133],"raters,":[37,148],"proved":[39],"its":[40,66,157],"effectiveness":[41,162],"in":[42,77,180,186,200],"last":[44],"sixty":[45],"years.":[46],"In":[47,153],"spite":[48],"that,":[50],"this":[51],"suffers":[53],"from":[54,110],"some":[55,98,171],"alleged":[56],"issues,":[57],"which":[58,94],"have":[59],"been":[60,168],"highlighted":[61],"since":[62],"1970s;":[64],"moreover,":[65],"value":[67],"strongly":[69],"dependent":[70],"on":[71,170],"prevalence":[73,109],"disease":[76],"considered":[79],"sample.":[80],"This":[81],"work":[82],"introduces":[83],"a":[84,187],"index,":[87],"informational":[89],"IA":[92,166],"),":[93],"seems":[95],"avoid":[97],"kappa\u2019s":[101],"flaws,":[102],"separates":[104],"contribution":[106],"nucleus":[112],"agreement.":[114,152],"These":[115],"goals":[116],"achieved":[118],"by":[119],"modelling":[120],"agreement\u2014in":[122],"dichotomous":[124],"multivalue":[126],"ordered-categorical":[127],"cases\u2014as":[128],"information":[130,144],"shared":[131],"raters":[134],"through":[135],"virtual":[137],"channel":[139],"connecting":[140],"them:":[141],"more":[143],"exchanged":[145],"higher":[150],"their":[151],"order":[154],"test":[156],"fair":[158],"behaviour":[159],"method,":[165],"has":[167],"tested":[169],"cases":[172],"known":[173],"be":[175],"problematic":[176],"for":[177],",":[179],"machine":[182],"learning":[183],"context":[184],"clinical":[188],"scenario":[189],"ultrasound":[192],"(US)":[193],"automated":[195],"breast":[196,204],"volume":[197],"scanner":[198],"(ABVS)":[199],"setting":[202],"cancer":[205],"imaging.":[206]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3096020573","counts_by_year":[{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":2}],"updated_date":"2024-12-29T04:22:34.313583","created_date":"2020-11-09"}