{"id":"https://openalex.org/W4226096495","doi":"https://doi.org/10.1109/access.2022.3158675","title":"Reproducibility in Computing Research: An Empirical Study","display_name":"Reproducibility in Computing Research: An Empirical Study","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W4226096495","doi":"https://doi.org/10.1109/access.2022.3158675"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2022.3158675","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/9668973/09732944.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"type":"article","type_crossref":"journal-article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/9668973/09732944.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5011210908","display_name":"Wullianallur Raghupathi","orcid":"https://orcid.org/0000-0001-9927-5343"},"institutions":[{"id":"https://openalex.org/I164389053","display_name":"Fordham University","ror":"https://ror.org/03qnxaf80","country_code":"US","type":"education","lineage":["https://openalex.org/I164389053"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wullianallur Raghupathi","raw_affiliation_strings":["Gabelli School of Business, Fordham University, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"Gabelli School of Business, Fordham University, New York, NY, USA","institution_ids":["https://openalex.org/I164389053"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061805847","display_name":"Viju Raghupathi","orcid":"https://orcid.org/0000-0002-5082-3166"},"institutions":[{"id":"https://openalex.org/I83041322","display_name":"Brooklyn College","ror":"https://ror.org/019k4jq75","country_code":"US","type":"education","lineage":["https://openalex.org/I83041322"]},{"id":"https://openalex.org/I174216632","display_name":"City University of New York","ror":"https://ror.org/00453a208","country_code":"US","type":"education","lineage":["https://openalex.org/I174216632"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Viju Raghupathi","raw_affiliation_strings":["Koppelman School of Business, Brooklyn College, City University of New York, Brooklyn, NY, USA"],"affiliations":[{"raw_affiliation_string":"Koppelman School of Business, Brooklyn College, City University of New York, Brooklyn, NY, USA","institution_ids":["https://openalex.org/I83041322","https://openalex.org/I174216632"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5111365236","display_name":"Jie Ren","orcid":"https://orcid.org/0000-0003-4803-0033"},"institutions":[{"id":"https://openalex.org/I164389053","display_name":"Fordham University","ror":"https://ror.org/03qnxaf80","country_code":"US","type":"education","lineage":["https://openalex.org/I164389053"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jie Ren","raw_affiliation_strings":["Gabelli School of Business, Fordham University, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"Gabelli School of Business, Fordham University, New York, NY, USA","institution_ids":["https://openalex.org/I164389053"]}]}],"institution_assertions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":1850,"currency":"USD","value_usd":1850,"provenance":"doaj"},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850,"provenance":"doaj"},"fwci":4.604,"has_fulltext":true,"fulltext_origin":"pdf","cited_by_count":17,"citation_normalized_percentile":{"value":0.999836,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":"10","issue":null,"first_page":"29207","last_page":"29223"},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9965,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9965,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9798,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9775,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[],"concepts":[{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.9616698},{"id":"https://openalex.org/C56666940","wikidata":"https://www.wikidata.org/wiki/Q788790","display_name":"Documentation","level":2,"score":0.73297393},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6170275},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.37915635},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.24698076},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.130081},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2022.3158675","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/9668973/09732944.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},{"is_oa":false,"landing_page_url":"https://doaj.org/article/093f25584c8d44fc8d25d44529ccf460","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2022.3158675","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/9668973/09732944.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"sustainable_development_goals":[],"grants":[],"datasets":[],"versions":[],"referenced_works_count":82,"referenced_works":["https://openalex.org/W1549945956","https://openalex.org/W1589382802","https://openalex.org/W1595114968","https://openalex.org/W1760642209","https://openalex.org/W1897139626","https://openalex.org/W1970588407","https://openalex.org/W1979461729","https://openalex.org/W1979916898","https://openalex.org/W1987777080","https://openalex.org/W1993266153","https://openalex.org/W2001776137","https://openalex.org/W2008985174","https://openalex.org/W2018164143","https://openalex.org/W2020859734","https://openalex.org/W2030620341","https://openalex.org/W2031739072","https://openalex.org/W2031934506","https://openalex.org/W2032593675","https://openalex.org/W2036318837","https://openalex.org/W2048060485","https://openalex.org/W2053814546","https://openalex.org/W2065302863","https://openalex.org/W2068069008","https://openalex.org/W2082256810","https://openalex.org/W2092865915","https://openalex.org/W2100267603","https://openalex.org/W2125564169","https://openalex.org/W2139210474","https://openalex.org/W2139319950","https://openalex.org/W2144981148","https://openalex.org/W2145409614","https://openalex.org/W2166378641","https://openalex.org/W2183919022","https://openalex.org/W2188566912","https://openalex.org/W2243230937","https://openalex.org/W2263593861","https://openalex.org/W2283438976","https://openalex.org/W2294316503","https://openalex.org/W2406493898","https://openalex.org/W2414283705","https://openalex.org/W2470809521","https://openalex.org/W2501465840","https://openalex.org/W2512030163","https://openalex.org/W2512184211","https://openalex.org/W2536000422","https://openalex.org/W2538248335","https://openalex.org/W2570788028","https://openalex.org/W2577330276","https://openalex.org/W2582905716","https://openalex.org/W2610833180","https://openalex.org/W2743156140","https://openalex.org/W2743491008","https://openalex.org/W2744444377","https://openalex.org/W2744947029","https://openalex.org/W2770036798","https://openalex.org/W2781178498","https://openalex.org/W2781579392","https://openalex.org/W2787225861","https://openalex.org/W2788572835","https://openalex.org/W2793484265","https://openalex.org/W2793993424","https://openalex.org/W2807964278","https://openalex.org/W2883754520","https://openalex.org/W2890056718","https://openalex.org/W2901297776","https://openalex.org/W2913589238","https://openalex.org/W2944196828","https://openalex.org/W2951702001","https://openalex.org/W2963957599","https://openalex.org/W2969669267","https://openalex.org/W2995281728","https://openalex.org/W2997072486","https://openalex.org/W3023516546","https://openalex.org/W3096956705","https://openalex.org/W3123868826","https://openalex.org/W3124333825","https://openalex.org/W3158320892","https://openalex.org/W3166999527","https://openalex.org/W3178778073","https://openalex.org/W4235099152","https://openalex.org/W4239292628","https://openalex.org/W585341442"],"related_works":["https://openalex.org/W638612878","https://openalex.org/W3106281778","https://openalex.org/W2748952813","https://openalex.org/W2417696084","https://openalex.org/W2413717610","https://openalex.org/W2368782778","https://openalex.org/W2140154150","https://openalex.org/W2087830269","https://openalex.org/W2003643616","https://openalex.org/W1973270181"],"abstract_inverted_index":{"In":[0,123],"computing,":[1],"research":[2,38,95,195],"findings":[3,112],"are":[4,65,86,109,135],"often":[5],"anecdotally":[6],"faulted":[7],"for":[8,88,105,132],"not":[9],"being":[10],"reproducible.":[11],"Numerous":[12],"empirical":[13],"studies":[14],"have":[15],"analyzed":[16],"the":[17,32,80,98,106,117,121,125,139,156,198,215,228],"reproducibility":[18,36,47,64,173,203,216,232],"of":[19,22,35,37,56,116,138,145,155,191,218,227,231],"a":[20,189],"variety":[21],"research.":[23,235],"Our":[24,111],"objective,":[25],"in":[26,39,151,182,188,202,233],"this":[27],"study,":[28],"is":[29,184],"to":[30,62,82,224],"quantify":[31],"current":[33],"state":[34],"computing":[40,183,234],"based":[41],"on":[42,101,214],"prior":[43],"research,":[44],"using":[45],"three":[46,53,70],"factors\u2014Method,":[48],"Data":[49,74],"and":[50,67,75,147,174,200,208],"Experiment\u2014to":[51],"measure":[52],"different":[54,143],"degrees":[55],"reproducibility.":[57,179],"Twenty-five":[58],"variables":[59,78,131,140],"traditionally":[60],"utilized":[61],"document":[63],"identified":[66],"grouped":[68],"into":[69],"factors,":[71],"namely":[72],"Method,":[73],"Experiment.":[76],"These":[77],"describe":[79],"extent":[81],"which":[83],"these":[84],"factors":[85],"documented":[87,119],"each":[89,133],"paper.":[90],"Approximately":[91],"100":[92],"randomly":[93],"selected":[94],"papers":[96,118,149],"from":[97],"International":[99],"Conference":[100],"Information":[102],"Systems":[103],"series,":[104],"year":[107],"2019,":[108],"surveyed.":[110],"suggest":[113],"that":[114,128,177],"none":[115],"all":[120],"variables.":[122],"fact,":[124],"results":[126],"show":[127],"relatively":[129],"few":[130],"factor":[134],"documented.":[136],"Some":[137],"vary":[141],"across":[142],"categories":[144],"papers,":[146],"most":[148],"fail":[150],"at":[152],"least":[153],"one":[154],"factors.":[157,193],"Reproducibility":[158,165],"scores":[159],"decrease":[160],"with":[161],"increased":[162],"documentation":[163,181],"requirements.":[164],"may":[166,196],"improve":[167],"over":[168,204],"time,":[169],"as":[170],"researchers":[171,207],"prioritize":[172],"utilize":[175],"methods":[176],"ensure":[178],"Research":[180],"remarkably":[185],"limited,":[186],"resulting":[187],"dearth":[190],"reproducible":[192],"Future":[194],"study":[197,222],"shifts":[199],"trends":[201],"time.":[205],"Meanwhile,":[206],"publishers":[209],"must":[210],"increase":[211],"their":[212,219],"focus":[213],"aspects":[217],"papers.":[220],"This":[221],"contributes":[223],"our":[225],"understanding":[226],"status":[229],"quo":[230]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4226096495","counts_by_year":[{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":5}],"updated_date":"2025-01-05T03:26:58.587476","created_date":"2022-05-05"}