{"id":"https://openalex.org/W4308043713","doi":"https://doi.org/10.1007/s10618-022-00866-9","title":"A methodology for refined evaluation of neural code completion approaches","display_name":"A methodology for refined evaluation of neural code completion approaches","publication_year":2022,"publication_date":"2022-11-01","ids":{"openalex":"https://openalex.org/W4308043713","doi":"https://doi.org/10.1007/s10618-022-00866-9"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10618-022-00866-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10618-022-00866-9.pdf","source":{"id":"https://openalex.org/S121920818","display_name":"Data Mining and Knowledge Discovery","issn_l":"1384-5810","issn":["1384-5810","1573-756X"],"is_oa":false,"is_in_doaj":false,"is_indexed_in_scopus":true,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319965","https://openalex.org/P4310319900"],"host_organization_lineage_names":["Springer Nature","Springer Science+Business Media"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"type":"article","type_crossref":"journal-article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10618-022-00866-9.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5025520865","display_name":"Kim Tuyen Le","orcid":"https://orcid.org/0000-0003-4075-1979"},"institutions":[{"id":"https://openalex.org/I223822909","display_name":"Heidelberg University","ror":"https://ror.org/038t36y30","country_code":"DE","type":"funder","lineage":["https://openalex.org/I223822909"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Kim Tuyen Le","raw_affiliation_strings":["Heidelberg University, Heidelberg, Germany"],"affiliations":[{"raw_affiliation_string":"Heidelberg University, Heidelberg, Germany","institution_ids":["https://openalex.org/I223822909"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050324267","display_name":"Gabriel Rashidi","orcid":null},"institutions":[{"id":"https://openalex.org/I223822909","display_name":"Heidelberg University","ror":"https://ror.org/038t36y30","country_code":"DE","type":"funder","lineage":["https://openalex.org/I223822909"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Gabriel Rashidi","raw_affiliation_strings":["Heidelberg University, Heidelberg, Germany"],"affiliations":[{"raw_affiliation_string":"Heidelberg University, Heidelberg, Germany","institution_ids":["https://openalex.org/I223822909"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5054003032","display_name":"Artur Andrzejak","orcid":"https://orcid.org/0000-0003-0150-8220"},"institutions":[{"id":"https://openalex.org/I223822909","display_name":"Heidelberg University","ror":"https://ror.org/038t36y30","country_code":"DE","type":"funder","lineage":["https://openalex.org/I223822909"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Artur Andrzejak","raw_affiliation_strings":["Heidelberg University, Heidelberg, Germany"],"affiliations":[{"raw_affiliation_string":"Heidelberg University, Heidelberg, Germany","institution_ids":["https://openalex.org/I223822909"]}]}],"institution_assertions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5025520865"],"corresponding_institution_ids":["https://openalex.org/I223822909"],"apc_list":{"value":2390,"currency":"EUR","value_usd":2990},"apc_paid":{"value":2390,"currency":"EUR","value_usd":2990},"fwci":1.522,"has_fulltext":true,"fulltext_origin":"pdf","cited_by_count":4,"citation_normalized_percentile":{"value":0.767763,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":79,"max":82},"biblio":{"volume":"37","issue":"1","first_page":"167","last_page":"204"},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9534,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9499,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.55187166}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8047743},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.55187166},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.54139036},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.53106},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.46739703},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.43076554},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4293359},{"id":"https://openalex.org/C199519371","wikidata":"https://www.wikidata.org/wiki/Q942695","display_name":"Source lines of code","level":3,"score":0.41270143},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.35280997},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.34245664},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.1923621},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.14941934},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10618-022-00866-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10618-022-00866-9.pdf","source":{"id":"https://openalex.org/S121920818","display_name":"Data Mining and Knowledge Discovery","issn_l":"1384-5810","issn":["1384-5810","1573-756X"],"is_oa":false,"is_in_doaj":false,"is_indexed_in_scopus":true,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319965","https://openalex.org/P4310319900"],"host_organization_lineage_names":["Springer Nature","Springer Science+Business Media"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10618-022-00866-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10618-022-00866-9.pdf","source":{"id":"https://openalex.org/S121920818","display_name":"Data Mining and Knowledge Discovery","issn_l":"1384-5810","issn":["1384-5810","1573-756X"],"is_oa":false,"is_in_doaj":false,"is_indexed_in_scopus":true,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319965","https://openalex.org/P4310319900"],"host_organization_lineage_names":["Springer Nature","Springer Science+Business Media"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"sustainable_development_goals":[],"grants":[{"funder":"https://openalex.org/F4320320881","funder_display_name":"Universit\u00e4t Heidelberg","award_id":null}],"datasets":[],"versions":[],"referenced_works_count":19,"referenced_works":["https://openalex.org/W2143861926","https://openalex.org/W2344444819","https://openalex.org/W2533695286","https://openalex.org/W2954451301","https://openalex.org/W3011564318","https://openalex.org/W3018933895","https://openalex.org/W3088925335","https://openalex.org/W3093464584","https://openalex.org/W3099302725","https://openalex.org/W3108032709","https://openalex.org/W3121707215","https://openalex.org/W3126095862","https://openalex.org/W3138203313","https://openalex.org/W3138429261","https://openalex.org/W3162689995","https://openalex.org/W3167197847","https://openalex.org/W3170092793","https://openalex.org/W3176740355","https://openalex.org/W3211801722"],"related_works":["https://openalex.org/W4388335561","https://openalex.org/W4385572700","https://openalex.org/W4312763760","https://openalex.org/W4282830668","https://openalex.org/W3025557260","https://openalex.org/W2997152889","https://openalex.org/W2970530566","https://openalex.org/W2967478618","https://openalex.org/W2149635184","https://openalex.org/W2135968687"],"abstract_inverted_index":{"Abstract":[0],"Code":[1,113],"completion":[2,62,167,213,230],"has":[3],"become":[4],"an":[5],"indispensable":[6],"feature":[7],"of":[8,42,53,59,71,88,102,124,160,169,194,209,225,252],"modern":[9],"Integrated":[10],"Development":[11],"Environments.":[12],"In":[13,105],"recent":[14],"years,":[15],"many":[16],"approaches":[17],"have":[18],"been":[19],"proposed":[20],"to":[21,29,39,79,120,215],"tackle":[22],"this":[23,106,161],"task.":[24],"However,":[25],"it":[26,77],"is":[27,202,219,240],"hard":[28],"compare":[30],"between":[31,99],"the":[32,40,57,60,81,97,100,103,122,142,158,165,186,195,207,236],"models":[33,63,214],"without":[34],"explicitly":[35],"re-evaluating":[36],"them":[37],"due":[38],"differences":[41,98,196],"used":[43],"benchmarks":[44,227],"(e.g.":[45,136],"datasets":[46],"and":[47,95,150,178,197,255],"evaluation":[48,188],"metrics).":[49],"Besides,":[50],"almost":[51],"all":[52,69],"these":[54],"works":[55],"report":[56],"accuracy":[58,168],"code":[61,72,134,166,212,229,246],"as":[64,250],"aggregated":[65,126],"metrics":[66],"averaged":[67],"over":[68],"types":[70,87,146,248],"tokens.":[73],"Such":[74],"evaluations":[75],"make":[76],"difficult":[78],"assess":[80],"potential":[82],"improvements":[83],"for":[84,133,222,228,244],"particularly":[85],"relevant":[86,132,245],"tokens":[89,143],"(i.e.":[90],"method":[91],"or":[92],"variable":[93],"names),":[94],"blur":[96],"performance":[101],"methods.":[104],"paper,":[107],"we":[108,233],"propose":[109],"a":[110,170,190,220,223],"methodology":[111,162],"called":[112],"Token":[114],"Type":[115],"Taxonomy":[116],"(":[117],"CT3":[118],")":[119],"address":[121],"issue":[123],"using":[125],"metrics.":[127],"We":[128,156,204],"identify":[129],"multiple":[130],"dimensions":[131],"prediction":[135],"syntax":[137],"type,":[138],"context,":[139],"length),":[140],"partition":[141],"into":[144],"meaningful":[145],"along":[147],"each":[148],"dimension,":[149],"compute":[151],"individual":[152],"accuracies":[153],"by":[154,163],"type.":[155],"illustrate":[157,216],"utility":[159],"comparing":[164],"Transformer-based":[171],"model":[172,239],"in":[173],"two":[174],"variants:":[175],"with":[176,179],"closed,":[177],"open":[180,237],"vocabulary.":[181],"Our":[182],"results":[183],"show":[184],"that":[185,217,235],"refined":[187],"provides":[189],"more":[191,242],"detailed":[192],"view":[193],"indicates":[198],"where":[199],"further":[200],"work":[201],"needed.":[203],"also":[205],"survey":[206],"state-of-the-art":[208],"Machine":[210],"Learning-based":[211],"there":[218],"demand":[221],"set":[224],"standardized":[226],"approaches.":[231],"Furthermore,":[232],"find":[234],"vocabulary":[238],"significantly":[241],"accurate":[243],"token":[247],"such":[249],"usage":[251],"(defined)":[253],"variables":[254],"literals.":[256]},"abstract_inverted_index_v3":null,"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4308043713","counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":2}],"updated_date":"2025-02-24T02:08:32.154040","created_date":"2022-11-07"}