{"id":"https://openalex.org/W4385570177","doi":"https://doi.org/10.18653/v1/2023.acl-long.342","title":"Measuring the Instability of Fine-Tuning","display_name":"Measuring the Instability of Fine-Tuning","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4385570177","doi":"https://doi.org/10.18653/v1/2023.acl-long.342"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.acl-long.342","pdf_url":"https://aclanthology.org/2023.acl-long.342.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"type":"article","type_crossref":"proceedings-article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2023.acl-long.342.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043954721","display_name":"Yupei Du","orcid":"https://orcid.org/0009-0008-6751-6112"},"institutions":[{"id":"https://openalex.org/I193662353","display_name":"Utrecht University","ror":"https://ror.org/04pp8hn57","country_code":"NL","type":"education","lineage":["https://openalex.org/I193662353"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Yupei Du","raw_affiliation_strings":["Utrecht University Utrecht, the Netherlands"],"affiliations":[{"raw_affiliation_string":"Utrecht University Utrecht, the Netherlands","institution_ids":["https://openalex.org/I193662353"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5004209770","display_name":"Dong Nguyen","orcid":"https://orcid.org/0000-0002-6062-3117"},"institutions":[{"id":"https://openalex.org/I193662353","display_name":"Utrecht University","ror":"https://ror.org/04pp8hn57","country_code":"NL","type":"education","lineage":["https://openalex.org/I193662353"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Dong Nguyen","raw_affiliation_strings":["Utrecht University Utrecht, the Netherlands"],"affiliations":[{"raw_affiliation_string":"Utrecht University Utrecht, the Netherlands","institution_ids":["https://openalex.org/I193662353"]}]}],"institution_assertions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"fulltext_origin":"pdf","cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":0,"max":67},"biblio":{"volume":null,"issue":null,"first_page":"6209","last_page":"6230"},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9996,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9996,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9993,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9794,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/granularity","display_name":"Granularity","score":0.8457391}],"concepts":[{"id":"https://openalex.org/C207821765","wikidata":"https://www.wikidata.org/wiki/Q405372","display_name":"Instability","level":2,"score":0.91724825},{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.8457391},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.69132215},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.6635955},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.59542936},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.47036096},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.24639946},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.19235826},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.17879829},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.16967723},{"id":"https://openalex.org/C57879066","wikidata":"https://www.wikidata.org/wiki/Q41217","display_name":"Mechanics","level":1,"score":0.074611634},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.acl-long.342","pdf_url":"https://aclanthology.org/2023.acl-long.342.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},{"is_oa":true,"landing_page_url":"https://dspace.library.uu.nl/handle/1874/430750","pdf_url":"https://dspace.library.uu.nl/bitstream/handle/1874/430750/2023.acl-long.342.pdf?sequence=1&isAllowed=y","source":{"id":"https://openalex.org/S4306401649","display_name":"Utrecht University Repository (Utrecht University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I193662353","host_organization_name":"Utrecht University","host_organization_lineage":["https://openalex.org/I193662353"],"host_organization_lineage_names":["Utrecht University"],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true},{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2302.07778","pdf_url":"http://arxiv.org/pdf/2302.07778","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.acl-long.342","pdf_url":"https://aclanthology.org/2023.acl-long.342.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"sustainable_development_goals":[{"score":0.57,"display_name":"Climate action","id":"https://metadata.un.org/sdg/13"}],"grants":[],"datasets":[],"versions":[],"referenced_works_count":47,"referenced_works":["https://openalex.org/W131533222","https://openalex.org/W1975879668","https://openalex.org/W1981358475","https://openalex.org/W2033582154","https://openalex.org/W2057069782","https://openalex.org/W2095705004","https://openalex.org/W2100235303","https://openalex.org/W2251939518","https://openalex.org/W242171001","https://openalex.org/W2551857043","https://openalex.org/W2560647685","https://openalex.org/W2923014074","https://openalex.org/W2942810103","https://openalex.org/W2963026768","https://openalex.org/W2963341956","https://openalex.org/W2964186069","https://openalex.org/W2965373594","https://openalex.org/W2975185270","https://openalex.org/W2978670439","https://openalex.org/W2979826702","https://openalex.org/W2996428491","https://openalex.org/W3034199299","https://openalex.org/W3034999214","https://openalex.org/W3040074755","https://openalex.org/W3099878876","https://openalex.org/W3126074026","https://openalex.org/W3128094008","https://openalex.org/W3172318343","https://openalex.org/W3175898847","https://openalex.org/W3182414949","https://openalex.org/W3197876970","https://openalex.org/W3211620297","https://openalex.org/W3214698142","https://openalex.org/W4212774754","https://openalex.org/W4225695770","https://openalex.org/W4229005866","https://openalex.org/W4243795479","https://openalex.org/W4286892891","https://openalex.org/W4287110638","https://openalex.org/W4287281000","https://openalex.org/W4287391717","https://openalex.org/W4287867774","https://openalex.org/W4292779060","https://openalex.org/W4295312788","https://openalex.org/W4297730150","https://openalex.org/W4322614701","https://openalex.org/W4385572956"],"related_works":["https://openalex.org/W4382701072","https://openalex.org/W4256502920","https://openalex.org/W4226090359","https://openalex.org/W2999756192","https://openalex.org/W2975817033","https://openalex.org/W2931688134","https://openalex.org/W2378857091","https://openalex.org/W2377919138","https://openalex.org/W2011624601","https://openalex.org/W103652678"],"abstract_inverted_index":{"Fine-tuning":[0],"pre-trained":[1],"language":[2],"models":[3],"on":[4,18],"downstream":[5],"tasks":[6],"with":[7],"varying":[8],"random":[9],"seeds":[10],"has":[11],"been":[12],"shown":[13],"to":[14,31],"be":[15],"unstable,":[16],"especially":[17],"small":[19],"datasets.":[20],"Many":[21],"previous":[22],"studies":[23,38],"have":[24],"investigated":[25],"this":[26,59],"instability":[27,69,98],"and":[28,64,90],"proposed":[29],"methods":[30],"mitigate":[32],"it.":[33],"However,":[34],"most":[35],"of":[36,44,56,70,81,109],"these":[37,82],"only":[39],"used":[40],"the":[41,88,110],"standard":[42],"deviation":[43],"performance":[45],"scores":[46],"(SD)":[47],"as":[48],"their":[49],"measure,":[50],"which":[51],"is":[52],"a":[53,77],"narrow":[54],"characterization":[55],"instability.":[57,112],"In":[58],"paper,":[60],"we":[61,75,86],"analyze":[62,87],"SD":[63],"six":[65],"other":[66],"measures":[67,94],"quantifying":[68],"different":[71,93],"granularity":[72],"levels.":[73],"Moreover,":[74],"propose":[76],"systematic":[78],"evaluation":[79],"framework":[80],"measures\u2019":[83],"validity.":[84],"Finally,":[85],"consistency":[89],"difference":[91],"between":[92],"by":[95],"reassessing":[96],"existing":[97],"mitigation":[99],"methods.":[100],"We":[101],"hope":[102],"our":[103],"results":[104],"will":[105],"inform":[106],"better":[107],"measurements":[108],"fine-tuning":[111]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4385570177","counts_by_year":[],"updated_date":"2025-01-02T12:55:19.007796","created_date":"2023-08-05"}