{"id":"https://openalex.org/W4285293606","doi":"https://doi.org/10.18653/v1/2022.acl-demo.17","title":"Dynatask: A Framework for Creating Dynamic AI Benchmark Tasks","display_name":"Dynatask: A Framework for Creating Dynamic AI Benchmark Tasks","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W4285293606","doi":"https://doi.org/10.18653/v1/2022.acl-demo.17"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2022.acl-demo.17","pdf_url":"https://aclanthology.org/2022.acl-demo.17.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"type":"article","type_crossref":"proceedings-article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2022.acl-demo.17.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026811404","display_name":"Tristan Thrush","orcid":null},"institutions":[{"id":"https://openalex.org/I2252078561","display_name":"Meta (Israel)","ror":"https://ror.org/02388em19","country_code":"IL","type":"company","lineage":["https://openalex.org/I2252078561","https://openalex.org/I4210114444"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Tristan Thrush","raw_affiliation_strings":["Facebook AI Research;"],"affiliations":[{"raw_affiliation_string":"Facebook AI Research;","institution_ids":["https://openalex.org/I2252078561"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086885471","display_name":"Kushal Tirumala","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kushal Tirumala","raw_affiliation_strings":["Coactive AI;"],"affiliations":[{"raw_affiliation_string":"Coactive AI;","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014436016","display_name":"Anmol Gupta","orcid":"https://orcid.org/0000-0003-0159-8780"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Anmol Gupta","raw_affiliation_strings":["The University of Hong Kong;"],"affiliations":[{"raw_affiliation_string":"The University of Hong Kong;","institution_ids":["https://openalex.org/I177725633","https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018864006","display_name":"Max Bartolo","orcid":"https://orcid.org/0009-0007-3301-7895"},"institutions":[{"id":"https://openalex.org/I4210113297","display_name":"Google (United Kingdom)","ror":"https://ror.org/024bc3e07","country_code":"GB","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210113297","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Max Bartolo","raw_affiliation_strings":["University College London; \u22c6 Google;"],"affiliations":[{"raw_affiliation_string":"University College London; \u22c6 Google;","institution_ids":["https://openalex.org/I4210113297","https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100707159","display_name":"Pedro Rodr\u00edguez","orcid":"https://orcid.org/0000-0001-8572-0725"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pedro Rodriguez","raw_affiliation_strings":["Coactive AI;"],"affiliations":[{"raw_affiliation_string":"Coactive AI;","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056133908","display_name":"Tariq Kane","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tariq Kane","raw_affiliation_strings":["Coactive AI;"],"affiliations":[{"raw_affiliation_string":"Coactive AI;","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064746496","display_name":"William Gaviria Rojas","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"William Gaviria Rojas","raw_affiliation_strings":["Coactive AI;"],"affiliations":[{"raw_affiliation_string":"Coactive AI;","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053488528","display_name":"Peter Mattson","orcid":"https://orcid.org/0000-0002-5984-238X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peter Mattson","raw_affiliation_strings":["Coactive AI;"],"affiliations":[{"raw_affiliation_string":"Coactive AI;","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062696000","display_name":"Adina Williams","orcid":"https://orcid.org/0000-0001-5281-3343"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Adina Williams","raw_affiliation_strings":["Coactive AI;"],"affiliations":[{"raw_affiliation_string":"Coactive AI;","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016956470","display_name":"Douwe Kiela","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Douwe Kiela","raw_affiliation_strings":["Coactive AI;"],"affiliations":[{"raw_affiliation_string":"Coactive AI;","institution_ids":[]}]}],"institution_assertions":[],"countries_distinct_count":4,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.674,"has_fulltext":true,"fulltext_origin":"pdf","cited_by_count":40,"citation_normalized_percentile":{"value":0.99997,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"174","last_page":"181"},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9999,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9999,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9989,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9941,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.777338},{"id":"https://openalex.org/keywords/adina","display_name":"Adina","score":0.6804551},{"id":"https://openalex.org/keywords/thrush","display_name":"Thrush","score":0.42185533}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.777338},{"id":"https://openalex.org/C2776002260","wikidata":"https://www.wikidata.org/wiki/Q245622","display_name":"Adina","level":3,"score":0.6804551},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6557534},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.42562878},{"id":"https://openalex.org/C2779835073","wikidata":"https://www.wikidata.org/wiki/Q26050","display_name":"Thrush","level":5,"score":0.42185533},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.41786593},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.35637623},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.16680074},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.069206744},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.055551052},{"id":"https://openalex.org/C66938386","wikidata":"https://www.wikidata.org/wiki/Q633538","display_name":"Structural engineering","level":1,"score":0.0},{"id":"https://openalex.org/C135628077","wikidata":"https://www.wikidata.org/wiki/Q220184","display_name":"Finite element method","level":2,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C2778220009","wikidata":"https://www.wikidata.org/wiki/Q160105","display_name":"Cervical cancer","level":3,"score":0.0},{"id":"https://openalex.org/C121608353","wikidata":"https://www.wikidata.org/wiki/Q12078","display_name":"Cancer","level":2,"score":0.0},{"id":"https://openalex.org/C2994539378","wikidata":"https://www.wikidata.org/wiki/Q184627","display_name":"Human papilloma virus","level":4,"score":0.0},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2022.acl-demo.17","pdf_url":"https://aclanthology.org/2022.acl-demo.17.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2204.01906","pdf_url":"https://arxiv.org/pdf/2204.01906","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2022.acl-demo.17","pdf_url":"https://aclanthology.org/2022.acl-demo.17.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"sustainable_development_goals":[{"display_name":"Quality education","score":0.79,"id":"https://metadata.un.org/sdg/4"}],"grants":[],"datasets":[],"versions":[],"referenced_works_count":30,"referenced_works":["https://openalex.org/W1754293002","https://openalex.org/W1840435438","https://openalex.org/W2103018059","https://openalex.org/W2108598243","https://openalex.org/W2166637769","https://openalex.org/W2923014074","https://openalex.org/W2943552823","https://openalex.org/W2963748441","https://openalex.org/W2963846996","https://openalex.org/W2966491090","https://openalex.org/W3034850762","https://openalex.org/W3105424285","https://openalex.org/W3105662186","https://openalex.org/W3171312896","https://openalex.org/W3171654528","https://openalex.org/W3171774521","https://openalex.org/W3173826007","https://openalex.org/W3174783113","https://openalex.org/W3176580738","https://openalex.org/W3177071108","https://openalex.org/W3193717459","https://openalex.org/W3197876970","https://openalex.org/W3206210649","https://openalex.org/W3212464620","https://openalex.org/W4229907684","https://openalex.org/W4287179329","https://openalex.org/W4287634363","https://openalex.org/W4287887244","https://openalex.org/W4300963525","https://openalex.org/W4327859259"],"related_works":["https://openalex.org/W620563080","https://openalex.org/W4384911122","https://openalex.org/W4205966478","https://openalex.org/W3204264611","https://openalex.org/W2382373870","https://openalex.org/W2363465389","https://openalex.org/W2350003927","https://openalex.org/W2067480270","https://openalex.org/W2036249425","https://openalex.org/W1979751130"],"abstract_inverted_index":{"Tristan":[0],"Thrush,":[1],"Kushal":[2],"Tirumala,":[3],"Anmol":[4],"Gupta,":[5],"Max":[6],"Bartolo,":[7],"Pedro":[8],"Rodriguez,":[9],"Tariq":[10],"Kane,":[11],"William":[12],"Gaviria":[13],"Rojas,":[14],"Peter":[15],"Mattson,":[16],"Adina":[17],"Williams,":[18],"Douwe":[19],"Kiela.":[20],"Proceedings":[21],"of":[22,27],"the":[23,28],"60th":[24],"Annual":[25],"Meeting":[26],"Association":[29],"for":[30],"Computational":[31],"Linguistics:":[32],"System":[33],"Demonstrations.":[34],"2022.":[35]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4285293606","counts_by_year":[{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":32},{"year":2022,"cited_by_count":2}],"updated_date":"2025-01-09T02:17:51.976364","created_date":"2022-07-14"}