{"id":"https://openalex.org/W3100894295","doi":"https://doi.org/10.18653/v1/2020.findings-emnlp.103","title":"Undersensitivity in Neural Reading Comprehension","display_name":"Undersensitivity in Neural Reading Comprehension","publication_year":2020,"publication_date":"2020-01-01","ids":{"openalex":"https://openalex.org/W3100894295","doi":"https://doi.org/10.18653/v1/2020.findings-emnlp.103","mag":"3100894295"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2020.findings-emnlp.103","pdf_url":"https://www.aclweb.org/anthology/2020.findings-emnlp.103.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"type":"article","type_crossref":"proceedings-article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://www.aclweb.org/anthology/2020.findings-emnlp.103.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5067022550","display_name":"Johannes Welbl","orcid":null},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Johannes Welbl","raw_affiliation_strings":["University College London"],"affiliations":[{"raw_affiliation_string":"University College London","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019106673","display_name":"Pasquale Minervini","orcid":"https://orcid.org/0000-0002-8442-602X"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Pasquale Minervini","raw_affiliation_strings":["University College London"],"affiliations":[{"raw_affiliation_string":"University College London","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018864006","display_name":"Max Bartolo","orcid":"https://orcid.org/0009-0007-3301-7895"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Max Bartolo","raw_affiliation_strings":["University College London"],"affiliations":[{"raw_affiliation_string":"University College London","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049427071","display_name":"Pontus Stenetorp","orcid":null},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Pontus Stenetorp","raw_affiliation_strings":["University College London"],"affiliations":[{"raw_affiliation_string":"University College London","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101404695","display_name":"Sebastian Riedel","orcid":"https://orcid.org/0000-0002-3655-2486"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Sebastian Riedel","raw_affiliation_strings":["University College London"],"affiliations":[{"raw_affiliation_string":"University College London","institution_ids":["https://openalex.org/I45129253"]}]}],"institution_assertions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.505,"has_fulltext":true,"fulltext_origin":"pdf","cited_by_count":20,"citation_normalized_percentile":{"value":0.625983,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":92,"max":93},"biblio":{"volume":null,"issue":null,"first_page":"1152","last_page":"1165"},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9729,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9729,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.9483,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14394","display_name":"Cognitive Science and Education Research","score":0.9288,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness","score":0.64477247},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.5462487},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep Neural Networks","score":0.49642593}],"concepts":[{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.8582983},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.77846694},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.65115625},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.64477247},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5662382},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.5462487},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.49642593},{"id":"https://openalex.org/C190470478","wikidata":"https://www.wikidata.org/wiki/Q2370229","display_name":"Invariant (physics)","level":2,"score":0.47605893},{"id":"https://openalex.org/C2778780117","wikidata":"https://www.wikidata.org/wiki/Q3269423","display_name":"Reading comprehension","level":3,"score":0.46944377},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.43823949},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.42553365},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.40790772},{"id":"https://openalex.org/C554936623","wikidata":"https://www.wikidata.org/wiki/Q199657","display_name":"Reading (process)","level":2,"score":0.36782742},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.33380282},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.11686978},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C37914503","wikidata":"https://www.wikidata.org/wiki/Q156495","display_name":"Mathematical physics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2020.findings-emnlp.103","pdf_url":"https://www.aclweb.org/anthology/2020.findings-emnlp.103.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2003.04808","pdf_url":"http://arxiv.org/pdf/2003.04808","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2020.findings-emnlp.103","pdf_url":"https://www.aclweb.org/anthology/2020.findings-emnlp.103.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true},"sustainable_development_goals":[{"display_name":"Quality education","score":0.85,"id":"https://metadata.un.org/sdg/4"}],"grants":[],"datasets":[],"versions":[],"referenced_works_count":45,"referenced_works":["https://openalex.org/W1673923490","https://openalex.org/W2153579005","https://openalex.org/W2460937040","https://openalex.org/W2551396370","https://openalex.org/W2557764419","https://openalex.org/W2604265558","https://openalex.org/W2767899794","https://openalex.org/W2788403449","https://openalex.org/W2799007037","https://openalex.org/W2799194071","https://openalex.org/W2887768933","https://openalex.org/W2889453388","https://openalex.org/W2890719433","https://openalex.org/W2896457183","https://openalex.org/W2911267026","https://openalex.org/W2913222130","https://openalex.org/W2949849869","https://openalex.org/W2949961827","https://openalex.org/W2951873305","https://openalex.org/W2962718684","https://openalex.org/W2962736243","https://openalex.org/W2962768284","https://openalex.org/W2962816513","https://openalex.org/W2963126845","https://openalex.org/W2963323070","https://openalex.org/W2963341956","https://openalex.org/W2963526187","https://openalex.org/W2963547127","https://openalex.org/W2963612171","https://openalex.org/W2963661177","https://openalex.org/W2963783970","https://openalex.org/W2963969878","https://openalex.org/W2964116600","https://openalex.org/W2964153729","https://openalex.org/W2965373594","https://openalex.org/W2969442125","https://openalex.org/W2977235550","https://openalex.org/W2994934025","https://openalex.org/W2996040088","https://openalex.org/W3014564055","https://openalex.org/W3035507081","https://openalex.org/W3128232076","https://openalex.org/W4289330434","https://openalex.org/W4294170691","https://openalex.org/W4300511536"],"related_works":["https://openalex.org/W4383221314","https://openalex.org/W4285785480","https://openalex.org/W3203790781","https://openalex.org/W3127875750","https://openalex.org/W3093978547","https://openalex.org/W3080754722","https://openalex.org/W2997056298","https://openalex.org/W2953536436","https://openalex.org/W2950183588","https://openalex.org/W2738001131"],"abstract_inverted_index":{"Current":[0],"reading":[1],"comprehension":[2],"methods":[3],"generalise":[4,145],"well":[5],"to":[6,35,102,122,158],"in-distribution":[7],"test":[8],"sets,":[9],"yet":[10],"perform":[11],"poorly":[12],"on":[13,19,40,95,135,161],"adversarially":[14],"selected":[15],"data.":[16],"Prior":[17],"work":[18],"adversarial":[20,66,111,118],"inputs":[21],"typically":[22],"studies":[23],"model":[24,79,133,175],"oversensitivity:":[25],"semantically":[26],"invariant":[27],"text":[28,49],"perturbations":[29],"that":[30,92],"cause":[31],"a":[32,78,173],"model\u2019s":[33,55],"prediction":[34,45,56],"change.":[36],"Here":[37],"we":[38],"focus":[39],"the":[41,54,74,82,136,168],"complementary":[42],"problem:":[43],"excessive":[44],"undersensitivity,":[46],"where":[47],"input":[48],"is":[50],"meaningfully":[51],"changed":[52],"but":[53],"does":[57],"not,":[58],"even":[59,87],"though":[60],"it":[61],"should.":[62],"We":[63,90],"formulate":[64],"an":[65],"attack":[67,127],"which":[68,77,120],"searches":[69],"among":[70],"semantic":[71],"variations":[72],"of":[73],"question":[75],"for":[76],"erroneously":[80],"predicts":[81],"same":[83],"answer,":[84],"and":[85,98,105,110,125,140,143,171],"with":[86],"higher":[88],"probability.":[89],"demonstrate":[91],"models":[93,144],"trained":[94],"both":[96],"SQuAD2.0":[97],"NewsQA":[99],"are":[100,155],"vulnerable":[101],"this":[103],"attack,":[104],"then":[106],"investigate":[107],"data":[108,124],"augmentation":[109],"training":[112,169],"as":[113,177,179],"defences.":[114],"Both":[115],"substantially":[116],"decrease":[117],"vulnerability,":[119],"generalises":[121],"held-out":[123,126],"spaces.":[128],"Addressing":[129],"undersensitivity":[130],"furthermore":[131],"improves":[132],"robustness":[134],"previously":[137],"introduced":[138],"ADDSENT":[139],"ADDONESENT":[141],"datasets,":[142],"better":[146],"when":[147],"facing":[148],"train":[149],"/":[150],"evaluation":[151],"distribution":[152],"mismatch:":[153],"they":[154],"less":[156],"prone":[157],"overly":[159],"rely":[160],"shallow":[162],"predictive":[163],"cues":[164],"present":[165],"only":[166],"in":[167],"set,":[170],"outperform":[172],"conventional":[174],"by":[176],"much":[178],"10.9%":[180],"F1.":[181]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3100894295","counts_by_year":[{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":8},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":1}],"updated_date":"2025-01-06T21:44:32.782482","created_date":"2020-11-23"}