{"id":"https://openalex.org/W4303438727","doi":"https://doi.org/10.48550/arxiv.2210.02157","title":"The Influence of Learning Rule on Representation Dynamics in Wide Neural Networks","display_name":"The Influence of Learning Rule on Representation Dynamics in Wide Neural Networks","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W4303438727","doi":"https://doi.org/10.48550/arxiv.2210.02157"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2210.02157","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false},"type":"preprint","type_crossref":"posted-content","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/abs/2210.02157","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039282308","display_name":"Blake Bordelon","orcid":"https://orcid.org/0000-0003-0455-9445"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bordelon, Blake","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5023195984","display_name":"Cengiz Pehlevan","orcid":"https://orcid.org/0000-0001-9767-6063"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pehlevan, Cengiz","raw_affiliation_strings":[],"affiliations":[]}],"institution_assertions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.824572,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":76,"max":79},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9943,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9943,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10581","display_name":"Neural dynamics and brain function","score":0.9803,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":0.9754,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/hebbian-theory","display_name":"Hebbian theory","score":0.68673575},{"id":"https://openalex.org/keywords/learning-rule","display_name":"Learning rule","score":0.65937173},{"id":"https://openalex.org/keywords/kernel","display_name":"Kernel (algebra)","score":0.52361995},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.45715046},{"id":"https://openalex.org/keywords/representation","display_name":"Representation","score":0.44776142}],"concepts":[{"id":"https://openalex.org/C111437709","wikidata":"https://www.wikidata.org/wiki/Q1277874","display_name":"Hebbian theory","level":3,"score":0.68673575},{"id":"https://openalex.org/C2779127903","wikidata":"https://www.wikidata.org/wiki/Q6510194","display_name":"Learning rule","level":3,"score":0.65937173},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.60963786},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5938301},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5746293},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.52361995},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4832364},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.45715046},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.44776142},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.4314249},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.43023035},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.42941117},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.41987187},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.37460124},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.09109083},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.0897305},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2210.02157","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2210.02157","pdf_url":"http://arxiv.org/pdf/2210.02157","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":false,"landing_page_url":"https://api.datacite.org/dois/10.48550/arxiv.2210.02157","pdf_url":null,"source":{"id":"https://openalex.org/S4393179698","display_name":"DataCite API","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I4210145204","host_organization_name":"DataCite","host_organization_lineage":["https://openalex.org/I4210145204"],"host_organization_lineage_names":["DataCite"],"type":"metadata"},"license":null,"license_id":null,"version":null}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2210.02157","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[],"grants":[],"datasets":[],"versions":[],"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4387970390","https://openalex.org/W3110559181","https://openalex.org/W2945998886","https://openalex.org/W2078098917","https://openalex.org/W2019283726","https://openalex.org/W1993110512","https://openalex.org/W1991841905","https://openalex.org/W1618190107","https://openalex.org/W1525325138","https://openalex.org/W1490081714"],"abstract_inverted_index":{"It":[0],"is":[1,86,104],"unclear":[2],"how":[3],"changing":[4],"the":[5,23,31,77,80,98,112,134,147,160,176,186,207,220,266],"learning":[6,15,32,59,75,144,213,264],"rule":[7],"of":[8,73,79,133,142,241,256],"a":[9,89,172,193,228],"deep":[10,37],"neural":[11,93],"network":[12],"alters":[13],"its":[14],"dynamics":[16,138],"and":[17,30,44,55,106,136,154,181,190,196,222,237,259],"representations.":[18],"To":[19],"gain":[20],"insight":[21],"into":[22],"relationship":[24],"between":[25,179,219],"learned":[26],"features,":[27,163],"function":[28,82],"approximation,":[29],"rule,":[33],"we":[34,150,200],"analyze":[35],"infinite-width":[36],"networks":[38,66,204],"trained":[39,205],"with":[40,123,171,216],"gradient":[41],"descent":[42],"(GD)":[43],"biologically-plausible":[45],"alternatives":[46],"including":[47],"feedback":[48,52,182],"alignment":[49,53,255],"(FA),":[50],"direct":[51],"(DFA),":[54],"error":[56],"modulated":[57,248],"Hebbian":[58],"(Hebb),":[60],"as":[61,63],"well":[62],"gated":[64],"linear":[65],"(GLN).":[67],"We":[68],"show":[69,251],"that,":[70],"for":[71,232],"each":[72,141],"these":[74,143],"rules,":[76],"evolution":[78,118],"output":[81],"at":[83],"infinite":[84],"width":[85],"governed":[87],"by":[88,140,175],"time":[90],"varying":[91],"effective":[92],"tangent":[94],"kernel":[95,236],"(eNTK).":[96],"In":[97,146,185],"lazy":[99,148,234],"training":[100],"limit,":[101,149],"this":[102,116],"eNTK":[103],"static":[105],"does":[107],"not":[108],"evolve,":[109],"while":[110,164],"in":[111,206,265],"rich":[113,187,208],"mean-field":[114],"regime":[115,209],"kernel's":[117],"can":[119,156,167],"be":[120],"determined":[121,174],"self-consistently":[122],"dynamical":[124],"mean":[125],"field":[126],"theory":[127],"(DMFT).":[128],"This":[129],"DMFT":[130],"enables":[131],"comparisons":[132],"feature":[135,212],"prediction":[137],"induced":[139],"rules.":[145],"find":[151,201],"that":[152,202],"DFA":[153,189],"Hebb":[155,249],"only":[157],"learn":[158],"using":[159],"last":[161,267],"layer":[162],"full":[165],"FA":[166,191,203],"utilize":[168,192],"earlier":[169],"layers":[170],"scale":[173],"initial":[177],"correlation":[178,218],"feedforward":[180],"weight":[183],"matrices.":[184],"regime,":[188],"temporally":[194],"evolving":[195],"depth-dependent":[197],"NTK.":[198],"Counterintuitively,":[199],"exhibit":[210],"more":[211],"if":[214],"initialized":[215],"smaller":[217],"forward":[221],"backward":[223],"pass":[224],"weights.":[225],"GLNs":[226],"admit":[227],"very":[229,252],"simple":[230],"formula":[231],"their":[233,242,257],"limit":[235],"preserve":[238],"conditional":[239],"Gaussianity":[240],"preactivations":[243],"under":[244],"gating":[245],"functions.":[246],"Error":[247],"rules":[250],"small":[253],"task-relevant":[254],"kernels":[258],"perform":[260],"most":[261],"task":[262],"relevant":[263],"layer.":[268]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4303438727","counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2}],"updated_date":"2025-01-22T17:45:42.034991","created_date":"2022-10-07"}