%PDF-1.6
%
72 0 obj
<<
/Length 10434
>>
stream
0 g 0 G
0 g 0 G
0 g 0 G
0 g 0 G
0 g 0 G
BT
/F74 14.3462 Tf 152.071 675.426 Td [(Self-Super)10(vised)-250(Models)-250(ar)18(e)-250(Continual)-250(Lear)15(ners)]TJ/F75 11.9552 Tf -31.286 -37.579 Td [(Enrico)-250(Fini)]TJ/F75 7.9701 Tf 54.133 2.577 Td [(*)]TJ
0 0.5 1 rg 0 0.5 1 RG
/F39 7.9701 Tf 4.982 1.761 Td [(1)]TJ/F42 7.9701 Tf 4.234 0 Td [(;)]TJ/F39 7.9701 Tf 2.352 0 Td [(2)]TJ
0 g 0 G
/F75 11.9552 Tf 19.676 -4.338 Td [(V)60(ictor)-250(G.)-250(T)45(urrisi)-250(da)-250(Costa)]TJ/F75 7.9701 Tf 123.926 2.577 Td [(*)]TJ
0 0.5 1 rg 0 0.5 1 RG
/F39 7.9701 Tf 4.982 1.761 Td [(1)]TJ
0 g 0 G
/F75 11.9552 Tf 19.676 -4.338 Td [(Xa)20(vier)-250(Alameda-Pineda)]TJ
0 0.5 1 rg 0 0.5 1 RG
/F39 7.9701 Tf 114.96 4.338 Td [(2)]TJ
0 g 0 G
/F75 11.9552 Tf -293.548 -18.007 Td [(Elisa)-250(Ricci)]TJ
0 0.5 1 rg 0 0.5 1 RG
/F39 7.9701 Tf 52.136 4.339 Td [(1)]TJ/F42 7.9701 Tf 4.234 0 Td [(;)]TJ/F39 7.9701 Tf 2.352 0 Td [(3)]TJ
0 g 0 G
/F75 11.9552 Tf 19.676 -4.339 Td [(Karteek)-250(Alahari)]TJ
0 0.5 1 rg 0 0.5 1 RG
/F39 7.9701 Tf 76.68 4.339 Td [(2)]TJ
0 g 0 G
/F75 11.9552 Tf 19.676 -4.339 Td [(Julien)-250(Mairal)]TJ
0 0.5 1 rg 0 0.5 1 RG
/F39 7.9701 Tf 63.421 4.339 Td [(2)]TJ
0 g 0 G
0 0.5 1 rg 0 0.5 1 RG
/F7 6.9738 Tf -243.768 -19.392 Td [(1)]TJ
0 g 0 G
/F75 9.9626 Tf 6.96 -3.615 Td [(Uni)25(v)15(ersity)-250(of)-250(T)35(rento)]TJ
0 0.5 1 rg 0 0.5 1 RG
/F7 6.9738 Tf 93.608 3.615 Td [(2)]TJ
0 g 0 G
/F75 9.9626 Tf 6.96 -3.615 Td [(Inria)]TJ/F75 6.9738 Tf 18.81 3.615 Td [(\206)]TJ
0 0.5 1 rg 0 0.5 1 RG
/F7 6.9738 Tf 12.453 0 Td [(3)]TJ
0 g 0 G
/F75 9.9626 Tf 6.96 -3.615 Td [(F)15(ondazione)-250(Bruno)-250(K)25(essler)]TJ
0 g 0 G
0 g 0 G
/F74 11.9552 Tf -170.321 -40.963 Td [(Abstract)]TJ/F80 9.9626 Tf -83.928 -30.404 Td [(Self-supervised)-538(models)-538(have)-538(been)-538(shown)-539(to)-538(pr)45(oduce)]TJ -11.955 -11.716 Td [(compar)15(able)-329(or)-329(better)-329(visual)-329(r)37(epr)37(esentations)-329(than)-330(their)-329(su-)]TJ 0 -11.716 Td [(pervised)-467(counterparts)-468(when)-467(tr)15(ained)-467(of)18(\003ine)-467(on)-468(unlabeled)]TJ 0 -11.716 Td [(data)-388(at)-388(scale)15(.)-724(Howe)15(ver)111(,)-423(their)-388(ef)18(\002cacy)-388(is)-388(catastr)45(ophically)]TJ 0 -11.716 Td [(r)37(educed)-258(in)-258(a)-259(Continual)-258(Learning)-258(\050CL\051)-258(scenario)-258(wher)37(e)-259(data)]TJ 0 -11.716 Td [(is)-388(pr)37(esented)-388(to)-388(the)-388(model)-388(sequentially)55(.)-724(In)-388(this)-388(paper)111(,)-423(we)]TJ 0 -11.716 Td [(show)-316(that)-317(self-supervised)-316(loss)-317(functi)1(o)-1(ns)-316(can)-316(be)-317(seamlessly)]TJ 0 -11.716 Td [(con)40(verted)-304(into)-305(distillation)-304(mec)15(hanisms)-304(for)-305(CL)-304(by)-304(adding)-305(a)]TJ 0 -11.716 Td [(pr)37(edictor)-300(network)-299(that)-300(maps)-300(the)-299(curr)37(ent)-300(state)-299(of)-300(the)-300(r)37(epr)37(e-)]TJ 0 -11.716 Td [(sentations)-351(to)-351(their)-351(past)-351(state)15(.)-612(This)-351(enables)-351(us)-351(to)-351(de)15(vise)-351(a)]TJ 0 -11.716 Td [(fr)15(ame)15(work)-264(for)-264(Continual)-263(self-supervised)-264(visual)-264(r)37(epr)37(esenta-)]TJ 0 -11.716 Td [(tion)-339(Learning)-339(that)-339(\050i\051)-339(signi\002cantly)-339(impr)45(o)10(ves)-340(t)1(he)-340(quality)-339(of)]TJ 0 -11.716 Td [(the)-302(learned)-301(r)37(epr)37(esentations,)-315(\050ii\051)-302(is)-301(compatible)-302(with)-302(se)15(ver)15(al)]TJ 0 -11.716 Td [(state-of-the-art)-418(self-supervised)-419(objectives,)-460(and)-418(\050iii\051)-419(needs)]TJ 0 -11.716 Td [(little)-293(to)-293(no)-293(hyperpar)15(ameter)-294(tuning)15(.)-439(W)92(e)-293(demonstr)15(ate)-294(the)-293(ef-)]TJ 0 -11.716 Td [(fectiveness)-224(of)-224(our)-223(appr)45(oac)15(h)-224(empirically)-224(by)-224(tr)15(aining)-224(six)-224(pop-)]TJ 0 -11.716 Td [(ular)-323(self-supervised)-322(models)-323(in)-323(various)-322(CL)-323(settings.)-528(Code:)]TJ
0 0 1 rg 0 0 1 RG
/F81 9.9626 Tf 0 -11.716 Td [(github.com/DonkeyShot21/cassle)]TJ
0 g 0 G
/F80 9.9626 Tf 179.327 0 Td [(.)]TJ/F74 11.9552 Tf -179.327 -38.095 Td [(1.)-250(Intr)18(oduction)]TJ/F75 9.9626 Tf 11.955 -23.431 Td [(During)-216(the)-215(last)-216(fe)25(w)-215(years,)-223(self-supervised)-215(learning)-216(\050SSL\051)]TJ -11.955 -11.716 Td [(has)-215(become)-214(the)-215(most)-215(popular)-215(paradigm)-214(for)-215(unsupervised)-215(vi-)]TJ 0 -11.716 Td [(sual)-330(representation)-331(learning)-330([)]TJ
0 0 1 rg 0 0 1 RG
[(3)]TJ
0 g 0 G
[(,)]TJ
0 0 1 rg 0 0 1 RG
[-184(7)]TJ
0 g 0 G
[(,)]TJ
0 0 1 rg 0 0 1 RG
[-184(8)]TJ
0 g 0 G
[(,)]TJ
0 0 1 rg 0 0 1 RG
[-183(13)]TJ
0 g 0 G
[(,)]TJ
0 0 1 rg 0 0 1 RG
[-184(14)]TJ
0 g 0 G
[(,)]TJ
0 0 1 rg 0 0 1 RG
[-184(26)]TJ
0 g 0 G
[(,)]TJ
0 0 1 rg 0 0 1 RG
[-183(28)]TJ
0 g 0 G
[(,)]TJ
0 0 1 rg 0 0 1 RG
[-184(58)]TJ
0 g 0 G
[(].)-552(In-)]TJ 0 -11.716 Td [(deed,)-318(under)-305(certain)-305(assumptions)-305(\050)]TJ/F80 9.9626 Tf 135.957 0 Td [(e)15(.g)15(.)]TJ/F75 9.9626 Tf 14.087 0 Td [(,)-318(of)25(\003ine)-305(training)-305(with)]TJ -150.044 -11.716 Td [(lar)18(ge)-226(amounts)-227(of)-226(data)-227(and)-226(resources\051,)-231(SSL)-227(methods)-226(are)-227(able)]TJ 0 -11.716 Td [(to)-210(e)15(xtract)-210(representations)-209(that)-210(match)-210(the)-210(quality)-210(of)-210(represen-)]TJ 0 -11.716 Td [(tations)-372(obtained)-372(with)-373(supervised)-372(learning,)-403(wi)1(thout)-373(requir)20(-)]TJ 0 -11.716 Td [(ing)-237(annotations.)-306(Ho)25(we)25(v)15(er)40(,)-239(these)-237(assumptions)-237(do)-237(not)-237(al)10(w)10(ays)]TJ 0 -11.716 Td [(hold)-251(in)-251(real-w)10(orld)-250(scenarios,)]TJ/F80 9.9626 Tf 116.416 0 Td [(e)15(.g)15(.)]TJ/F75 9.9626 Tf 14.087 0 Td [(,)-251(when)-251(ne)25(w)-251(unlabeled)-250(data)]TJ -130.503 -11.716 Td [(are)-234(made)-234(a)20(v)25(ailable)-234(progressi)25(v)15(ely)-233(o)15(v)15(er)-234(time.)-305(In)-234(f)10(act,)-237(in)-234(order)]TJ 0 -11.716 Td [(to)-314(inte)15(grate)-315(ne)25(w)-314(kno)25(wledge)-315(into)-314(the)-315(model,)-330(training)-315(needs)]TJ 0 -11.716 Td [(to)-337(be)-336(repeated)-337(on)-337(the)-336(whole)-337(dataset,)-358(which)-337(is)-337(impractical,)]TJ 0 -11.716 Td [(e)15(xpensi)25(v)15(e,)-233(and)-230(sometimes)-229(e)25(v)15(en)-229(impossible)-230(when)-229(old)-229(data)-230(is)]TJ 0 -11.716 Td [(not)-242(a)20(v)25(ailable.)-307(This)-241(issue)-242(is)-242(e)15(xacerbated)-241(by)-242(the)-241(f)10(act)-242(that)-242(SSL)]TJ 0 -11.716 Td [(models)-250(are)-250(notoriously)-250(computationally)-250(e)15(xpensi)25(v)15(e)-250(to)-250(train.)]TJ
0 g 0 G
ET
q
1 0 0 1 50.112 99.576 cm
[]0 d 0 J 0.398 w 0 0 m 94.499 0 l S
Q
BT
/F75 5.9776 Tf 60.971 91.653 Td [(*)]TJ/F75 6.9738 Tf 3.487 -1.492 Td [(Enrico)-250(Fini)-250(and)-250(V)60(ictor)-250(G.)-250(T)45(urrisi)-250(da)-250(Costa)-250(contrib)20(uted)-250(equally)65(.)]TJ/F75 5.9776 Tf -3.487 -6.348 Td [(\206)]TJ/F75 6.9738 Tf 3.487 -2.813 Td [(Uni)25(v)65(.)-310(Grenoble)-250(Alpes,)-250(CNRS,)-250(Grenoble)-250(INP)111(,)-250(LJK,)-250(38000)-250(Grenoble,)-250(France.)]TJ
0 g 0 G
0 g 0 G
0 g 0 G
0 g 0 G
ET
1 0 0 1 310.043 471.851 cm
q
.50299 0 0 .50299 0 0 cm
q
1 0 0 1 0 0 cm
/Im1 Do
Q
Q
1 0 0 1 0 -119.198 cm
q
.50299 0 0 .50299 0 0 cm
q
1 0 0 1 0 0 cm
/Im2 Do
Q
Q
0 g 0 G
1 0 0 1 -310.043 -352.653 cm
BT
/F75 8.9664 Tf 308.862 347.127 Td [(Figure)-370(1.)-672(Linear)-370(e)25(v)25(aluation)-370(accurac)15(y)-371(of)-370(representations)-370(learned)]TJ 0 -10.739 Td [(with)-396(dif)25(ferent)-397(self-supervised)-396(methods)-397(on)-396(class-incremental)-396(CI-)]TJ 0 -10.74 Td [(F)74(AR100)-416(and)-416(ImageNet100.)-808(In)-416(blue)-416(the)-416(accurac)15(y)-416(of)-417(SSL)-415(\002ne-)]TJ 0 -10.74 Td [(tuning,)-387(in)-360(green)-360(the)-360(impro)15(v)15(ement)-360(brought)-360(by)-360(CaSSLe.)-640(The)-360(red)]TJ 0 -10.739 Td [(dashed)-250(line)-250(is)-250(the)-250(accurac)15(y)-250(attained)-250(by)-250(supervised)-250(\002ne-tuning.)]TJ
0 g 0 G
0 g 0 G
/F75 9.9626 Tf 11.955 -18.068 Td [(Continual)-238(learning)-238(\050CL\051)-238(studies)-238(the)-238(ability)-239(of)-238(neural)-238(net-)]TJ -11.955 -11.716 Td [(w)10(orks)-327(to)-327(learn)-327(tasks)-328(sequentially)65(.)-541(Prior)-327(art)-327(in)-327(the)-328(\002eld)-327(fo-)]TJ 0 -11.716 Td [(cuses)-304(on)-304(mitig)5(ating)-303(catastrophic)-304(for)18(getting)-304([)]TJ
0 0 1 rg 0 0 1 RG
[(17)]TJ
0 g 0 G
[(,)]TJ
0 0 1 rg 0 0 1 RG
[-166(23)]TJ
0 g 0 G
[(,)]TJ
0 0 1 rg 0 0 1 RG
[-166(25)]TJ
0 g 0 G
[(,)]TJ
0 0 1 rg 0 0 1 RG
[-166(40)]TJ
0 g 0 G
[(].)]TJ 0 -11.716 Td [(Common)-283(benchmarks)-284(in)-283(the)-283(CL)-284(literature)-283(e)25(v)25(aluate)-283(the)-284(dis-)]TJ 0 -11.716 Td [(criminati)25(v)15(e)-361(performance)-362(of)-361(classi\002ers)-361(learned)]TJ/F80 9.9626 Tf 190.23 0 Td [(with)-361(super)20(-)]TJ -190.23 -11.716 Td [(vision)]TJ/F75 9.9626 Tf 27.032 0 Td [(from)-324(non-stationary)-325(distrib)20(utions.)-533(In)-324(this)-324(paper)40(,)-343(we)]TJ -27.032 -11.716 Td [(tackle)-399(the)-400(same)-399(for)18(getting)-399(phenomenon)-400(in)-399(the)-399(conte)15(xt)-400(of)]TJ 0 -11.716 Td [(SSL.)-448(Unsupervised)-449(representation)-448(learning)-448(is)-448(indeed)-449(ap-)]TJ 0 -11.716 Td [(pealing)-265(for)-265(sequential)-265(learning)-265(since)-265(it)-265(does)-265(not)-265(require)-265(hu-)]TJ 0 -11.716 Td [(man)-203(annotations,)-213(which)-203(are)-203(particularly)-203(hard)-203(to)-204(obtain)-203(when)]TJ 0 -11.716 Td [(ne)25(w)-258(data)-258(is)-258(generated)-257(on-the-\003y)65(.)-334(This)-258(setup,)-260(called)-258(Contin-)]TJ 0 -11.716 Td [(ual)-192(Self-Supervised)-192(Learning)-192(\050CSSL\051,)-192(is)-193(surprisingly)-192(under)20(-)]TJ 0 -11.716 Td [(in)40(v)15(estig)5(ated)-250(in)-250(the)-250(literature.)]TJ 11.955 -17.645 Td [(In)-352(this)-352(w)10(ork,)-378(we)-352(propose)-352(CaSSLe,)-378(a)-352(simple)-352(and)-352(ef)25(fec-)]TJ -11.955 -11.716 Td [(ti)25(v)15(e)-224(frame)25(w)10(ork)-224(for)-224(CSSL)-224(of)-224(visual)-225(representat)1(ions)-225(based)-224(on)]TJ 0 -11.716 Td [(the)-388(intuition)-388(that)-388(SSL)-388(models)-388(are)-388(intrinsically)-388(capable)-388(of)]TJ 0 -11.716 Td [(learning)-254(continually)65(,)-254(and)-254(that)-254(SSL)-254(losses)-253(can)-254(be)-254(seamlessly)]TJ 0 -11.716 Td [(con)40(v)15(erted)-213(into)-212(distillation)-213(losses.)-298(Our)-212(k)10(e)15(y)-213(idea)-213(is)-212(to)-213(train)-213(the)]TJ
0 g 0 G
-13.741 -29.888 Td [(1)]TJ
0 g 0 G
ET
endstream
endobj
65 0 obj
<<
/Type /XObject
/Subtype /Form
/FormType 1
/PTEX.FileName (./figures/cifar100.pdf)
/PTEX.PageNumber 1
/BBox [0 0 465 214]
/Group 83 0 R
/Resources <<
/Font << /Font2 85 0 R>>
/Pattern <<
>>/XObject <<
>>/ExtGState <<
/Alpha0 86 0 R
/Alpha1 87 0 R
>>/ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>>
/Length 16121
/Filter /FlateDecode
>>
stream
x}ُfIvWJ20`a"2Ʋ@`K,p4 g?qoTutόe*qcFbY>?^V#.~?W~V1;DEo^~W֟~a%o/_c?w?kyESϵ)&5_~bfw`(MW&~BJp`
^ԯS26_%]CwŇ+ZV=1h[KKJ%Zo/9/yKD+_vغ#oBH}w(4Tro^o.=y>JeDW!0Zb|Zp
/2iO g%i^4h' ,Z({"YDo5E{"n O&̯zajqG_?k-إ{^@i%"S-R}*R45XRɥgYR]rH{Jn)xH[;ό}k8˳rvS3k*T,iKEWKC* J.RVC>=%o)pkuߣc
}ysSg=yHE9o)_KC* )_5>w閲!UV)_DT̺DZs2=YwUF[jRѥfʐ|KTв6{9w
-|sgᚰC*ZfXé/_R<*bn+k~5"3_YS^Hfc }&7H\r1y\ą?>%!RxKoң"Dnh^soB>V^
(ذ?4_Qen&t|H@Cnw R'%HƷn?D
d]$ox7?:y&iCjxQOiD$2\&ݝlF7e@i1P_MoE Iɀ3A) 'kb()
ߣETSM17-y1D=)Cjղ$=$l/\ Հ#Nj_/ggGh q$u^爆-!Vǒ|g֢T*ӵMsCQ>ъmF?54ĸ
,dĉ"fR 2EVXr-\*AҶ<<(O~ȋ1X0-2\o9,ٷJwhY|p8%i-f{=ǻ-: