{"id":"https://openalex.org/W4398796719","doi":"https://doi.org/10.48550/arxiv.2405.14239","title":"Harmony: A Joint Self-Supervised and Weakly-Supervised Framework for\n Learning General Purpose Visual Representations","display_name":"Harmony: A Joint Self-Supervised and Weakly-Supervised Framework for\n Learning General Purpose Visual Representations","publication_year":2024,"publication_date":"2024-05-23","ids":{"openalex":"https://openalex.org/W4398796719","doi":"https://doi.org/10.48550/arxiv.2405.14239"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.14239","pdf_url":"http://arxiv.org/pdf/2405.14239","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"type":"preprint","type_crossref":"posted-content","indexed_in":["arxiv"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://arxiv.org/pdf/2405.14239","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5092998246","display_name":"Mohammed Baharoon","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Baharoon, Mohammed","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063162883","display_name":"Jonathan Klein","orcid":"https://orcid.org/0000-0001-6560-0988"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Klein, Jonathan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5039203502","display_name":"Dominik L. Michels","orcid":"https://orcid.org/0000-0002-1621-325X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Michels, Dominik L.","raw_affiliation_strings":[],"affiliations":[]}],"institution_assertions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":0,"max":84},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9318,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9318,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10531","display_name":"Advanced Vision and Imaging","score":0.929,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9274,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/harmony","display_name":"Harmony (Music)","score":0.7300672}],"concepts":[{"id":"https://openalex.org/C2776453491","wikidata":"https://www.wikidata.org/wiki/Q5659234","display_name":"Harmony (color)","level":2,"score":0.7300672},{"id":"https://openalex.org/C18555067","wikidata":"https://www.wikidata.org/wiki/Q8375051","display_name":"Joint (building)","level":2,"score":0.54410195},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.49694422},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48311946},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.40991867},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4050557},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.38026732},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0937956},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.06646231},{"id":"https://openalex.org/C170154142","wikidata":"https://www.wikidata.org/wiki/Q150737","display_name":"Architectural engineering","level":1,"score":0.0},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.14239","pdf_url":"http://arxiv.org/pdf/2405.14239","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.14239","pdf_url":"http://arxiv.org/pdf/2405.14239","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[],"grants":[],"datasets":[],"versions":[],"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4394896187","https://openalex.org/W4386462264","https://openalex.org/W4364306694","https://openalex.org/W4312192474","https://openalex.org/W4306674287","https://openalex.org/W4283697347","https://openalex.org/W3170094116","https://openalex.org/W3107602296","https://openalex.org/W3046775127","https://openalex.org/W2961085424"],"abstract_inverted_index":{"Vision-language":[0],"contrastive":[1],"learning":[2,7,56,207],"frameworks":[3],"like":[4,47,209],"CLIP":[5,126,150],"enable":[6],"representations":[8],"from":[9],"natural":[10],"language":[11],"supervision,":[12],"and":[13,49,87,118,142,151,157,161,175,184,188,211],"provide":[14],"strong":[15],"zero-shot":[16,176],"classification":[17,177],"capabilities.":[18],"However,":[19],"due":[20],"to":[21,35,40,62,90,107],"the":[22,25,33,52,60,67,120,148,152],"nature":[23],"of":[24],"supervisory":[26],"signal":[27],"in":[28,70,173],"these":[29,167],"paradigms,":[30],"they":[31],"lack":[32],"ability":[34,61],"learn":[36,63,91],"localized":[37],"features,":[38],"leading":[39,154],"degraded":[41],"performance":[42,172],"on":[43,109,115,178,182,191,197],"dense":[44],"prediction":[45],"tasks":[46,141,215],"segmentation":[48,181,190],"detection.":[50],"On":[51,217],"other":[53,205],"hand,":[54],"self-supervised":[55,206],"methods":[57,208],"have":[58],"shown":[59],"granular":[64],"representations,":[65],"complementing":[66],"high-level":[68],"features":[69,93],"vision-language":[71,83],"training.":[72],"In":[73],"this":[74],"work,":[75],"we":[76],"present":[77],"Harmony,":[78],"a":[79,195],"framework":[80,103],"that":[81,94,144,202],"combines":[82],"training":[84],"with":[85],"discriminative":[86],"generative":[88],"self-supervision":[89],"visual":[92],"can":[95],"be":[96],"generalized":[97],"across":[98,137,213],"vision":[99,139],"downstream":[100,140],"tasks.":[101],"Our":[102],"is":[104,221],"specifically":[105],"designed":[106],"work":[108],"web-scraped":[110],"data":[111],"by":[112,129],"not":[113],"relying":[114],"negative":[116],"examples":[117],"addressing":[119],"one-to-one":[121],"correspondence":[122],"issue":[123],"using":[124],"soft":[125],"targets":[127],"generated":[128],"an":[130],"EMA":[131],"model.":[132],"We":[133,199],"comprehensively":[134],"evaluate":[135],"Harmony":[136,169,203],"various":[138],"find":[143],"it":[145],"significantly":[146],"outperforms":[147,204],"baseline":[149],"previously":[153],"joint":[155],"self":[156],"weakly-supervised":[158],"methods,":[159,168],"MaskCLIP":[160],"SLIP.":[162],"Specifically,":[163],"when":[164,193],"comparing":[165],"against":[166],"shows":[170],"superior":[171],"fine-tuning":[174],"ImageNet-1k,":[179],"semantic":[180],"ADE20K,":[183],"both":[185],"object":[186],"detection":[187],"instance":[189],"MS-COCO,":[192],"pre-training":[194],"ViT-S/16":[196],"CC3M.":[198],"also":[200],"show":[201],"iBOT":[210],"MAE":[212],"all":[214],"evaluated.":[216],"https://github.com/MohammedSB/Harmony":[218],"our":[219],"code":[220],"publicly":[222],"available.":[223]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4398796719","counts_by_year":[],"updated_date":"2024-12-07T11:30:13.486783","created_date":"2024-05-25"}