{"id":"https://openalex.org/W4400104234","doi":"https://doi.org/10.48550/arxiv.2406.17565","title":"MemServe: Context Caching for Disaggregated LLM Serving with Elastic\n Memory Pool","display_name":"MemServe: Context Caching for Disaggregated LLM Serving with Elastic\n Memory Pool","publication_year":2024,"publication_date":"2024-06-25","ids":{"openalex":"https://openalex.org/W4400104234","doi":"https://doi.org/10.48550/arxiv.2406.17565"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.17565","pdf_url":"http://arxiv.org/pdf/2406.17565","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"type":"preprint","type_crossref":"posted-content","indexed_in":["arxiv"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://arxiv.org/pdf/2406.17565","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063322816","display_name":"Cunchen Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Cunchen","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111241693","display_name":"Heyang Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Heyang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029468076","display_name":"Junhao Hu","orcid":"https://orcid.org/0000-0003-3538-2785"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Junhao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030419876","display_name":"Jiang Xu","orcid":"https://orcid.org/0000-0001-7722-8933"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Jiang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101527007","display_name":"Xusheng Chen","orcid":"https://orcid.org/0000-0002-2807-9780"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Xusheng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103174002","display_name":"Tao Xie","orcid":"https://orcid.org/0000-0002-0467-5422"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xie, Tao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100334369","display_name":"Chenxi Wang","orcid":"https://orcid.org/0000-0002-1451-3101"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Chenxi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033922611","display_name":"Sa Wang","orcid":"https://orcid.org/0000-0002-9629-0860"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Sa","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101807410","display_name":"Yungang Bao","orcid":"https://orcid.org/0000-0001-6565-5276"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bao, Yungang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101738013","display_name":"Ninghui Sun","orcid":"https://orcid.org/0000-0002-1953-1392"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Ninghui","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5111117324","display_name":"Yizhou Shan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shan, Yizhou","raw_affiliation_strings":[],"affiliations":[]}],"institution_assertions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":0,"max":84},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10080","display_name":"Energy Efficient Wireless Sensor Networks","score":0.9877,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10080","display_name":"Energy Efficient Wireless Sensor Networks","score":0.9877,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.9873,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10444","display_name":"Context-Aware Activity Recognition Systems","score":0.942,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[],"concepts":[{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.66130495},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.54364157},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.19690144},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.17565","pdf_url":"http://arxiv.org/pdf/2406.17565","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false}],"best_oa_location":{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.17565","pdf_url":"http://arxiv.org/pdf/2406.17565","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[],"grants":[],"datasets":[],"versions":[],"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4396701345","https://openalex.org/W4396696052","https://openalex.org/W4395014643","https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2382290278","https://openalex.org/W2376932109","https://openalex.org/W2358668433","https://openalex.org/W2001405890"],"abstract_inverted_index":{"Large":[0],"language":[1],"model":[2],"(LLM)":[3],"serving":[4,63],"has":[5],"transformed":[6],"from":[7],"stateless":[8],"to":[9],"stateful":[10],"systems,":[11],"utilizing":[12],"techniques":[13],"like":[14],"context":[15,70],"caching":[16,71],"and":[17,25,46,59,104],"disaggregated":[18,73],"inference.":[19],"These":[20],"optimizations":[21],"extend":[22],"the":[23,28,76],"lifespan":[24],"domain":[26],"of":[27],"KV":[29,60],"cache,":[30],"necessitating":[31],"a":[32,39,81,89],"new":[33],"architectural":[34],"approach.":[35],"We":[36],"present":[37],"MemServe,":[38],"unified":[40],"system":[41],"that":[42,84,97],"integrates":[43],"both":[44],"inter-request":[45],"intra-request":[47],"optimizations.":[48],"MemServe":[49,68,98],"introduces":[50],"MemPool,":[51],"an":[52],"elastic":[53],"memory":[54,58],"pool":[55],"managing":[56],"distributed":[57],"caches":[61],"across":[62],"instances.":[64],"Using":[65],"MemPool":[66],"APIs,":[67],"combines":[69],"with":[72],"inference":[74],"for":[75],"first":[77],"time,":[78],"supported":[79],"by":[80],"global":[82,90],"scheduler":[83],"enhances":[85],"cache":[86],"reuse":[87],"through":[88],"prompt":[91],"tree-based":[92],"locality-aware":[93],"policy.":[94],"Tests":[95],"show":[96],"significantly":[99],"improves":[100],"job":[101],"completion":[102],"time":[103],"time-to-first-time.":[105]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4400104234","counts_by_year":[],"updated_date":"2024-12-15T08:37:43.319673","created_date":"2024-06-28"}