{"id":"https://openalex.org/W4391212851","doi":"https://doi.org/10.48550/arxiv.2401.12961","title":"Chatterbox: Robust Transport for LLM Token Streaming under Unstable Network","display_name":"Chatterbox: Robust Transport for LLM Token Streaming under Unstable Network","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4391212851","doi":"https://doi.org/10.48550/arxiv.2401.12961"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2401.12961","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false},"type":"preprint","type_crossref":"posted-content","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/abs/2401.12961","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5003455316","display_name":"Hanchen Li","orcid":"https://orcid.org/0009-0005-9980-028X"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"funder","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hanchen Li","raw_affiliation_strings":["University of Chicago"],"affiliations":[{"raw_affiliation_string":"University of Chicago","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100350539","display_name":"Yuhan Liu","orcid":"https://orcid.org/0009-0002-5957-5071"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"funder","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuhan Liu","raw_affiliation_strings":["University of Chicago"],"affiliations":[{"raw_affiliation_string":"University of Chicago","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004850046","display_name":"Yihua Cheng","orcid":"https://orcid.org/0009-0006-3924-6886"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"funder","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yihua Cheng","raw_affiliation_strings":["University of Chicago"],"affiliations":[{"raw_affiliation_string":"University of Chicago","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022254723","display_name":"Siddhant Ray","orcid":"https://orcid.org/0000-0003-0265-2144"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"funder","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Siddhant Ray","raw_affiliation_strings":["University of Chicago"],"affiliations":[{"raw_affiliation_string":"University of Chicago","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036272233","display_name":"Kuntai Du","orcid":"https://orcid.org/0000-0002-3964-4079"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"funder","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kuntai Du","raw_affiliation_strings":["University of Chicago"],"affiliations":[{"raw_affiliation_string":"University of Chicago","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103258769","display_name":"Junchen Jiang","orcid":"https://orcid.org/0000-0002-6877-1683"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"funder","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Junchen Jiang","raw_affiliation_strings":["University of Chicago"],"affiliations":[{"raw_affiliation_string":"University of Chicago","institution_ids":["https://openalex.org/I40347166"]}]}],"institution_assertions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":0,"max":77},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.9997,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.9997,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10742","display_name":"Peer-to-Peer Network Technologies","score":0.9974,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12238","display_name":"Green IT and Sustainability","score":0.9971,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[],"concepts":[{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.8141935},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.62840563},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.5892654},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.362352}],"mesh":[],"locations_count":3,"locations":[{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2401.12961","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2401.12961","pdf_url":"http://arxiv.org/pdf/2401.12961","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":false,"landing_page_url":"https://api.datacite.org/dois/10.48550/arxiv.2401.12961","pdf_url":null,"source":{"id":"https://openalex.org/S4393179698","display_name":"DataCite API","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I4210145204","host_organization_name":"DataCite","host_organization_lineage":["https://openalex.org/I4210145204"],"host_organization_lineage_names":["DataCite"],"type":"metadata"},"license":null,"license_id":null,"version":null}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2401.12961","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[],"grants":[],"datasets":[],"versions":[],"referenced_works_count":23,"referenced_works":["https://openalex.org/W1496108277","https://openalex.org/W1709547319","https://openalex.org/W1964154540","https://openalex.org/W2118647282","https://openalex.org/W2129636357","https://openalex.org/W2545990795","https://openalex.org/W2790105919","https://openalex.org/W2798388120","https://openalex.org/W2921769405","https://openalex.org/W3038441907","https://openalex.org/W3210089935","https://openalex.org/W4230038347","https://openalex.org/W4247981085","https://openalex.org/W4256648168","https://openalex.org/W4281758439","https://openalex.org/W4301710896","https://openalex.org/W4310561894","https://openalex.org/W4324297016","https://openalex.org/W4376167141","https://openalex.org/W4377864171","https://openalex.org/W4387321091","https://openalex.org/W4392489911","https://openalex.org/W4395686786"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W4388335561","https://openalex.org/W4385572700","https://openalex.org/W4385009901","https://openalex.org/W4307309205","https://openalex.org/W4288261899","https://openalex.org/W2970530566","https://openalex.org/W2967478618","https://openalex.org/W2899084033","https://openalex.org/W2748952813"],"abstract_inverted_index":{"To":[0],"render":[1],"each":[2,22,142],"generated":[3,126],"token":[4,23,45,54,101,178],"in":[5,104,134,149],"real-time":[6],"for":[7,225],"users,":[8],"the":[9,31,34,52,68,75,135,150,158,186,200,210,217],"Large":[10],"Language":[11],"Model":[12],"(LLM)":[13],"server":[14],"generates":[15],"tokens":[16,72,127,133,147],"one":[17,19,63],"by":[18,162,182,191,196],"and":[20,112,195],"streams":[21],"(or":[24],"group":[25],"of":[26,70,177,213],"a":[27,83,117],"few":[28],"tokens)":[29],"through":[30],"network":[32,50],"to":[33,42,185,199,208,219,227],"user":[35],"right":[36],"after":[37],"generation,":[38],"which":[39,123],"we":[40,86,115,170,215],"refer":[41],"as":[43,128,130],"LLM":[44,53,105],"streaming.":[46],"However,":[47],"under":[48,95,167],"unstable":[49,96],"conditions,":[51],"streaming":[55,102],"experience":[56],"could":[57,66],"suffer":[58,91],"greatly":[59],"from":[60,92,109],"stalls":[61,94,160],"since":[62],"packet":[64,143,202],"loss":[65],"block":[67],"rendering":[69,179],"later":[71],"even":[73],"if":[74],"packets":[76],"containing":[77],"them":[78],"arrive":[79],"on":[80],"time.":[81],"With":[82],"measurement":[84],"study,":[85],"show":[87,171],"that":[88,107,141],"current":[89],"applications":[90,194],"increased":[93],"networks.":[97],"For":[98],"this":[99],"emerging":[100],"problem":[103],"Chatbots":[106,218],"differs":[108],"previous":[110],"multimedia":[111],"text":[113],"applications,":[114],"propose":[116],"novel":[118],"transmission":[119],"scheme,":[120],"called":[121],"Eloquent,":[122],"puts":[124],"newly":[125],"well":[129],"currently":[131],"unacknowledged":[132],"next":[136],"outgoing":[137],"packet.":[138],"This":[139],"ensures":[140],"contains":[144],"some":[145],"new":[146],"and,":[148],"meantime,":[151],"is":[152],"independently":[153],"rendered":[154],"when":[155],"received,":[156],"avoiding":[157],"aforementioned":[159],"caused":[161],"missing":[163],"packets.":[164],"Through":[165],"simulation":[166],"various":[168],"networks,":[169],"Eloquent":[172,207],"reduces":[173],"stall":[174],"ratio":[175],"(proportion":[176],"wait":[180],"time)":[181],"71.0%":[183],"compared":[184,198],"retransmission":[187],"method":[188],"commonly":[189],"used":[190],"real":[192],"chatbot":[193],"31.6%":[197],"baseline":[201],"duplication":[203],"scheme.":[204],"By":[205],"tailoring":[206],"fit":[209],"token-by-token":[211],"generation":[212],"LLM,":[214],"enable":[216],"respond":[220],"like":[221],"an":[222],"eloquent":[223],"speaker":[224],"users":[226],"better":[228],"enjoy":[229],"pervasive":[230],"AI.":[231]},"abstract_inverted_index_v3":null,"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4391212851","counts_by_year":[],"updated_date":"2025-04-19T23:31:55.658214","created_date":"2024-01-25"}