[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-639360-105":59,"doc-detail-639360-en":129},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":122,"head_meta":124,"extra_data":126,"updated_unix":128},105,"en","thus-spake-long-context-large-language-model-survey-overview","Thus Spake Long-Context Large Language Model - survey overview","","Long context is a central theme in natural language processing and a key competitive driver for large language models, yet extending context length introduces substantial architectural, infrastructural, training, and evaluation challenges. Over the past two years, context windows have expanded to millions of tokens, shifting research from length extrapolation toward end-to-end methods for architecture, infrastructure, training pipelines, and benchmark design. This survey provides a lifecycle view of long-context LLMs and concludes with 10 open research questions.",{"@graph":69,"@context":121},[70,84,104],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/thus-spake-long-context-large-language-model-survey-overview/639360/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":98,"encodingFormat":97,"isAccessibleForFree":99,"interactionStatistic":100},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/thus-spake-long-context-large-language-model-survey-overview/639360.png","ImageObject",300,407,{"name":92,"@type":93},"cksankcs","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-22",true,{"@type":101,"interactionType":102,"userInteractionCount":4},"InteractionCounter",{"@type":103},"ViewAction",{"@type":105,"mainEntity":106},"FAQPage",[107,113,117],{"name":108,"@type":109,"acceptedAnswer":110},"What does the survey focus on regarding long-context LLMs?","Question",{"text":111,"@type":112},"It gives a global picture of the lifecycle of long-context LLMs from four perspectives: architecture, infrastructure, training, and evaluation, and summarizes the full spectrum of long-context technologies.","Answer",{"name":114,"@type":109,"acceptedAnswer":115},"Why is extending context length difficult for LLMs?",{"text":116,"@type":112},"Longer contexts create obstacles across multiple dimensions, including pre-training window limits and KV-cache overhead for architecture, memory pressure and throughput reduction for infrastructure, dataset quantity and quality issues for training, and more uncovered failure modes for evaluation.",{"name":118,"@type":109,"acceptedAnswer":119},"How has context length progressed in recent years, according to the text?",{"text":120,"@type":112},"The text states that open-source LLM context lengths grew from around 2k to about 2M from April 2023 to February 2024, reflecting rapid development in long-context research.","https://schema.org",{"og:url":83,"og:type":123,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":125,"canonical":83},"index,follow",{"doc_id":127,"site_id":62},639360,1792703265,{"code":4,"msg":5,"data":130},{"doc_id":127,"user_id":131,"nickname":92,"user_avatar":132,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":133,"file_id":134,"file_url":135,"file_type":136,"file_size":137,"view_count":4,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":138,"language":139,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":140,"faqs":141,"seo_title":142,"seo_description":67,"update_tm":128,"read_time":143},3985747882755,"https://ap-avatar.wpscdn.com/davatar_29158cc5080c5b710cf443261637dec0","arXiv :2502 . 17129v2 [ cs .CL] 11 Nov 2025  \nThus Spake Long-Context Large Language Model  \nXiaoran Liu1,2,3Ruixiao Li2,3Mianqiu Huang2Zhigeng Liu2,3Yuerong Song2,3 Qipeng Guo1,3†, Siyang He2,3, Qiqi Wang2,3, Linlin Li4, Qun Liu4,  \nZiwei He3, Yaqian Zhou2, Xuanjing Huang2, Xipeng Qiu2,3†  \n1Shanghai AI Lab, 2School of Computer Science, Fudan University, 3Shanghai Innovation Institute, 4Huawei Noah’s Ark Lab  \n[xrliu24@m.fudan.edu.cn](xrliu24@m.fudan.edu.cn), [guoqipeng@pjlab.org.cn](guoqipeng@pjlab.org.cn), [xpqiu@fudan.edu.cn](xpqiu@fudan.edu.cn)  \nAbstract  \nLong context is an important topic in Natural Language Processing (NLP), running through the development of NLP architectures, and offers immense opportunities for Large Language Models (LLMs), giving LLMs the lifelong learning potential akin to humans. Unfortunately, the pursuit of a long context is accompanied by numerous obstacles. Nevertheless, long context remains a core competitive advantage for LLMs. In the past two years, the context length of LLMs has achieved a breakthrough extension to millions of tokens. Moreover, research on long-context LLMs has expanded beyond length extrapolation to a comprehensive focus on architecture, infrastructure, training, and evaluation technologies.  \nInspired by the symphonic poem, Thus Spake Zarathustra, we draw an analogy between the journey of extending the context of LLM and the attempts of humans to transcend their mortality. In this survey, we will illustrate how LLM struggles between the tremendous need for a longer context and its equal need to accept the fact that it is ultimately finite. To achieve this, we give a global picture of the lifecycle of long-context LLMs from four perspectives: architecture, infrastructure, training, and evaluation, showcasing the full spectrum of long-context technologies. At the end of this survey, we will present 10 unanswered questions currently faced by longcontext LLMs. We hope this survey can serve as a systematic introduction to research on long-context LLMs.  \nVideo: [https://www.bilibili.com/video/BV1](https://www.bilibili.com/video/BV1) 1h9AYoEYj.  \nGithub: [https://github.com/OpenMOSS/Thus-Spake-Long-Context-LLM](https://github.com/OpenMOSS/Thus-Spake-Long-Context-LLM).  \nFigure 1: An overview of Thus Spake Long-Context Large Language Model.  \n∗ Equal contribution.† Corresponding Author.  \n1 Introduction  \nResearch on long-context capability has been an important topic in Natural Language Processing (NLP), reflected in the evolutionary trajectory of mainstream architectures. This evolution shows a consistent progression toward increasing context length, from the Bagof-Word models (Har) with no concept of context to CNNs (LeCun et al., 1995) with local receptive fields, then to LSTMs (Schmidhuber et al., 1997) characterized with an explicit long short-term memory, and currently to Transformer featured with modeling long-range dependencies (Vaswani et al., 2017), as well as the recent discussions on the SSM-Mamba series (Gu et al., 2020; Gu & Dao, 2023; Dao & Gu, 2024) that challenges the dominance of Transformers from the perspective of history storage. Researchers hope models, especially the Large Language Model (LLM) (OpenAI, 2023; Sun et al., 2024c; Reid et al., 2024; Meta, 2024a), can possess life-long context, rather than being limited by a fixed window size.  \nIn a 1k context, LLM may only understand a short fairy tale. In a 4k context, the reading comprehension may be limited to an arXiv paper (Shaham et al., 2022) . In a 32k to 128k context, LLM may process a hundreds-page detective novel in its entirety and successfully infer the identity of the murderer (Xu et al., 2024f; Wang et al., 2024b) . When the context length extends to 512k, even a novel as lengthy as Ulysses or a novel series could be input and understood as a whole (Jacobs et al., 2023) . When the context length reaches 2M, the model may learn new knowledge through many-shot long In-Context Learning (ICL) (","cbCaia9ziJknnV46","https://ap.wps.com/l/cbCaia9ziJknnV46","pdf",1428054,92,"English","# Introduction\n## Evolution of long-context capabilities\n## Challenges with increasing context length\n## Recent progress and survey scope","[{\"question\":\"What does the survey focus on regarding long-context LLMs?\",\"answer\":\"It gives a global picture of the lifecycle of long-context LLMs from four perspectives: architecture, infrastructure, training, and evaluation, and summarizes the full spectrum of long-context technologies.\"},{\"question\":\"Why is extending context length difficult for LLMs?\",\"answer\":\"Longer contexts create obstacles across multiple dimensions, including pre-training window limits and KV-cache overhead for architecture, memory pressure and throughput reduction for infrastructure, dataset quantity and quality issues for training, and more uncovered failure modes for evaluation.\"},{\"question\":\"How has context length progressed in recent years, according to the text?\",\"answer\":\"The text states that open-source LLM context lengths grew from around 2k to about 2M from April 2023 to February 2024, reflecting rapid development in long-context research.\"}]","Thus Spake Long-Context Large Language Model - survey overview | PDF",232]