[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-149245-105":59,"doc-detail-149245-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","griffin-mixing-gated-linear-recurrences-with-local-attention-for-efficient-language-models","Griffin - Mixing Gated Linear Recurrences with Local Attention for Efficient Language Models","","Recurrent neural networks enable fast inference and efficient long-sequence scaling, yet remain difficult to train and extend. This work introduces Hawk, an RNN using gated linear recurrences, and Griffin, a hybrid model combining gated linear recurrences with local attention. Hawk outperforms reported Mamba results, while Griffin matches Llama-2 performance with far fewer tokens. The study also demonstrates longer-context extrapolation, training hardware efficiency comparable to Transformers, and improved inference throughput and latency, including distributed training sharding up to 14B parameters.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/griffin-mixing-gated-linear-recurrences-with-local-attention-for-efficient-language-models/149245/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/griffin-mixing-gated-linear-recurrences-with-local-attention-for-efficient-language-models/149245.png","ImageObject",300,407,{"name":92,"@type":93},"Himbo","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-29","2026-08-27",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",16,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"What are Hawk and Griffin in this paper?","Question",{"text":113,"@type":114},"Hawk is an RNN built with gated linear recurrences. Griffin is a hybrid model that mixes gated linear recurrences with local attention and integrates them with MLP blocks.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"How do the models compare to Mamba and Llama-2?",{"text":118,"@type":114},"Hawk exceeds the reported performance of Mamba on downstream tasks, while Griffin matches Llama-2 performance despite being trained on substantially fewer tokens.",{"name":120,"@type":111,"acceptedAnswer":121},"What efficiency improvements are reported versus Transformers during inference?",{"text":122,"@type":114},"The paper reports lower inference latency on long sequences and significantly higher throughput than MQA Transformers, while training efficiency remains comparable to Transformers on TPU-v3.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},149245,1787797149,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":140,"language":141,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":67,"update_tm":130,"read_time":145,"preview_page_count":4},687197100911,"https://ap-avatar.wpscdn.com/avatar/a000239b6f1da00475?x-image-process=image/resize,m_fixed,w_180,h_180&k=1785132997149421697","arXiv :2402 . 19427v1 [ cs .LG] 29 Feb 2024  \nGriffin: Mixing Gated Linear Recurrences with Local Attention for Efficient Language Models  \nSoham De* 1, Samuel L. Smith* 1, Anushan Fernando*1, Aleksandar Botev*1, George Cristian-Muraru*1, Albert Gu2 , Ruba Haroun1 , Leonard Berrada1 , Yutian Chen1 , Srivatsan Srinivasan1 , Guillaume Desjardins1 , Arnaud Doucet1 , David Budden1 , Yee Whye Teh1 , Razvan Pascanu1 , Nando De Freitas1 and Caglar Gulcehre1  \n* Equal contributions, 1 Google DeepMind, 2Work done while at Google DeepMind  \nRecurrent neural networks (RNNs) have fast inference and scale efficiently on long sequences, but they are difficult to train and hard to scale. We propose Hawk, an RNN with gated linear recurrences, and Griffin, a hybrid model that mixes gated linear recurrences with local attention. Hawk exceeds the reported performance of Mamba on downstream tasks, while Griffin matches the performance of Llama-2 despite being trained on over 6 times fewer tokens. We also show that Griffin can extrapolate on sequences significantly longer than those seen during training. Our models match the hardware efficiency of Transformers during training, and during inference they have lower latency and significantly higher throughput. We scale Griffin up to 14B parameters, and explain how to shard our models for efficient distributed training.  \n1. Introduction  \nRecurrent neural networks (RNNs) played a central role in the early days of deep learning and NLP research (Elman, 1990; Siegelmann and Sontag, 1991; Hochreiter and Schmidhuber, 1997; Mikolov et al., 2010; Bahdanau et al., 2014; Sutskever et al., 2014), and achieved practical success in many applications, including Google’s first end to end machine translation system (Wu et al., 2016) . However in recent years, both deep learning and NLP have been dominated by the Transformer architecture (Vaswaniet al., 2017), which interleaves multi-layer perceptrons (MLPs) and multi-head attention (MHA) . Transformers achieve better performance than RNNs in practice and are also very efficient at utilizing modern hardware (Kaplan et al., 2020) . Transformer-based large language models trained on massive datasets collected from the web have achieved remarkable success (Brown et al., 2020; Rae et al., 2021; Hoffmann et al., 2022; Touvron et al., 2023; Achiam et al., 2023; Gemini Team Google, 2023) .  \nDespite their successes, Transformers are difficult to scale efficiently to long sequences due to the quadratic complexity of global attention. Additionally, the linear growth of the Key-Value (KV) cache with the sequence length makes Transformers slow during inference. Although Multi-Query Attention (MQA) (Shazeer, 2019) partially mitigates this issue by reducing the cache size by a constant factor, the cache still grows linearly in sequence length. Recurrent language models present a compelling alternative as they compress the entire sequence into a fixed-sized hidden state which is updated iteratively. However to replace Transformers, new RNN models must demonstrate not only comparable performance at scale but also achieve similar hardware efficiency (Gu et al., 2021a; Mehta et al., 2022; Smith et al., 2022; Orvieto et al., 2023b; Dao et al., 2022b; Poli et al., 2023; Gu and Dao, 2023) .  \nIn this work, we propose the RG-LRU layer, a novel gated linear recurrent layer, around which we design a new recurrent block to replace MQA. We build two new models using this recurrent block: Hawk, a model which interleaves MLPs with recurrent blocks, and Griffin, a hybrid model which interleaves MLPs with a mixture of recurrent blocks and local attention (Beltagy et al., 2020) . We show that:  \n1. Hawk and Griffin exhibit power law scaling between held-out loss and training FLOPs, up to and beyond 7B parameters (Figure 1(a)), as previously observed for Transformers (Kaplan et al., 2020) .  \n2. Griffin achieves slightly lower held-out loss than strong Transformer baselines at all model scales. ","cbCaibBhVLuW5V7g","https://ap.wps.com/l/cbCaibBhVLuW5V7g","pdf",951680,25,"English","# Introduction\n## Model Architecture\n## Scaling and Throughput Evaluation\n## Extrapolation to Longer Sequences\n## Training Efficiency and Inference Performance","[{\"question\":\"What are Hawk and Griffin in this paper?\",\"answer\":\"Hawk is an RNN built with gated linear recurrences. Griffin is a hybrid model that mixes gated linear recurrences with local attention and integrates them with MLP blocks.\"},{\"question\":\"How do the models compare to Mamba and Llama-2?\",\"answer\":\"Hawk exceeds the reported performance of Mamba on downstream tasks, while Griffin matches Llama-2 performance despite being trained on substantially fewer tokens.\"},{\"question\":\"What efficiency improvements are reported versus Transformers during inference?\",\"answer\":\"The paper reports lower inference latency on long sequences and significantly higher throughput than MQA Transformers, while training efficiency remains comparable to Transformers on TPU-v3.\"}]","Griffin - Mixing Gated Linear Recurrences with Local Attention for Efficient Language Models | PDF",63]