[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-273481-105":59,"doc-detail-273481-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","does-the-lm-head-create-a-harmful-gradient-bottleneck-a-causal-test","Does the LM Head Create a Harmful Gradient Bottleneck? A Causal Test","","The language-model head maps a hidden state into a vocabulary logits space, making its transpose capable of returning only a limited set of independent gradient directions. Godey and Artzi claim this projection is a harmful optimization bottleneck. A causal backward-only intervention preserves ordinary logits and the exact LM-head parameter update while reducing only the gradient rank into the Transformer; across five paired seeds it increases validation loss. A factorized forward-head intervention increases loss substantially more, and controls indicate failures are confounded or diagnostic signals are weak.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/does-the-lm-head-create-a-harmful-gradient-bottleneck-a-causal-test/273481/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/does-the-lm-head-create-a-harmful-gradient-bottleneck-a-causal-test/273481.png","ImageObject",300,407,{"name":92,"@type":93},"Aurora","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-27","2026-09-15",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",11,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"What claim does the paper test about the LM head and gradients?","Question",{"text":113,"@type":114},"It tests whether the LM head’s projection from hidden states to vocabulary logits creates a harmful gradient bottleneck that meaningfully harms optimization.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"How does the paper perform its causal test?",{"text":118,"@type":114},"It uses a backward-only intervention: keeping ordinary logits and the exact LM-head parameter update while reducing only the rank of the gradient sent into the Transformer.",{"name":120,"@type":111,"acceptedAnswer":121},"What do the experiments show about backward rank reduction versus forward-decoder changes?",{"text":122,"@type":114},"Reducing backward rank increases validation loss, but changing the forward decoder via a low-rank factorized head increases loss substantially more; thus the forward restriction is more harmful.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},273481,1789471648,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":140,"language":141,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":67,"update_tm":130,"read_time":145,"preview_page_count":4},4810365810221,"https://ap-avatar.wpscdn.com/davatar_155a257f0dc6eb9ab79c44ca47cae57d","AI RESEARCH  \nDoes the LM Head Create a Harmful Gradient Bottleneck? A Causal Test  \nAnand Murugan • August 17, 2026 • Research article • A causal test ofthe LM‑head bottleneck claim  \narXiv :2608 . 16671v1 [ cs .CL] 17 Aug 2026  \nIN THIS ARTICLE  \nMain Findings  \nWhat “Lost” Means Here  \nA Clean Test of Backward Rank Compact Byte‑Level Language Model Larger Subword Language Model  \nIs the Remaining Logit Error Already Used? SpamLang: Repeated Tokens Are Not Independent Mapping Examples  \nIncreasing the Output Vocabulary Without Changing the Text  \nDo Projection Measurements Predict Learning? Secondary Evidence: Tested Auxiliary Feedback Routes  \nWhat the Experiments Establish Relation to Earlier Work Limits of This Study Reproducing the Results  \nConclusion  \nAbstract  \nThe language-model head maps a hidden state of width 􀓹 to a vocabulary of size 􀔋 , so its transpose can return at most 􀓹 independent directions to the Transformer. Godey and Artzi argue that this severe projection is a harmful optimization bottleneck. We separate the geometry from the causal claim. Our backward-only intervention keeps the ordinary logits and the exact LM-head parameter update while reducing only the rank of the gradient sent into the Transformer. Across five paired seeds on byte-level and BPE-8192 WikiText-2 models, reducing backward rank increases validation loss. An equally ranked factorized forward head, however, increases loss substantially more. At half rank in the larger model, the backward-only loss increase is 0.0586 (95% CI [0.0167, 0.1005]), while  \nthe factorized forward head increases loss by 0.1795 ([0.1547, 0.2042]) . The vocabularyspace residual also contributes to the ordinary LM-head update, and removing that contribution is harmful. Additional controls show that repeated-token failures are confounded by the number of independently sampled symbols, that adding never-target output classes does not impair learning, and that projection diagnostics do not reliably predict progress in our runs. Tested auxiliary feedback routes do not beat tuned backpropagation. These results confirm strong geometric compression but do not establish that it is a harmful optimization bottleneck.  \nThe language-model head, or LM head, maps the Transformer's final hidden state to one logit for every token in the vocabulary. For hidden width 􀓹 and vocabulary size 􀔋 , the standard head is a trainable matrix 􀔌 ∈ ℝ􀕷 ×􀕥 :  \n􀔩 = 􀔌 ℎ,  \nwhere ℎ ∈ ℝ􀕥 and 􀔩 ∈ ℝ 􀕷 . Backpropagation sends the logit gradient back to the hidden state through:  \n􀔖ℎ = 􀔌􀕵 􀔖􀖕 .  \nThe hidden-state gradient therefore depends only on the component of 􀔖􀖕 in the column space of 􀔌 , a subspace of dimension at most 􀓹. When 􀔋 ≫ 􀓹, most of the Euclidean norm of 􀔖􀖕 can lie outside that subspace.  \nGodey and Artzi call this a gradient bottleneck. Their paper, Lost in Backpropagation: The LM Head is a Gradient Bottleneck, reports that the return path often retains only 1–5% of the logitgradient norm. In other words, 95–99% lies outside the directions that can immediately reach the final hidden state.  \nThat geometric fact is real. The open question is what it means.  \nA large projection residual does not by itself show that useful learning information was destroyed. A 􀓹-dimensional state necessarily has only 􀓹 local degrees of freedom. The remaining logit-space directions may be incompatible across examples, not representable by the shared model, useful through other parameters, or unimportant for reducing future loss.  \nThis leaves two separate experimental questions:  \n1. If we keep the usual logits and LM-head update, but remove some directions only from the gradient sent into the Transformer, how much does training degrade?  \n2. If we keep standard backpropagation intact, can we compress the vocabulary-space residual, inject it into earlier Transformer layers and improve training?  \nThe first question gives a clean causal control for the original paper's low-rank-head experiment. We compare th","cbCainea43UOCGWB","https://ap.wps.com/l/cbCainea43UOCGWB","pdf",341239,17,"English","# Abstract\n## Main Findings\n## What “Lost” Means Here\n## A Clean Test of Backward Rank Compact Byte-Level Language Model\n## SpamLang: Repeated Tokens Are Not Independent Mapping Examples\n## Increasing the Output Vocabulary Without Changing the Text\n## Do Projection Measurements Predict Learning?\n## Secondary Evidence: Tested Auxiliary Feedback Routes\n## What the Experiments Establish Relation to Earlier Work\n## Limits of This Study\n## Reproducing the Results\n## Conclusion","[{\"question\":\"What claim does the paper test about the LM head and gradients?\",\"answer\":\"It tests whether the LM head’s projection from hidden states to vocabulary logits creates a harmful gradient bottleneck that meaningfully harms optimization.\"},{\"question\":\"How does the paper perform its causal test?\",\"answer\":\"It uses a backward-only intervention: keeping ordinary logits and the exact LM-head parameter update while reducing only the rank of the gradient sent into the Transformer.\"},{\"question\":\"What do the experiments show about backward rank reduction versus forward-decoder changes?\",\"answer\":\"Reducing backward rank increases validation loss, but changing the forward decoder via a low-rank factorized head increases loss substantially more; thus the forward restriction is more harmful.\"}]","Does the LM Head Create a Harmful Gradient Bottleneck? A Causal Test | PDF",43]