[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-327237-105":59,"doc-detail-327237-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","muon-as-a-residual-connection","MUON AS A RESIDUAL CONNECTION","","Muon has recently emerged as one of the most effective optimizers for training large neural networks, yet its empirical success has been explained through multiple, sometimes competing, viewpoints. This paper proposes a simple mechanistic interpretation: Muon behaves as an implicit residual connection during training. By orthogonalizing the update, Muon may trade some immediate gradient fidelity for improved representation preservation for downstream layers. The authors study this trade-off in controlled linear optimization settings and derive design guidance for optimizers balancing local descent with downstream usability.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/muon-as-a-residual-connection/327237/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/muon-as-a-residual-connection/327237.png","ImageObject",300,407,{"name":92,"@type":93},"Finn","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-02","2026-09-21",true,{"@type":102,"interactionType":103,"userInteractionCount":39},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What main interpretation does the paper propose for Muon?","Question",{"text":112,"@type":113},"Muon is interpreted as an implicit residual connection during training, linking optimizer behavior to the reasons residual connections work well.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"How does orthogonalizing the update affect optimization?",{"text":117,"@type":113},"Orthogonalizing the update can reduce immediate gradient fidelity while improving representation preservation for downstream layers.",{"name":119,"@type":110,"acceptedAnswer":120},"What do the controlled linear optimization experiments study?",{"text":121,"@type":113},"They examine the trade-off where Muon can learn representations that fit a local target more slowly, yet are easier for downstream layers to exploit.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},327237,1790615803,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":39,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":139,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":144,"read_time":145,"preview_page_count":4},34359740700684,"https://ap-avatar.wpscdn.com/avatar/1f400023980c374ae676?_k=1777273430885731487","arXiv :2607 .0 1 124v 1 [ cs .LG] 1 Jul 2026  \nMUON AS A RESIDUAL CONNECTION  \nHao Huang  \nCollege of Computer Science and Technology Zhejiang University  \nHangzhou, Zhejiang, China [goodhao@zju.edu.cn](goodhao@zju.edu.cn)  \nABSTRACT  \nMuon has recently emerged as one of the most effective optimizers for training large neural networks, yet its empirical success has been explained from several different perspectives. In this paper, we propose a simple mechanistic interpretation: Muon can be understood as an implicit residual connection during training.  \nSpecifically, orthogonalizing the update can sacrifice some immediate gradient fidelity while improving representation preservation for downstream layers. We study this trade-off in controlled linear optimization settings, where Muon can learn representations that are slower to fit a local target but easier for downstream layers to exploit. Our results suggest a conceptual explanation for Muon and a design perspective for optimizers that balance local descent with downstream usability.1  \n1 INTRODUCTION  \nIn recent years, Muon has attracted considerable attention as a highly effective optimizer for deep neural networks, delivering state-of-the-art performance on a wide range of architectures, from CNNs to LLMs.  \nMuon is built upon a simple idea—orthogonalizing the updates of matrix parameters. Yet, despite its simplicity, its empirical success has inspired a surprisingly diverse collection of sophisticated explanations, including spectral steepest descent, sharpness, curvature, and long-tail learning.  \nA natural question arises before introducing yet another explanation of Muon: why does Muon need a new explanation at all? Rather than viewing these explanations as mutually exclusive, we begin from a broader perspective on scientific explanation. As emphasized in the philosophy of science, a scientific phenomenon often admits multiple valid explanations, each highlighting different causal mechanisms, levels of abstraction, or explanatory purposes.  \nDifferent explanations are valuable not only because they improve our understanding of a phenomenon, but also because they inspire different directions for future research. For example, interpreting Muon through the lens of spectral-norm steepest descent naturally motivates the exploration of optimizers based on alternative matrix norms. In contrast, explaining Muon’s success through curvature suggests incorporating richer second-order information into optimizer design. From this perspective, a new explanation is valuable not merely as another interpretation, but as a new source of hypotheses and algorithmic design principles.  \nOur goal is therefore not to replace existing explanations, but to introduce a new explanatory perspective that complements them. Specifically, we argue that Muon can be understood as an implicit residual connection, providing a simple and intuitive framework for reasoning about how optimization can affect downstream representation usability.  \n1Code is available at [https://github.com/huanghao-sss/muon_interpretation](https://github.com/huanghao-sss/muon_interpretation).  \n2 BACKGROUND  \n2. 1 THE MUON OPTIMIZER  \nMuon is a recently proposed optimizer that has attracted considerable attention due to its remarkable empirical success, from benchmarks such as NanoGPT and CIFAR-10 to frontier large language models including Kimi K2 and DeepSeek V4 .  \nAt its core, Muon replaces the standard parameter update W ← W + ∆W with an orthogonalized update W ← W + Orth(∆W), where Orth(∆W) denotes the orthogonal matrix closest to ∆W.  \n2. 2 EXISTING INTERPRETATIONS OF MUON  \nExisting interpretations of Muon emphasize several complementary aspects of its behavior. The spectral-norm steepest-descent view interprets the orthogonalized update as the steepest descent direction under the matrix spectral norm (Bernstein & Newhouse, 2024), while the trust-region view derives the Muon step as the exact solution of a non-Euclidean","cbCaiaZORzsdJzCm","https://ap.wps.com/l/cbCaiaZORzsdJzCm","pdf",947331,15,"English","# Abstract\n# Introduction\n# Background\n## The Muon Optimizer\n## Existing Interpretations of Muon\n## Residual Connections in Deep Learning","[{\"question\":\"What main interpretation does the paper propose for Muon?\",\"answer\":\"Muon is interpreted as an implicit residual connection during training, linking optimizer behavior to the reasons residual connections work well.\"},{\"question\":\"How does orthogonalizing the update affect optimization?\",\"answer\":\"Orthogonalizing the update can reduce immediate gradient fidelity while improving representation preservation for downstream layers.\"},{\"question\":\"What do the controlled linear optimization experiments study?\",\"answer\":\"They examine the trade-off where Muon can learn representations that fit a local target more slowly, yet are easier for downstream layers to exploit.\"}]","MUON AS A RESIDUAL CONNECTION | PDF",1789976779,38]