[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-327649-105":3,"detail-sidebar-cat-0-en-105":81,"doc-detail-327649-en":130},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":74,"head_meta":76,"extra_data":78,"updated_unix":80},105,"en","lace-svd-loss-aware-svd-with-cumulative-error-correction-for-llm-compression","LACE-SVD - Loss-Aware SVD with Cumulative Error Correction for LLM Compression","","Large language models (LLMs) face growing memory and compute demands, making efficient compression essential. The document presents LACE-SVD, a loss-aware SVD framework that addresses two gaps in prior SVD compression: heuristic rank-budget allocation ignoring layer-wise loss sensitivity, and accumulation of local reconstruction errors through the residual stream. LACE-SVD estimates calibration negative-log-likelihood increases to allocate per-layer ranks under a global budget, then applies closed-form local updates and propagation-aware correction with an acceptance gate, improving layer-output fidelity and global behavior during compression.",{"@graph":14,"@context":73},[15,34,56],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/research-report/","Research & Report",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/lace-svd-loss-aware-svd-with-cumulative-error-correction-for-llm-compression/327649/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/lace-svd-loss-aware-svd-with-cumulative-error-correction-for-llm-compression/327649.png","ImageObject",300,407,{"name":42,"@type":43},"Noah","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-11-07","2026-09-21",true,{"@type":52,"interactionType":53,"userInteractionCount":55},"InteractionCounter",{"@type":54},"ViewAction",19,{"@type":57,"mainEntity":58},"FAQPage",[59,65,69],{"name":60,"@type":61,"acceptedAnswer":62},"What problem does LACE-SVD aim to solve in SVD-based LLM compression?","Question",{"text":63,"@type":64},"It targets two issues: rank budgets chosen without considering layer-wise loss sensitivity, and local reconstruction errors that accumulate through the residual stream and cause larger global deviation.","Answer",{"name":66,"@type":61,"acceptedAnswer":67},"How does LACE-SVD allocate rank budgets across layers?",{"text":68,"@type":64},"It estimates how candidate layer-wise compression ratios change calibration negative-log-likelihood, then solves a budget-constrained allocation to choose one ratio per layer under a global parameter budget.",{"name":70,"@type":61,"acceptedAnswer":71},"What is cumulative error correction in this method?",{"text":72,"@type":64},"After SVD decomposition, LACE-SVD refines low-rank factors with closed-form local updates and applies propagation-aware correction for modules writing to the residual stream, using an acceptance gate to keep improvements on calibration data.","https://schema.org",{"og:url":32,"og:type":75,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":77,"canonical":32},"index,follow",{"doc_id":79,"site_id":7},327649,1790116734,{"code":4,"msg":82,"data":83},"success",[84,88,92,96,101,106,111,115,120,123,127],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":85,"show_sort_weight":86,"slug":87},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":89,"show_sort_weight":90,"slug":91},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":93,"show_sort_weight":94,"slug":95},"Exam",70,"exam",{"id":97,"doc_module":4,"doc_module_name":25,"category_name":98,"show_sort_weight":99,"slug":100},5,"Comic",60,"comic",{"id":102,"doc_module":4,"doc_module_name":25,"category_name":103,"show_sort_weight":104,"slug":105},6,"Technology",50,"technology",{"id":107,"doc_module":4,"doc_module_name":25,"category_name":108,"show_sort_weight":109,"slug":110},7,"Healthcare",40,"healthcare",{"id":112,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":113,"slug":114},8,30,"research-report",{"id":116,"doc_module":4,"doc_module_name":25,"category_name":117,"show_sort_weight":118,"slug":119},9,"Religion & Spirituality",20,"religion-spirituality",{"id":118,"doc_module":4,"doc_module_name":25,"category_name":121,"show_sort_weight":118,"slug":122},"World Cup","world-cup",{"id":124,"doc_module":4,"doc_module_name":25,"category_name":125,"show_sort_weight":124,"slug":126},10,"Lifestyle","lifestyle",{"id":55,"doc_module":4,"doc_module_name":25,"category_name":128,"show_sort_weight":97,"slug":129},"General","general",{"code":4,"msg":82,"data":131},{"doc_id":79,"user_id":132,"nickname":42,"user_avatar":133,"doc_module":4,"category_id":112,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":55,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":139,"language":140,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":12,"update_tm":144,"read_time":113,"preview_page_count":4},8796095462418,"https://ap-avatar.wpscdn.com/avatar/80000253c1241d02b47?x-image-process=image/resize,m_fixed,w_180,h_180&k=1778826106357471780","LACE-SVD: Loss-Aware SVD with Cumulative Error Correction for LLM  \nCompression  \nZhuowen Liu1 Longkun Hao2 Shiyu Feng3 Xiaowen Chang4 Ruiqun Li5 Changqun Li6†  \narXiv :2607 .03057v 1 [ cs .LG] 3 Jul 2026  \nAbstract  \nThe rapid growth in the parameter scale of large language models (LLMs) has created a strong demand for efficient compression techniques. As a hardware-agnostic and highly compatible approach, low-rank compression has been widely adopted to reduce both memory footprint and computational cost. However, existing SVD-based methods are still largely driven by local reconstruction objectives, overlooking two critical limitations: rank budgets are often allocated without explicitly considering layer-wise loss sensitivity, and local approximation errors can propagate and accumulate through the residual stream, leading to amplified global deviations from the original model. To address these issues, we propose LACESVD, a Loss-Aware SVD framework with Cumulative Error correction for LLM compression. LACE-SVD first estimates the calibration negative-log-likelihood increase induced by candidate layer-wise compression ratios and solves a budget-constrained allocation problem to assign rank budgets. It then refines the compressed model with closed-form local updatesand introduces a propagation-aware correction for residual-stream output modules, reducing layer-output discrepancy as a proxy for cumulative error propagation. Experimental results demonstrate that at a high compression ratio (0.6), the WikiText-2 PPL of our method on LLaMA-7B (32.57) is significantly better than that of Dobi-SVD (46 . 18) .  \n1 Introduction  \nLarge language models (LLMs), including GPT Family (Achiam et al., 2023 ; Dettmers et al., 2022), LLaMA Family (Touvron et al., 2023a ; Grattafiori et al., 2024), and Qwen Family (Bai et al., 2023 ; Yang et al., 2025) have achieved remarkable performance across commonsense reasoning, question answering, document understanding, and few-shot learning. However, their strong capabilities come with substantial memory and computational costs,  \nmaking billion-parameter models difficult to deploy under practical latency, memory, and energy constraints. To address this challenge, post-training compression has become an important research direction, including quantization (Frantar et al., 2022 ; Hu et al., 2025), pruning (Ashkboos et al., 2024), knowledge distillation (Hinton et al., 2015), and low-rank decomposition (Golub et al., 1987 ; Yuan et al., 2023) . Among these approaches, singular value decomposition (SVD) based compression is particularly attractive because it directly factorizes dense weight matrices into low-rank components, reducing both model size and matrix multiplication cost without requiring large-scale retraining.  \nRecent SVD-based compression methods (Yuan et al., 2023 ; Li et al., 2025 ; Wang et al.) improve over vanilla truncated SVD by using activation statistics or calibration data. These methods rescale or whiten weight matrices before decomposition so that the low-rank approximation better matches the activation distribution of each layer. Although effective, they are still mainly driven by local reconstruction objectives. That is, they aim to preserve the output of each compressed module or layer under calibration activations, while the global language modeling behavior is only indirectly considered.  \nThis local view leads to two limitations. First, different layers exhibit different sensitivity to compression. A uniform rank ratio may over-compress loss-critical layers and under-compress robust layers, resulting in a suboptimal use of the global parameter budget. Second, reconstruction errors are not isolated across layers. Once an early layer is compressed, its output perturbation enters the residual stream and changes the input distribution of subsequent layers. These perturbations can accumulate through attention and MLP blocks, causing the compressed model to deviate from the origin","cbCais21W3CFUUwv","https://ap.wps.com/l/cbCais21W3CFUUwv","pdf",1523251,12,"English","# Abstract\n# Introduction","[{\"question\":\"What problem does LACE-SVD aim to solve in SVD-based LLM compression?\",\"answer\":\"It targets two issues: rank budgets chosen without considering layer-wise loss sensitivity, and local reconstruction errors that accumulate through the residual stream and cause larger global deviation.\"},{\"question\":\"How does LACE-SVD allocate rank budgets across layers?\",\"answer\":\"It estimates how candidate layer-wise compression ratios change calibration negative-log-likelihood, then solves a budget-constrained allocation to choose one ratio per layer under a global parameter budget.\"},{\"question\":\"What is cumulative error correction in this method?\",\"answer\":\"After SVD decomposition, LACE-SVD refines low-rank factors with closed-form local updates and applies propagation-aware correction for modules writing to the residual stream, using an acceptance gate to keep improvements on calibration data.\"}]","LACE-SVD - Loss-Aware SVD with Cumulative Error Correction for LLM Compression | PDF",1789977730]