[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-290229-105":59,"doc-detail-290229-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","extrapolating-from-regularised-solutions-for-solving-ill-conditioned-linear-systems-in-machine-learning","Extrapolating from Regularised Solutions for Solving Ill-Conditioned Linear Systems in Machine Learning","","Rapid prototyping of machine learning algorithms often depends on lightweight numerical linear algebra. For ill-conditioned linear systems, Tikhonov regularised inversion with a nugget is a standard prototype approach, yet nugget selection is difficult and data-adaptive strategies can block automatic differentiation while causing training instabilities. The method autonugget addresses these issues by combining multiple linear solves with Richardson extrapolation to improve numerical stability and accuracy, and it is fully compatible with JAX for end-to-end automatic differentiation.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/extrapolating-from-regularised-solutions-for-solving-ill-conditioned-linear-systems-in-machine-learning/290229/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/extrapolating-from-regularised-solutions-for-solving-ill-conditioned-linear-systems-in-machine-learning/290229.png","ImageObject",300,407,{"name":92,"@type":93},"Finn","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-05","2026-09-17",true,{"@type":102,"interactionType":103,"userInteractionCount":39},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What is the nugget in Tikhonov regularisation and why is it important?","Question",{"text":112,"@type":113},"The nugget is the regularisation parameter σ in (A + σI)^−1 b. It improves numerical conditioning, but the critical minimum value for stable computation is generally unknown and depends on A and b.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"Why can data-adaptive nugget selection hinder end-to-end training?",{"text":117,"@type":113},"Because σ is often chosen using procedures depending on quantities that are not differentiable with respect to model parameters, automatic differentiation with respect to θ in Aθ and bθ is precluded.",{"name":119,"@type":110,"acceptedAnswer":120},"How does autonugget improve accuracy compared with using a single nugget?",{"text":121,"@type":113},"It performs multiple linear solves for a set of nuggets and combines the results using Richardson extrapolation, reducing error compared with approximations derived from only one nugget.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},290229,1789641020,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":39,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":139,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":129,"read_time":144,"preview_page_count":4},34359740700684,"https://ap-avatar.wpscdn.com/avatar/1f400023980c374ae676?_k=1777273430885731487","Extrapolating from Regularised Solutions for Solving IllConditioned Linear Systems in Machine Learning  \nDisha Hegde [d.hegde@soton. ac.uk](d.hegde@soton. ac.uk)  \nUniversity of Southampton, UK  \nJon Cockayne [jon. cockayne@soton. ac.uk](jon. cockayne@soton. ac.uk)  \nUniversity of Southampton, UK  \nChris. J. Oates [chris. oates@newcastle. ac.uk](chris. oates@newcastle. ac.uk)  \nNewcastle University, UK The Alan Turing Institute, UK  \nReviewed on OpenReview: [https: // openreview. net/ forum? id= fqbkenUpRa](https: // openreview. net/ forum? id= fqbkenUpRa)  \nAbstract  \nRapid prototyping of algorithms is a critical step in modern machine learning. Most algorithms exploit linear algebra, creating a need for lightweight numerical routines which – while potentially sub-optimal for the task at hand – can be rapidly implemented. For the numerical solution of ill-conditioned linear systems of equations, the standard solution for prototyping is Tikhonov-regularised inversion using a nugget. However, selection of the size of nugget is often difficult, and the use of data-adaptive procedures precludes automatic differentiation, introducing instabilities into end-to-end training. Further, while data-adaptive procedures perform multiple linear solves to select the size of nugget, only the result of one such solve is returned, which we argue is wasteful. This paper aims to circumvent the above difficulties, presenting autonugget; a Python package for automatic and stable numerical solution of linear systems suitable for rapid prototyping, and fully compatible with automatic differentiation using JAX. autonugget combines multiple linear solves using Richardson extrapolation to determine the solution of the ill-conditioned system, improving in accuracy over approximations based on a single nugget.  \n1 Introduction  \nConsider the problem of numerically solving Ax = b for x ∈ Rd. It will be assumed that A is a symmetric positive definite matrix, but that A is (possibly severely) ill-conditioned and, as a result, solving the linear system with standard methods is challenging. This setting is ubiquitous in applied machine learning like Bayesian optimisation (Garnett, 2023), emulation (Gramacy, 2020), and probabilistic numerics (Hennig et al., 2022) .  \nIn machine learning, a widely-used approach to improving the conditioning of the system so that it can be solved is to use Tikhonov regularisation:  \nxσ := (A + σI)−1b, σ ≥ 0 , (1)  \nwhere xσ → x in exact arithmetic as the regularisation parameter σ → 0. The regularisation parameter σ is referred to as a nugget. For sufficiently regular matrices A, such as positive semi-definite matrices, the conditioning of A + σI improves monotonically as σ is increased. However, in practice this often results ina conflation of two ideals:  \n1. Regularisation: as a technique for penalising large values in the solution vector, improving generalisation performance. In this setting the focus is on σ > 0.  \n2. Ill-conditioning: as a technique for improving the numerical performance of solver routines. In this setting the quantity of interest is σ = 0, but the ill-conditioning of A causes numerical problems in the computational pipeline.  \nIn this paper, the focus is solely on the latter: we assume that the nugget is being used only as a tool to improve the conditioning of A.  \nWhen using Tikhonov regularisation to improve conditioning, one usually takes σ to be just large enough that a numerical approximation ˆxσ to xσ can be obtained to a required level of precision using a direct method (e.g. LU-factorisation) . The popularity of Tikhonov-regularisation for prototyping of algorithms is due to the simplicity with which it can be implemented, and the presence of only a single degree of freedom tobe selected. For subsequent code optimisation, one would ideally upgrade from a Tikhonov-regularised direct method to a more scalable or sophisticated linear algebra routine, such as a preconditioned iterative method (Trefet","cbCaicPRaZUtWkCO","https://ap.wps.com/l/cbCaicPRaZUtWkCO","pdf",1963730,26,"English","# Introduction\n## Linear systems and ill-conditioning\n## Tikhonov regularisation and the nugget\n## Three key issues: nugget size, end-to-end training, data-efficiency\n## Proposed solution: autonugget","[{\"question\":\"What is the nugget in Tikhonov regularisation and why is it important?\",\"answer\":\"The nugget is the regularisation parameter σ in (A + σI)^−1 b. It improves numerical conditioning, but the critical minimum value for stable computation is generally unknown and depends on A and b.\"},{\"question\":\"Why can data-adaptive nugget selection hinder end-to-end training?\",\"answer\":\"Because σ is often chosen using procedures depending on quantities that are not differentiable with respect to model parameters, automatic differentiation with respect to θ in Aθ and bθ is precluded.\"},{\"question\":\"How does autonugget improve accuracy compared with using a single nugget?\",\"answer\":\"It performs multiple linear solves for a set of nuggets and combines the results using Richardson extrapolation, reducing error compared with approximations derived from only one nugget.\"}]","Extrapolating from Regularised Solutions for Solving Ill-Conditioned Linear Systems in Machine Learning | PDF",66]