[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-346404-105":59,"doc-detail-346404-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","adaptive-subgradient-methods-for-online-learning-and-stochastic-optimization","Adaptive Subgradient Methods for Online Learning and Stochastic Optimization","","A new family of subgradient methods is introduced for online learning and stochastic convex optimization, using information from the geometry of previously observed data to guide gradient-based updates. The approach adaptively modifies proximal functions, simplifying learning-rate selection while providing regret guarantees comparable to the best proximal function chosen in hindsight. Efficient algorithms are developed for empirical risk minimization with common regularizers and constraints, and experiments show adaptive methods outperform non-adaptive baselines.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/adaptive-subgradient-methods-for-online-learning-and-stochastic-optimization/346404/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/adaptive-subgradient-methods-for-online-learning-and-stochastic-optimization/346404.png","ImageObject",300,407,{"name":92,"@type":93},"Eliana","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-16","2026-09-22",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",11,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"How do the proposed adaptive subgradient methods differ from standard subgradient algorithms?","Question",{"text":113,"@type":114},"They dynamically incorporate geometric information from earlier iterations to make gradient updates more informative, rather than following a fixed, data-oblivious procedure.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"What is the role of proximal functions in the framework?",{"text":118,"@type":114},"The methods adaptively modify the proximal function that controls gradient steps, which simplifies choosing a learning rate and yields regret guarantees.",{"name":120,"@type":111,"acceptedAnswer":121},"What improvements are shown in experiments?",{"text":122,"@type":114},"The adaptive subgradient methods outperform state-of-the-art non-adaptive subgradient algorithms on empirical studies tied to the theoretical analysis.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},346404,1791291957,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":140,"language":141,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":67,"update_tm":145,"read_time":146,"preview_page_count":19,"preview_urls":147},4398048949847,"https://ap-avatar.wpscdn.com/avatar/400002536579ef2da7f?_k=1778318612642679267","Adaptive Subgradient Methods for  \nOnline Learning and Stochastic Optimization∗  \nJohn Duchi JDUCHI @ CS . BERKELEY. EDU  \nComputer Science Division University of California, Berkeley Berkeley, CA 94720 USA  \nElad Hazan EHAZAN @IE . TECHNION . AC . IL  \nTechnion-Israel Institute of Technology  \nTechnion City  \nHaifa, 32000, Israel  \nYoram Singer SINGER @ GOOGLE . COM  \nGoogle  \n1600 Amphitheatre Parkway Mountain View, CA 94043 USA  \nEditor: Tong Zhang  \nAbstract  \nWe present a new family of subgradient methods that dynamically incorporate knowledge of the geometry of the data observed in earlier iterations to perform more informative gradient-based learning. Metaphorically, the adaptation allows us to ﬁnd needles in haystacks in the form of very predictive but rarely seen features. Our paradigm stems from recent advances in stochastic optimization and online learning which employ proximal functions to control the gradient steps of the algorithm. We describe and analyze an apparatus for adaptively modifying the proximal function, which signiﬁcantly simpliﬁes setting a learning rate and results in regret guarantees that are provably as good as the best proximal function that can be chosen in hindsight. We give several efﬁcient algorithms for empirical risk minimization problems with common and important regularization functions and domain constraints. We experimentally study our theoretical analysis and show that adaptive subgradient methods outperform state-of-the-art, yet non-adaptive, subgradient algorithms.  \nKeywords: subgradient methods, adaptivity, online learning, stochastic convex optimization  \n1. Introduction  \nIn many applications of online and stochastic learning, the input instances are of very high dimension, yet within any particular instance only a few features are non-zero. It is often the case, however, that infrequently occurring features are highly informative and discriminative. The informativeness of rare features has led practitioners to craft domain-speciﬁc feature weightings, such as TF-IDF (Salton and Buckley, 1988), which pre-emphasize infrequently occurring features. We use this old idea as a motivation for applying modern learning-theoretic techniques to the problem of online and stochastic learning, focusing concretely on (sub)gradient methods.  \n∗ . A preliminary version of this work was published in COLT 2010 .  \n􀀍c2011 John Duchi, Elad Hazan and Yoram Singer.  \nDUCHI, HAZAN AND SINGER  \nStandard stochastic subgradient methods largely follow a predetermined procedural scheme that is oblivious to the characteristics of the data being observed. In contrast, our algorithms dynamically incorporate knowledge of the geometry of the data observed in earlier iterations to perform more informative gradient-based learning. Informally, our procedures give frequently occurring features very low learning rates and infrequent features high learning rates, where the intuition is that each time an infrequent feature is seen, the learner should “take notice.” Thus, the adaptation facilitates ﬁnding and identifying very predictive but comparatively rare features.  \n1.1 The Adaptive Gradient Algorithm  \nBefore introducing our adaptive gradient algorithm, which we term ADAGRAD, we establish notation. Vectors and scalars are lower case italic letters, such as x ∈ X. We denote a sequence of vectors by subscripts, that is, xt, xt+1 , . . ., and entries of each vector by an additional subscript, for example, xt ,j . The subdifferential set of a function f evaluated at x is denoted ∂f (x), and a particular vector in the subdifferential set is denoted by f ′(x) ∈ ∂f (x) or gt ∈ ∂ft (xt) . When a function is differentiable, we write ∇f (x) . We use hx,yi to denote the inner product between x and y. The Bregman divergence associated with a strongly convex and differentiable function ψ is  \nBψ (x,y) = ψ(x) −ψ(y) −h∇ψ(y), x −yi .  \nWe also make frequent use of the following two matrices. Let g 1:t = [g 1 ··· gt] denote the ma","cbCaioeOvogLGu5b","https://ap.wps.com/l/cbCaioeOvogLGu5b","pdf",307882,39,"English","# Introduction\n## The Adaptive Gradient Algorithm","[{\"question\":\"How do the proposed adaptive subgradient methods differ from standard subgradient algorithms?\",\"answer\":\"They dynamically incorporate geometric information from earlier iterations to make gradient updates more informative, rather than following a fixed, data-oblivious procedure.\"},{\"question\":\"What is the role of proximal functions in the framework?\",\"answer\":\"The methods adaptively modify the proximal function that controls gradient steps, which simplifies choosing a learning rate and yields regret guarantees.\"},{\"question\":\"What improvements are shown in experiments?\",\"answer\":\"The adaptive subgradient methods outperform state-of-the-art non-adaptive subgradient algorithms on empirical studies tied to the theoretical analysis.\"}]","Adaptive Subgradient Methods for Online Learning and Stochastic Optimization | PDF",1790061306,98,[148,152,156,160],{"page_no":8,"url":149,"width":150,"height":151},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-16/455facfc5e53fe967662d7587054826f.webp",983,1272,{"page_no":14,"url":153,"width":154,"height":155},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-16/9e4996835ac6bc92fee1fb0b585d793a.webp",982,1270,{"page_no":81,"url":157,"width":158,"height":159},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-16/4e0e08445bd5c6cf390add368c9e8e94.webp",981,1269,{"page_no":19,"url":161,"width":162,"height":163},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-16/a2d949243efc5df95b03023141ab210c.webp",980,1268]