[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-513156-105":59,"doc-detail-513156-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","equilibrium-bandits-learning-optimal-equilibria-of-unknown-dynamics","Equilibrium Bandits - Learning Optimal Equilibria of Unknown Dynamics","","Consider a decision-maker that repeatedly selects one action to control an unknown dynamical system for a finite horizon. Fixing an action drives the system toward a unique equilibrium that depends on the action, while the decision-maker only observes noisy end-of-turn rewards. Learning which equilibria yield higher rewards is valuable, but waiting for convergence is costly, creating a distinct regret structure. The proposed Upper Equilibrium Concentration Bound (UECB) switches actions quickly using convergence bounds, achieving logarithmic-type regret with dependence on worst-case approximate convergence time and extending to epidemic and game control.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/equilibrium-bandits-learning-optimal-equilibria-of-unknown-dynamics/513156/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/equilibrium-bandits-learning-optimal-equilibria-of-unknown-dynamics/513156.png","ImageObject",300,407,{"name":92,"@type":93},"Oliver","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-20","2026-10-06",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",12,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"What is the equilibrium bandit problem studied in the document?","Question",{"text":113,"@type":114},"The document studies learning in which an agent selects actions that determine the equilibrium the unknown system converges to, but receives noisy rewards over time while convergence may be slow and costly.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"Why do standard bandit algorithms incur linear regret here?",{"text":118,"@type":114},"Because the reward process is not i.i.d.; expected reward evolves as the system approaches equilibrium, creating memory and preventing the usual stochastic independence assumptions.",{"name":120,"@type":111,"acceptedAnswer":121},"How does UECB reduce the cost of waiting for convergence?",{"text":122,"@type":114},"UECB plays actions in consecutive epochs and uses convergence bounds to estimate how much reward can be gained by waiting until equilibrium, switching away quickly when waiting is not worthwhile.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},513156,1791318465,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":44,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":144,"read_time":145,"preview_page_count":19,"preview_urls":146},8796095461610,"https://ap-avatar.wpscdn.com/davatar_276721f389ce27ea32af1340a28f341c","Equilibrium Bandits: Learning Optimal Equilibria of Unknown  \nDynamics  \nSiddharth Chandak  \nStanford University, Department of Electrical Engineering USA[chandaks@stanford.edu](chandaks@stanford.edu)  \nIlai Bistritz  \nStanford University, Department of Electrical Engineering USA [bistritz@stanford.edu](bistritz@stanford.edu)  \nNicholas Bambos  \nStanford University, Department of Electrical Engineering USA [bambos@stanford.edu](bambos@stanford.edu)  \nABSTRACT  \nConsider a decision-maker that can pick one out of 􀀠 actions to control an unknown system, for 􀀩 turns. The actions are interpreted as different configurations or policies. Holding the same action fixed, the system asymptotically converges to a unique equilibrium, as a function of this action. The dynamics ofthe system are unknown to the decision-maker, which can only observe a noisy reward atthe end of every turn. The decision-maker wants to maximize its accumulated reward over the 􀀩 turns. Learning what equilibriaare better results in higher rewards, but waiting for the system to converge to equilibrium costs valuable time. Existing bandit algorithms, either stochastic or adversarial, achieve linear (trivial) regret for this problem. We present a novel algorithm, termed Upper Equilibrium Concentration Bound (UECB), that knows to switch an action quickly if it is not worth it to wait until the equilibrium is reached. This is enabled by employing ‘convergence bounds’ to determine how far the system is from equilibrium. We prove that UECB achieves a regret of O(log(􀀩 ) + 􀁧􀀲 log(􀁧􀀲) + 􀁧􀀲 log log(􀀩 )) for this “equilibrium bandit problem” where 􀁧􀀲 is the worst case approximate convergence time to equilibrium. We then show that both epidemic control and game control are special cases of equilibrium bandits, where 􀁧􀀲 log 􀁧􀀲 typically dominates the regret. We then test UECB numerically for both of these applications.  \nKEYWORDS  \nonline learning; multiagent systems; game theory  \nACM Reference Format:  \nSiddharth Chandak, Ilai Bistritz, and Nicholas Bambos. 2023. Equilibrium Bandits: Learning Optimal Equilibria of Unknown Dynamics. In Proc. of the 22nd International Conference on Autonomous Agents and Multiagent Systems (AAMAS 2023), London, United Kingdom, May 29 – June 2, 2023, IFAAMAS, 9 pages.  \n1 INTRODUCTION  \nMany large-scale complex systems reach an equilibrium over time. Examples include epidemics, transportation, markets, and supply chains. With no planning, the global performance at this equilibrium can be poor. When a decision-maker can control some parameters of such a system, it can influence the equilibrium that the system converges to. Examples are changing the frequencies of subway lines, or the masking and isolation policies during an epidemic. However, a model for the dynamics of these large-scale  \nProc. of the 22nd International Conference on Autonomous Agents and Multiagent Systems (AAMAS 2023), A. Ricci, W. Yeoh, N. Agmon, B. An (eds.), May 29 – June 2, 2023, London, United Kingdom. © 2023 International Foundation for Autonomous Agents and Multiagent Systems ([www.ifaamas.org](www.ifaamas.org)). All rights reserved.  \ncomplex systems is rarely available. Instead, the decision-maker can only observe the impact of its decisions in real time. It is infeasible to allow the system to converge to equilibrium under each policy since this will waste significant time on suboptimal policies. This introduces a learning problem of controlling such systems with‘bandit feedback’ [18] .  \nMotivated by this, we consider an agent that takes an action from a discrete set of actions at each timestep. There is an underlying system that evolves with time depending on the agent’s action. At each timestep, the agent receives a noisy reward as a function of its action and the ‘state’ of the system. The key aspect of the underlying system is that if we fix the action, it would asymptotically converge to a unique equilibrium as a function of this action. However, waiting too lon","cbCaisWkQnh27UsW","https://ap.wps.com/l/cbCaisWkQnh27UsW","pdf",1206057,"English","# Abstract\n# Keywords\n# Introduction\n## Equilibrium control and bandit feedback\n## Equilibrium bandits and regret\n## Challenges from non-i.i.d. reward and equilibrium noise\n## UECB approach","[{\"question\":\"What is the equilibrium bandit problem studied in the document?\",\"answer\":\"The document studies learning in which an agent selects actions that determine the equilibrium the unknown system converges to, but receives noisy rewards over time while convergence may be slow and costly.\"},{\"question\":\"Why do standard bandit algorithms incur linear regret here?\",\"answer\":\"Because the reward process is not i.i.d.; expected reward evolves as the system approaches equilibrium, creating memory and preventing the usual stochastic independence assumptions.\"},{\"question\":\"How does UECB reduce the cost of waiting for convergence?\",\"answer\":\"UECB plays actions in consecutive epochs and uses convergence bounds to estimate how much reward can be gained by waiting until equilibrium, switching away quickly when waiting is not worthwhile.\"}]","Equilibrium Bandits - Learning Optimal Equilibria of Unknown Dynamics | PDF",1791281061,23,[147,151,155,159],{"page_no":8,"url":148,"width":149,"height":150},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/92cf2e5725f01492e6993cf992fcbb6e.webp",983,1272,{"page_no":14,"url":152,"width":153,"height":154},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/ef59b34d332793fcea960c5ee931965e.webp",982,1270,{"page_no":81,"url":156,"width":157,"height":158},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/220d2ad7adaae8b28a0897bf16de4cc3.webp",981,1269,{"page_no":19,"url":160,"width":161,"height":162},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/b7f824123b8b10d30694cff221813995.webp",980,1268]