[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-140816-105":59,"doc-detail-140816-en":125},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":118,"head_meta":120,"extra_data":122,"updated_unix":124},105,"en","dancegrpo-unleashing-grpo-on-visual-generation","DanceGRPO - Unleashing GRPO on Visual Generation","","Recent advances in generative AI have transformed visual content creation, yet aligning model outputs with human preferences remains a major challenge. While reinforcement learning (RL) is widely used for preference alignment, prior approaches such as DDPO and DPOK struggle with unstable optimization when scaling to large and diverse prompt sets. This paper introduces DanceGRPO, an adaptation of Group Relative Policy Optimization (GRPO) for visual generation, delivering stable training across diffusion models and rectified flows. Experiments across complex real-world settings and multiple reward models show improvements up to 181% on established benchmarks.",{"@graph":69,"@context":117},[70,84,100],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/dancegrpo-unleashing-grpo-on-visual-generation/140816/",{"url":83,"name":65,"@type":85,"author":86,"headline":65,"publisher":89,"fileFormat":92,"inLanguage":63,"description":67,"dateModified":93,"datePublished":94,"encodingFormat":92,"isAccessibleForFree":95,"interactionStatistic":96},"DigitalDocument",{"name":87,"@type":88},"Ezra","Person",{"url":74,"name":90,"@type":91},"DocShare","Organization","application/pdf","2026-09-11","2026-08-25",true,{"@type":97,"interactionType":98,"userInteractionCount":24},"InteractionCounter",{"@type":99},"ViewAction",{"@type":101,"mainEntity":102},"FAQPage",[103,109,113],{"name":104,"@type":105,"acceptedAnswer":106},"What problem does DanceGRPO address in visual generation?","Question",{"text":107,"@type":108},"DanceGRPO targets the difficulty of aligning visual generation outputs with human preferences while keeping RL-based optimization stable when scaling to large and diverse prompt sets.","Answer",{"name":110,"@type":105,"acceptedAnswer":111},"How does the paper adapt GRPO for visual generation tasks?",{"text":112,"@type":108},"It reformulates sampling of diffusion models and rectified flows via stochastic differential equations (SDEs) and applies Group Relative Policy Optimization (GRPO) to stabilize training.",{"name":114,"@type":105,"acceptedAnswer":115},"What performance gains does DanceGRPO report on benchmarks?",{"text":116,"@type":108},"The paper reports that DanceGRPO outperforms baseline methods by up to 181% across multiple benchmarks including HPS-v2.1, CLIP Score, VideoAlign, and GenEval.","https://schema.org",{"og:url":83,"og:type":119,"og:title":65,"og:site_name":90,"og:description":67},"article",{"robots":121,"canonical":83},"index,follow",{"doc_id":123,"site_id":62},140816,1787641647,{"code":4,"msg":5,"data":126},{"doc_id":123,"user_id":127,"nickname":87,"user_avatar":128,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":129,"file_id":130,"file_url":131,"file_type":132,"file_size":133,"view_count":24,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":41,"language":134,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":135,"faqs":136,"seo_title":137,"seo_description":67,"update_tm":124,"read_time":138},1099514068035,"https://ap-avatar.wpscdn.com/davatar_276721f389ce27ea32af1340a28f341c","arXiv :2505 .07818v4 [ cs .CV] 28 Aug 2025  \nDanceGRPO: Unleashing GRPO on Visual Generation  \nZeyue Xue 1 ,2 , Jie Wu 1‡, Yu Gao 1 , Fangyuan Kong 1 , Lingting Zhu2 , Mengzhao Chen2 , Zhiheng Liu2 , Wei Liu 1 , Qiushan Guo 1 , Weilin Huang 1†, Ping Luo2†  \n1 ByteDance Seed, 2 The University of Hong Kong  \n†Corresponding authors, ‡Project lead  \nAbstract  \nRecent advances in generative AI have revolutionized visual content creation, yet aligning model outputs with human preferences remains a critical challenge. While Reinforcement Learning (RL) has emerged as a promising approach for fine-tuning generative models, existing methods like DDPO and DPOK face fundamental limitations-particularly their inability to maintain stable optimization when scaling to large and diverse prompt sets, severely restricting their practical utility. This paper presents DanceGRPO, a framework that addresses these limitations through an innovative adaptation of Group Relative Policy Optimization (GRPO) for visual generation tasks. Our key insight is that GRPO’s inherent stability mechanisms uniquely position it to overcome the optimization challenges that plague prior RL-based approaches on visual generation. DanceGRPO establishes several significant advances: First, it demonstrates consistent and stable policy optimization across multiple modern generative paradigms, including both diffusion models and rectified flows. Second, it maintains robust performance when scaling to complex, real-world scenarios encompassing three key tasks and four foundation models. Third, it shows remarkable versatility in optimizing for diverse human preferences as captured by five distinct reward models assessing image/video aesthetics, text-image alignment, video motion quality, and binary feedback. Our comprehensive experiments reveal that DanceGRPO outperforms baseline methods by up to 181% across multiple established benchmarks, including HPS-v2.1, CLIP Score, VideoAlign, and GenEval. Our results establish DanceGRPO as a robust and versatile solution for scaling Reinforcement Learning from Human Feedback (RLHF) tasks in visual generation, offering new insights into harmonizing reinforcement learning and visual synthesis.  \nDate: May 1, 2025  \nProject Page: [https://dancegrpo.github.io/](https://dancegrpo.github.io/)  \nCode: [https://github.com/XueZeyue/DanceGRPO](https://github.com/XueZeyue/DanceGRPO)  \n1 Introduction  \nRecent advances in generative models—particularly diffusion models [1–4] and rectified flows [5–7]—have transformed visual content creation by improving output quality and versatility in image and video generation. While pretraining establishes foundational data distributions, integrating human feedback during training proves critical for aligning outputs with human preferences and aesthetic criteria [8] . Existing methods face notable limitations: ReFL [9–11] relies on differentiable reward models, which introduce VRAM inefficiency in video generation and require several extensive engineering efforts, while DPO variants (Diffusion-DPO [12, 13], Flow-DPO [14], OnlineVPO [15]) achieve only marginal visual quality improvements. Reinforcement learning  \n(RL)-based methods [16 , 17], which optimize rewards as black-box objectives, offer potential solutions but introduce three unresolved challenges: (1) the Ordinary Differential Equations (ODEs)-based sampling of rectified flow models conflict with Markov Decision Process formulations; (2) prior policy gradient approaches (DDPO [18], DPOK [19]) show instability when scaling beyond small datasets (e.g., \u003C100 prompts); and (3) existing methods remain unvalidated for video generation tasks.  \nThis work addresses these gaps by reformulating the sampling of diffusion models and rectified flows via Stochastic Differential Equations (SDEs) and applying Group Relative Policy Optimization (GRPO) [20, 21] to stabilize the training process. In this paper, we pioneer the adaptation of GRPO to visual generation tasks","cbCaikTlM9g4rnvU","https://ap.wps.com/l/cbCaikTlM9g4rnvU","pdf",26736080,"English","# Introduction\n## Problem and limitations of existing RLHF methods\n## DanceGRPO approach and key insight\n## Evaluation scope and analysis plan\n## Contributions overview","[{\"question\":\"What problem does DanceGRPO address in visual generation?\",\"answer\":\"DanceGRPO targets the difficulty of aligning visual generation outputs with human preferences while keeping RL-based optimization stable when scaling to large and diverse prompt sets.\"},{\"question\":\"How does the paper adapt GRPO for visual generation tasks?\",\"answer\":\"It reformulates sampling of diffusion models and rectified flows via stochastic differential equations (SDEs) and applies Group Relative Policy Optimization (GRPO) to stabilize training.\"},{\"question\":\"What performance gains does DanceGRPO report on benchmarks?\",\"answer\":\"The paper reports that DanceGRPO outperforms baseline methods by up to 181% across multiple benchmarks including HPS-v2.1, CLIP Score, VideoAlign, and GenEval.\"}]","DanceGRPO - Unleashing GRPO on Visual Generation | PDF",76]