[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-290241-105":59,"doc-detail-290241-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","adaptivestep-automatically-dividing-reasoning-step-through-model-confidence","AdaptiveStep - Automatically Dividing Reasoning Step through Model Confidence","","AdaptiveStep addresses shortcomings in training Process Reward Models by replacing rule-based reasoning step decomposition with an automatic division driven by the model’s confidence in predicting the next word. This design highlights true decision points, improving information quality for reward-model training and downstream tasks. The method requires no manual annotation and is validated on mathematical reasoning and code generation, where AdaptiveStep-trained PRMs achieve state-of-the-art Best-of-N performance, exceed greedy token-level value-guided decoding, and reduce construction cost by over 30% versus open-source baselines, supported by analysis and code release.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/adaptivestep-automatically-dividing-reasoning-step-through-model-confidence/290241/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/adaptivestep-automatically-dividing-reasoning-step-through-model-confidence/290241.png","ImageObject",300,407,{"name":92,"@type":93},"Finn","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-25","2026-09-17",true,{"@type":102,"interactionType":103,"userInteractionCount":39},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What problem does AdaptiveStep solve in training Process Reward Models?","Question",{"text":112,"@type":113},"It addresses the limitation of rule-based decomposition that creates coarse reasoning steps with low decision-making information and difficulties applying them when step boundaries are hard to define.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"How does AdaptiveStep decide where to split reasoning steps?",{"text":117,"@type":113},"It divides reasoning steps based on the model’s confidence in predicting the next word, using confidence to identify critical breaking points automatically.",{"name":119,"@type":110,"acceptedAnswer":120},"What experimental results does AdaptiveStep achieve?",{"text":121,"@type":113},"AdaptiveStep-trained PRMs reach state-of-the-art Best-of-N performance in mathematical reasoning and code generation, outperforming greedy search with token-level value-guided decoding, and reduce construction costs by over 30% compared with existing open-source PRMs.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},290241,1789641069,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":39,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":139,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":129,"read_time":36,"preview_page_count":4},34359740700684,"https://ap-avatar.wpscdn.com/avatar/1f400023980c374ae676?_k=1777273430885731487","AdaptiveStep: Automatically Dividing Reasoning Step through Model  \nConfidence  \nYuliang Liu * 1 2 Junjie Lu * 3 Chaofeng Qu 4 Zhaoling Chen 1 Zefan Cai 5 Jason Klein Liu 4 Chonghan Liu 4 Yunhui Xia 4 Li Zhao 6 Jiang Bian 6 Chuheng Zhang 6 Wei Shen 4 Zhouhan Lin 2 7  \nAbstract  \nCurrent approaches for training Process Reward Models (PRMs) often involve decomposing responses into multiple reasoning steps using rulebased techniques, such as using predefined placeholder tokens or setting the reasoning step’s length to a fixed size. These approaches overlook the fact that certain words don’t usually indicate true decision points. To address this, we propose AdaptiveStep, a method that divides reasoning steps based on the model’s confidence in predicting the next word, offering more information on decision-making at each step, improving downstream tasks like reward model training. Moreover, our method requires no manual annotation.  \nExperiments with AdaptiveStep-trained PRMs in mathematical reasoning and code generation show that the outcome PRM achieves state-of-the-art Best-of-N performance, surpassing greedy search strategy with token-level value-guided decoding, while also reducing construction costs by over 30% compared to existing open-source PRMs.  \nWe also provide a thorough analysis and case study on its performance, transferability, and generalization capabilities. We provide our code on [https://github.com/Lux0926/ASPRM](https://github.com/Lux0926/ASPRM).  \n1. Introduction  \nLarge language models (LLMs) have demonstrated exceptional performance across various tasks. However, even most advanced LLMs struggle to generate correct solutions when facing complex reasoning problems, such as mathe-  \n*Equal contribution 1Nanjing University 2 Shanghai Innovation Institute 3University of Technology Sydney 4Independent 5UW-Madison 6MSRA 7 Shanghai Jiaotong University. Correspondence to: Chuheng Zhang \u003C[zhangchuheng123@live.com](zhangchuheng123@live.com) >, Wei Shen \u003C[shenwei0917@126.com](shenwei0917@126.com) >, Zhouhan Lin \u003C[lin.zhouhan@gmail.com](lin.zhouhan@gmail.com) >.  \nProceedings of the 42 nd International Conference on Machine Learning, Vancouver, Canada. PMLR 267, 2025 . Copyright 2025 by the author(s) .  \nManually Divided Steps  \nInformative Automatic  \nHighly informative Automatic General  \nFigure 1: Rule-based reasoning step dividing (e.g., using line breaks or a fixed number of tokens) is automated but results in low informativeness at the end of the step and is difficult to apply in domains that are hard to define rules. In contrast, manual step division provides high informativeness but is costly to scale and heavily reliant on the experts’ domain knowledge. AdaptiveStep, which divides steps based on model confidence, addresses these challenges by offering automation, efficiency, high informativeness, and applicability across various domains.  \nmatical reasoning and code generation tasks (Huang et al., 2024; Tyen et al., 2024; Mirzadeh et al., 2024; Shen & Zhang, 2024) . To address these challenges using the stepwise Chain of Thought (CoT) approach (Wei et al., 2023), various strategies have been proposed by the research community (Qin et al., 2024; DeepSeek-AI et al., 2025; Team et al., 2025) . One promising method is training Process Reward Models (PRMs), which offer more fine-grained rewards at each reasoning step compared to Outcome Reward Models (ORMs), guiding the LLM to generate higherquality responses than the original model output (Shao et al., 2024; Sessa et al., 2024; Gao et al., 2024) .  \nHowever, as illustrated in Figure 1, existing PRMs typically divide a model’s response into multiple reasoning steps using rule-based methods, such as chopping with a pre-defined symbol. This results in a series of coarse reasoning step divisions that lack decision-making information at steps (Wanget al., 2024a; Lightman et al., 2023) . Moreover, rule-based methods also face challenges when applied to tasks where  \nthe ","cbCaiuLkvPo9Pcxf","https://ap.wps.com/l/cbCaiuLkvPo9Pcxf","pdf",4217240,16,"English","# Abstract\n# 1. Introduction\n## Large language models and reasoning challenges\n## Process Reward Models and existing step-division methods\n## AdaptiveStep approach and contribution\n## Experiments and evaluation (Best-of-N)","[{\"question\":\"What problem does AdaptiveStep solve in training Process Reward Models?\",\"answer\":\"It addresses the limitation of rule-based decomposition that creates coarse reasoning steps with low decision-making information and difficulties applying them when step boundaries are hard to define.\"},{\"question\":\"How does AdaptiveStep decide where to split reasoning steps?\",\"answer\":\"It divides reasoning steps based on the model’s confidence in predicting the next word, using confidence to identify critical breaking points automatically.\"},{\"question\":\"What experimental results does AdaptiveStep achieve?\",\"answer\":\"AdaptiveStep-trained PRMs reach state-of-the-art Best-of-N performance in mathematical reasoning and code generation, outperforming greedy search with token-level value-guided decoding, and reduce construction costs by over 30% compared with existing open-source PRMs.\"}]","AdaptiveStep - Automatically Dividing Reasoning Step through Model Confidence | PDF"]