[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-328000-105":59,"doc-detail-328000-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","multi-turn-on-policy-distillation-with-prefix-replay-abstract","Multi-Turn On-Policy Distillation with Prefix Replay - Abstract","","We study on-policy distillation (OPD) for agentic tasks where an LLM agent interacts with an environment over multiple turns and a student imitates a teacher using multi-turn histories. Fully online OPD is costly because each update needs fresh student rollouts and teacher queries at visited histories. Replayed-Prefix On-Policy Distillation (ReOPD) avoids new environment interaction by replaying pre-collected teacher trajectories as prefixes: the student acts at selected steps while the teacher provides dense supervision per step. We identify a prefix trap in multi-turn OPD, where student-on-policy histories can make teacher guidance unreliable, creating a shift between student occupancy and teacher reliability. ReOPD treats this as reliability-aware prefix distribution design with a step-decaying sampling schedule emphasizing early, lower-shift prefixes. Across multi-turn mathematical reasoning with Python and search environments spanning multiple teacher and student model scales, ReOPD preserves or improves OPD-level accuracy, uses zero tool calls during student training, and is at least 4× faster per training step. ReOPD converts expensive agent-environment interaction into a reusable offline resource, enabling scalable distillation across tools, tasks, and environments.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/multi-turn-on-policy-distillation-with-prefix-replay-abstract/328000/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/multi-turn-on-policy-distillation-with-prefix-replay-abstract/328000.png","ImageObject",300,407,{"name":92,"@type":93},"River Wang","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-24","2026-09-21",true,{"@type":102,"interactionType":103,"userInteractionCount":44},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What problem does ReOPD address compared with fully online OPD?","Question",{"text":112,"@type":113},"Fully online OPD requires fresh student rollouts and teacher queries at visited histories for each update. ReOPD avoids this by replaying pre-collected teacher trajectories as prefixes, providing per-step supervision without new environment interaction during student training.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"What is the 'prefix trap' in multi-turn on-policy distillation?",{"text":117,"@type":113},"Improving student on-policy behavior across histories can increase relevance to the student, but may force teacher queries on histories where the teacher’s target is unreliable. This creates a two-sided distribution shift between student occupancy and teacher reliability.",{"name":119,"@type":110,"acceptedAnswer":120},"How does ReOPD handle reliability in prefix selection?",{"text":121,"@type":113},"ReOPD uses a reliability-aware prefix distribution design with a simple step-decaying sampling schedule that emphasizes early prefixes, which correspond to lower distribution shift and higher reliability.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},328000,1790352925,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":44,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":139,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":144,"read_time":26,"preview_page_count":4},1099514067438,"https://ap-avatar.wpscdn.com/avatar/100002539ee87300030?x-image-process=image/resize,m_fixed,w_180,h_180&k=1780474512215547542","Multi-Turn On-Policy Distillation with Prefix Replay Baohao Liao∗†12 Hanze Dong∗1 Christof Monz2 Xinxing Xu 1 Li Dong 1 Furu Wei 1  \n1 Microsoft Research 2 University of Amsterdam  \n[https://aka.ms/GeneralAI](https://aka.ms/GeneralAI)  \narXiv :2607 .04763v 1 [ cs .LG] 6 Jul 2026  \nAccuracy (%)  \n55  \n50  \n45  \n40  \n35  \n30  \nAbstract  \nWe study on-policy distillation (OPD) for agentic tasks, where an LLM agent interacts with an environment over multiple turns and a student imitates a teacher over these multi-turn interaction histories. Fully online OPD is costly because each update requires fresh student rollouts through the environment and teacher queries at visited histories. We propose Replayed-Prefix On-Policy Distillation (ReOPD), an off-environment alternative that reuses pre-collected teacher trajectories as replayed prefixes: the student acts at selected steps, while the teacher provides dense per-step supervision without executing new environment interactions. We show that multi-turn OPD introduces a prefix trap: making histories more student-on-policy improves relevance to the student, but can query the teacher on histories where its target is unreliable. This creates a two-sided distribution shift between student occupancy and teacher reliability. ReOPD addresses this by treating multi-turn OPD as a reliability-aware prefix distribution design and implements it with a simple step-decaying sampling schedule that emphasizes early, lower-shift prefixes. Across mathematical reasoning with Python and search environments over multiple teacher and student model scales, ReOPD preserves or improves OPD-level accuracy, uses zero tool calls during student training, and is at least 4 × faster per training step than OPD. ReOPD therefore turns expensive agent-environment interaction into a reusable offline resource, enabling scalable distillation across tools, tasks, and environments.  \n\n| \u003Cbr>53.7  SFT |  |  |  |  |  |  |  |  |  |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| 51.0 |  |  |  |  |  |  OPD\u003Cbr> ReOPD |  |  |  |\n|  |  |  |  |  |  |  |  |  |  |\n|  |  |  |  |  |  |  |  |  |  |\n| 45.0 |  |  |  |  |  |  |  |  |  |\n|  |  |  |  |  |  |  |  |  |  |\n|  |  |  |  | 39.1 39.0 |  |  |  |  |  |\n|  |  |  |  |  |  |  |  |  |  |\n|  |  |  |  |  |  |  |  |  |  |\n|  |  |  |  |  | 32.1 |  |  |  |  |\n|  |  |  |  |  |  |  |  |  |  |\n\nMath with Python Search  \nTime (s) for rollout / step  \n175  \n150  \n125  \n100  \n75  \n50  \n25  \n0  \n\n|  |  |  |  |  |  |  |  |\n| --- | --- | --- | --- | --- | --- | --- | --- |\n|  |  |  |  |  |  |  |  |\n|  |  |  | 4.2× |  |  |  |  |\n|  |  |  |  |  |  |  |  |\n|  |  |  |  |  | 64 |  | |\n|  |  |  |  |  |  |  |  |\n|  |  | 40 |  |  |  |  |  |\n|  |  |  |  |  |  |  |  |\n|  |  |  |  |  |  | 7 |  |\n|  |  |  |  |  |  |  |  |\n\n169  \n9. 1×  \nMath with Python Search  \n\\#Tool ca l ls / trajectory  \n15.0  \n12.5  \n10.0  \n7.5  \n5.0  \n2.5  \n0.0  \nMath with Python Search  \nFigure 1: ReOPD keeps the benefits of on-policy distillation while removing environment interaction. ReOPD matches or improves OPD accuracy, but trains much faster per step and eliminates tool calls during training by replaying teacher-recorded prefixes instead of executing fresh environment rollouts. The student/teacher models are Qwen3-4B-Instruct-2507 and Qwen3-8B, respectively.  \n∗ Equal Contribution.  \n†Work done during an internship at Microsoft.  \nstudent action  \n􀀢!  \nstudent agent  \n􀀣! observation  \nStudent rollout (online)  \n􀀣\" 􀀢\"  \nEnvironment  \n(always alive)  \nTeacher queried OPD update  \nteacher agent  \n􀀣\\# 􀀢\\# … 􀀣$ 􀀢$  \n􀀢$\" 􀀢$\\# … 􀀢$$  \ntargets for student’s actions  \nTurn: 1  \n􀀥􀀥  \n2 ⋯ 􀀤 ⋯ 􀀥 − 1 􀀥  \nUniform weights  \nAll steps weighted equally  \n􀀣$ student action 􀀣$ teacher action  \n􀀣%$  \ntarget from teacher for student’s action  \n􀀢!… 􀀣$ 􀀢$  \nTeacher rollout (once or during RL)  \nOffline prefix replay Reliability-aware weighting  \nteacher action  \n􀀢!  \nteacher agent  \n􀀢! 􀀣! 􀀢\"  \n􀀢! 􀀣! 􀀢\"  \n􀀢! 􀀣! 􀀢\"  \nstudent agent  \nacts at ste","cbCaieC4LBeNPM6L","https://ap.wps.com/l/cbCaieC4LBeNPM6L","pdf",1095193,24,"English","# Abstract\n# Introduction","[{\"question\":\"What problem does ReOPD address compared with fully online OPD?\",\"answer\":\"Fully online OPD requires fresh student rollouts and teacher queries at visited histories for each update. ReOPD avoids this by replaying pre-collected teacher trajectories as prefixes, providing per-step supervision without new environment interaction during student training.\"},{\"question\":\"What is the 'prefix trap' in multi-turn on-policy distillation?\",\"answer\":\"Improving student on-policy behavior across histories can increase relevance to the student, but may force teacher queries on histories where the teacher’s target is unreliable. This creates a two-sided distribution shift between student occupancy and teacher reliability.\"},{\"question\":\"How does ReOPD handle reliability in prefix selection?\",\"answer\":\"ReOPD uses a reliability-aware prefix distribution design with a simple step-decaying sampling schedule that emphasizes early prefixes, which correspond to lower distribution shift and higher reliability.\"}]","Multi-Turn On-Policy Distillation with Prefix Replay - Abstract | PDF",1789978469]